Fixed unicode issues with sqlite3 database queries

sickbeard/db.py

@@ -65,6 +65,7 @@ class DBConnection(object):
         self.close()
         self.connection = sqlite3.connect(dbFilename(self.filename, self.suffix), 20, check_same_thread=False)
         self.connection.isolation_level = None
+        self.connection.text_factory = self._unicode_text_factory
 
         if self.row_type == "dict":
             self.connection.row_factory = self._dict_factory
@@ -94,9 +95,17 @@ class DBConnection(object):
             cursor.close()
 
     def _execute(self, cursor, query, args):
+        def convert(x):
+            if isinstance(x, basestring):
+                try:
+                    x = unicode(x).decode(sickbeard.SYS_ENCODING)
+                except:
+                    pass
+            return x
         try:
-            if args == None:
+            if not args:
                 return cursor.execute(query)
+            #args = map(convert, args)
             return cursor.execute(query, args)
         except sqlite3.OperationalError as e:
             logger.log(u"DB error: " + ex(e), logger.ERROR)
@@ -245,6 +254,9 @@ class DBConnection(object):
             columns[column['name']] = {'type': column['type']}
         return columns
 
+    def _unicode_text_factory(self, x):
+        return unicode(x, 'utf-8')
+
     # http://stackoverflow.com/questions/3300464/how-can-i-get-dict-from-sqlite-query
     def _dict_factory(self, cursor, row):
         d = {}

sickbeard/scene_exceptions.py

@@ -179,25 +179,25 @@ def retrieve_exceptions():
                 logger.log(u"Check scene exceptions update failed. Unable to get URL: " + url, logger.ERROR)
                 continue
 
-            else:
-                setLastRefresh(sickbeard.indexerApi(indexer).name)
+            setLastRefresh(sickbeard.indexerApi(indexer).name)
 
-                # each exception is on one line with the format indexer_id: 'show name 1', 'show name 2', etc
-                for cur_line in url_data.splitlines():
-                    cur_line = cur_line.decode('utf-8')
-                    indexer_id, sep, aliases = cur_line.partition(':')  # @UnusedVariable
+            # each exception is on one line with the format indexer_id: 'show name 1', 'show name 2', etc
+            for cur_line in url_data.splitlines():
+                indexer_id, sep, aliases = cur_line.partition(':')  # @UnusedVariable
 
-                    if not aliases:
-                        continue
+                if not aliases:
+                    continue
 
-                    indexer_id = int(indexer_id)
+                indexer_id = int(indexer_id)
 
-                    # regex out the list of shows, taking \' into account
-                    # alias_list = [re.sub(r'\\(.)', r'\1', x) for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
-                    alias_list = [{re.sub(r'\\(.)', r'\1', x): -1} for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
-                    exception_dict[indexer_id] = alias_list
-                    del alias_list
-                del url_data
+                # regex out the list of shows, taking \' into account
+                # alias_list = [re.sub(r'\\(.)', r'\1', x) for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
+                alias_list = [{re.sub(r'\\(.)', r'\1', x): -1} for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
+                exception_dict[indexer_id] = alias_list
+                del alias_list
+
+            # cleanup
+            del url_data
 
     # XEM scene exceptions
     _xem_exceptions_fetcher()
@@ -233,9 +233,6 @@ def retrieve_exceptions():
 
             # if this exception isn't already in the DB then add it
             if cur_exception not in existing_exceptions:
-
-                cur_exception = ek.ss(cur_exception)
-
                 myDB.action("INSERT INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?)",
                             [cur_indexer_id, cur_exception, curSeason])
                 changed_exceptions = True
@@ -267,8 +264,6 @@ def update_scene_exceptions(indexer_id, scene_exceptions, season=-1):
         exceptionsCache[indexer_id][season] = scene_exceptions
 
     for cur_exception in scene_exceptions:
-        cur_exception = ek.ss(cur_exception)
-
         myDB.action("INSERT INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?)",
                     [indexer_id, cur_exception, season])
 

tests/encoding_tests.py

@@ -15,11 +15,12 @@ DEBUG = VERBOSE = False
 
 class EncodingTests(unittest.TestCase):
     def test_encoding(self):
-        strings = ['Les Enfants De La Télé', u'\x89']
+        strings = [u'הערוץ הראשון', 'Les Enfants De La Télé', u'\x89']
 
         for s in strings:
             try:
                 print ek.ss(s)
+                print unicode(s).decode('UTF-8')
             except Exception, e:
                 print ex(e)