From 86e7912c412b2b8f6e4678d14d575a8553e09ac0 Mon Sep 17 00:00:00 2001 From: echel0n Date: Fri, 5 Dec 2014 20:13:50 -0800 Subject: [PATCH] Possible fix for issues #1016, #993, and #1024 - Unicode decode/encode issues --- sickbeard/encodingKludge.py | 14 +++++++------- sickbeard/exceptions.py | 6 +++--- sickbeard/failed_history.py | 4 ++-- sickbeard/history.py | 4 ++-- sickbeard/logger.py | 3 ++- sickbeard/notifiers/emailnotify.py | 10 +++++----- sickbeard/notifiers/kodi.py | 14 +++++++------- sickbeard/notifiers/plex.py | 4 ++-- sickbeard/nzbSplitter.py | 4 +--- sickbeard/scene_exceptions.py | 6 +++--- sickbeard/show_name_helpers.py | 2 +- sickbeard/tvcache.py | 4 ++-- sickbeard/webserve.py | 4 ++-- tests/encoding_tests.py | 16 +++++++++------- 14 files changed, 48 insertions(+), 47 deletions(-) diff --git a/sickbeard/encodingKludge.py b/sickbeard/encodingKludge.py index 2ea13606..1f60633e 100644 --- a/sickbeard/encodingKludge.py +++ b/sickbeard/encodingKludge.py @@ -20,16 +20,16 @@ import os import traceback import sickbeard -from sickbeard import logger - import six import chardet +from sickbeard import logger + # This module tries to deal with the apparently random behavior of python when dealing with unicode <-> utf-8 # encodings. It tries to just use unicode, but if that fails then it tries forcing it to utf-8. Any functions # which return something should always return unicode. -def toUnicode(x): +def _toUnicode(x): try: if isinstance(x, unicode): return x @@ -51,10 +51,10 @@ def toUnicode(x): except: logger.log('Unable to decode value "%s..." : %s ' % (repr(x)[:20], traceback.format_exc()), logger.WARNING) ascii_text = str(x).encode('string_escape') - return toUnicode(ascii_text) + return _toUnicode(ascii_text) def ss(x): - u_x = toUnicode(x) + u_x = _toUnicode(x) try: return u_x.encode(sickbeard.SYS_ENCODING) @@ -69,7 +69,7 @@ def fixListEncodings(x): if not isinstance(x, (list, tuple)): return x else: - return filter(lambda x: x != None, map(toUnicode, x)) + return filter(lambda x: x != None, map(_toUnicode, x)) def ek(func, *args, **kwargs): @@ -81,6 +81,6 @@ def ek(func, *args, **kwargs): if isinstance(result, (list, tuple)): return fixListEncodings(result) elif isinstance(result, str): - return toUnicode(result) + return _toUnicode(result) else: return result diff --git a/sickbeard/exceptions.py b/sickbeard/exceptions.py index db3b5735..1e9114e8 100644 --- a/sickbeard/exceptions.py +++ b/sickbeard/exceptions.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with SickRage. If not, see . -from sickbeard.encodingKludge import toUnicode +from sickbeard import encodingKludge as ek def ex(e): """ @@ -32,11 +32,11 @@ def ex(e): if arg is not None: if isinstance(arg, (str, unicode)): - fixed_arg = toUnicode(arg) + fixed_arg = ek.ss(arg) else: try: - fixed_arg = u"error " + toUnicode(str(arg)) + fixed_arg = u"error " + ek.ss(str(arg)) except: fixed_arg = None diff --git a/sickbeard/failed_history.py b/sickbeard/failed_history.py index 0fc1484b..0fda51ea 100644 --- a/sickbeard/failed_history.py +++ b/sickbeard/failed_history.py @@ -26,7 +26,7 @@ from sickbeard.exceptions import ex, EpisodeNotFoundException from sickbeard.history import dateFormat from sickbeard.common import Quality from sickbeard.common import WANTED, FAILED -from sickbeard.encodingKludge import toUnicode +from sickbeard import encodingKludge as ek def prepareFailedName(release): """Standardizes release name for failed DB""" @@ -36,7 +36,7 @@ def prepareFailedName(release): fixed = fixed.rpartition(".")[0] fixed = re.sub("[\.\-\+\ ]", "_", fixed) - fixed = toUnicode(fixed) + fixed = ek.ss(fixed) return fixed diff --git a/sickbeard/history.py b/sickbeard/history.py index cb1a8486..2df25904 100644 --- a/sickbeard/history.py +++ b/sickbeard/history.py @@ -20,7 +20,7 @@ import db import datetime from sickbeard.common import SNATCHED, SUBTITLED, FAILED, Quality -from sickbeard.encodingKludge import toUnicode +from sickbeard import encodingKludge as ek dateFormat = "%Y%m%d%H%M%S" @@ -28,7 +28,7 @@ dateFormat = "%Y%m%d%H%M%S" def _logHistoryItem(action, showid, season, episode, quality, resource, provider, version=-1): logDate = datetime.datetime.today().strftime(dateFormat) - resource = toUnicode(resource) + resource = ek.ss(resource) myDB = db.DBConnection() myDB.action( diff --git a/sickbeard/logger.py b/sickbeard/logger.py index 463dafac..9ed8f900 100644 --- a/sickbeard/logger.py +++ b/sickbeard/logger.py @@ -26,6 +26,7 @@ import threading import logging import sickbeard +import encodingKludge as ek from sickbeard import classes @@ -282,7 +283,7 @@ class SBRotatingLogHandler(object): meThread = threading.currentThread().getName() message = meThread + u" :: " + toLog - out_line = message.encode('utf-8') + out_line = ek.ss(message) sb_logger = logging.getLogger('sickbeard') setattr(sb_logger, 'db', lambda *args: sb_logger.log(DB, *args)) diff --git a/sickbeard/notifiers/emailnotify.py b/sickbeard/notifiers/emailnotify.py index 1dac6758..74d292be 100644 --- a/sickbeard/notifiers/emailnotify.py +++ b/sickbeard/notifiers/emailnotify.py @@ -29,7 +29,7 @@ import sickbeard from sickbeard import logger, common from sickbeard import db -from sickbeard.encodingKludge import toUnicode +from sickbeard import encodingKludge as ek from sickbeard.exceptions import ex @@ -51,7 +51,7 @@ class EmailNotifier: ep_name: The name of the episode that was snatched title: The title of the notification (optional) """ - ep_name = toUnicode(ep_name) + ep_name = ek.ss(ep_name) if sickbeard.EMAIL_NOTIFY_ONSNATCH: show = self._parseEp(ep_name) @@ -86,7 +86,7 @@ class EmailNotifier: ep_name: The name of the episode that was downloaded title: The title of the notification (optional) """ - ep_name = toUnicode(ep_name) + ep_name = ek.ss(ep_name) if sickbeard.EMAIL_NOTIFY_ONDOWNLOAD: show = self._parseEp(ep_name) @@ -121,7 +121,7 @@ class EmailNotifier: ep_name: The name of the episode that was downloaded lang: Subtitle language wanted """ - ep_name = toUnicode(ep_name) + ep_name = ek.ss(ep_name) if sickbeard.EMAIL_NOTIFY_ONSUBTITLEDOWNLOAD: show = self._parseEp(ep_name) @@ -198,7 +198,7 @@ class EmailNotifier: return False def _parseEp(self, ep_name): - ep_name = toUnicode(ep_name) + ep_name = ek.ss(ep_name) sep = " - " titles = ep_name.split(sep) diff --git a/sickbeard/notifiers/kodi.py b/sickbeard/notifiers/kodi.py index 12468b32..4bbf879f 100644 --- a/sickbeard/notifiers/kodi.py +++ b/sickbeard/notifiers/kodi.py @@ -26,7 +26,7 @@ import sickbeard from sickbeard import logger from sickbeard import common from sickbeard.exceptions import ex -from sickbeard.encodingKludge import toUnicode +from sickbeard import encodingKludge as ek try: @@ -236,9 +236,9 @@ class KODINotifier: base64string = base64.encodestring('%s:%s' % (username, password))[:-1] authheader = "Basic %s" % base64string req.add_header("Authorization", authheader) - logger.log(u"Contacting KODI (with auth header) via url: " + toUnicode(url), logger.DEBUG) + logger.log(u"Contacting KODI (with auth header) via url: " + ek.ss(url), logger.DEBUG) else: - logger.log(u"Contacting KODI via url: " + toUnicode(url), logger.DEBUG) + logger.log(u"Contacting KODI via url: " + ek.ss(url), logger.DEBUG) response = urllib2.urlopen(req) result = response.read().decode(sickbeard.SYS_ENCODING) @@ -248,7 +248,7 @@ class KODINotifier: return result except (urllib2.URLError, IOError), e: - logger.log(u"Warning: Couldn't contact KODI HTTP at " + toUnicode(url) + " " + ex(e), + logger.log(u"Warning: Couldn't contact KODI HTTP at " + ek.ss(url) + " " + ex(e), logger.WARNING) return False @@ -379,9 +379,9 @@ class KODINotifier: base64string = base64.encodestring('%s:%s' % (username, password))[:-1] authheader = "Basic %s" % base64string req.add_header("Authorization", authheader) - logger.log(u"Contacting KODI (with auth header) via url: " + toUnicode(url), logger.DEBUG) + logger.log(u"Contacting KODI (with auth header) via url: " + ek.ss(url), logger.DEBUG) else: - logger.log(u"Contacting KODI via url: " + toUnicode(url), logger.DEBUG) + logger.log(u"Contacting KODI via url: " + ek.ss(url), logger.DEBUG) try: response = urllib2.urlopen(req) @@ -401,7 +401,7 @@ class KODINotifier: return False except IOError, e: - logger.log(u"Warning: Couldn't contact KODI JSON API at " + toUnicode(url) + " " + ex(e), + logger.log(u"Warning: Couldn't contact KODI JSON API at " + ek.ss(url) + " " + ex(e), logger.WARNING) return False diff --git a/sickbeard/notifiers/plex.py b/sickbeard/notifiers/plex.py index d0d20688..f9ddeff8 100644 --- a/sickbeard/notifiers/plex.py +++ b/sickbeard/notifiers/plex.py @@ -25,7 +25,7 @@ import sickbeard from sickbeard import logger from sickbeard import common from sickbeard.exceptions import ex -from sickbeard.encodingKludge import toUnicode +from sickbeard import encodingKludge as ek from sickbeard.notifiers.kodi import KODINotifier @@ -77,7 +77,7 @@ class PLEXNotifier(KODINotifier): return self._notify_pmc("Testing Plex notifications from SickRage", "Test Notification", host, username, password, force=True) - def update_library(self): + def update_library(self, showName=None): """Handles updating the Plex Media Server host via HTTP API Plex Media Server currently only supports updating the whole video library and not a specific path. diff --git a/sickbeard/nzbSplitter.py b/sickbeard/nzbSplitter.py index 39d1df63..1cbf10c4 100644 --- a/sickbeard/nzbSplitter.py +++ b/sickbeard/nzbSplitter.py @@ -29,8 +29,6 @@ from sickbeard import encodingKludge as ek from sickbeard.exceptions import ex from name_parser.parser import NameParser, InvalidNameException, InvalidShowException -from sickbeard.encodingKludge import toUnicode - def getSeasonNZBs(name, urlData, season): try: @@ -85,7 +83,7 @@ def createNZBString(fileElements, xmlns): for curFile in fileElements: rootElement.append(stripNS(curFile, xmlns)) - return xml.etree.ElementTree.tostring(toUnicode(rootElement)) + return xml.etree.ElementTree.tostring(ek.ss(rootElement)) def saveNZB(nzbName, nzbString): diff --git a/sickbeard/scene_exceptions.py b/sickbeard/scene_exceptions.py index f4154449..98c2389e 100644 --- a/sickbeard/scene_exceptions.py +++ b/sickbeard/scene_exceptions.py @@ -27,7 +27,7 @@ from sickbeard import helpers from sickbeard import name_cache from sickbeard import logger from sickbeard import db -from sickbeard.encodingKludge import toUnicode +from sickbeard import encodingKludge as ek exception_dict = {} anidb_exception_dict = {} @@ -234,7 +234,7 @@ def retrieve_exceptions(): # if this exception isn't already in the DB then add it if cur_exception not in existing_exceptions: - cur_exception = toUnicode(cur_exception) + cur_exception = ek.ss(cur_exception) myDB.action("INSERT INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?)", [cur_indexer_id, cur_exception, curSeason]) @@ -267,7 +267,7 @@ def update_scene_exceptions(indexer_id, scene_exceptions, season=-1): exceptionsCache[indexer_id][season] = scene_exceptions for cur_exception in scene_exceptions: - cur_exception = toUnicode(cur_exception) + cur_exception = ek.ss(cur_exception) myDB.action("INSERT INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?)", [indexer_id, cur_exception, season]) diff --git a/sickbeard/show_name_helpers.py b/sickbeard/show_name_helpers.py index 736cbf42..357649a0 100644 --- a/sickbeard/show_name_helpers.py +++ b/sickbeard/show_name_helpers.py @@ -234,7 +234,7 @@ def isGoodResult(name, show, log=True, season=-1): all_show_names = allPossibleShowNames(show, season=season) showNames = map(sanitizeSceneName, all_show_names) + all_show_names - showNames += map(ek.toUnicode, all_show_names) + showNames += map(ek.ss, all_show_names) for curName in set(showNames): if not show.is_anime: diff --git a/sickbeard/tvcache.py b/sickbeard/tvcache.py index 17443c19..8e04ab70 100644 --- a/sickbeard/tvcache.py +++ b/sickbeard/tvcache.py @@ -34,7 +34,7 @@ from sickbeard.exceptions import AuthException from sickbeard.rssfeeds import RSSFeeds from sickbeard import clients from name_parser.parser import NameParser, InvalidNameException, InvalidShowException -from sickbeard.encodingKludge import toUnicode +from sickbeard import encodingKludge as ek class CacheDBConnection(db.DBConnection): def __init__(self, providerName): @@ -277,7 +277,7 @@ class TVCache(): # get quality of release quality = parse_result.quality - name = toUnicode(name) + name = ek.ss(name) # get release group release_group = parse_result.release_group diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index f9f9c1d5..a47904fb 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -578,7 +578,7 @@ class IndexerWebUI(MainHandler): def _munge(string): - return ek.toUnicode(string).encode('utf-8', 'xmlcharrefreplace') + return ek.ss(string).encode('utf-8', 'xmlcharrefreplace') def _getEpisode(show, season=None, episode=None, absolute=None): if show is None: @@ -3315,7 +3315,7 @@ class ErrorLogs(MainHandler): for x in reversed(data): - x = ek.toUnicode(x) + x = ek.ss(x) match = re.match(regex, x) if match: diff --git a/tests/encoding_tests.py b/tests/encoding_tests.py index 60851e68..513ca9c1 100644 --- a/tests/encoding_tests.py +++ b/tests/encoding_tests.py @@ -1,12 +1,12 @@ +# coding=utf-8 import unittest -#import test_lib as test import sys, os.path sys.path.append(os.path.abspath('..')) sys.path.append(os.path.abspath('../lib')) import sickbeard -from encodingKludge import toUnicode +from sickbeard import encodingKludge as ek from sickbeard.exceptions import ex sickbeard.SYS_ENCODING = 'UTF-8' @@ -15,11 +15,13 @@ DEBUG = VERBOSE = False class EncodingTests(unittest.TestCase): def test_encoding(self): - s = u'\x89' - try: - print toUnicode(s).encode('utf-8', 'xmlcharrefreplace') - except Exception, e: - print ex(e) + strings = ['Les Enfants De La Télé', u'\x89'] + + for s in strings: + try: + print ek.ss(s) + except Exception, e: + print ex(e) if __name__ == "__main__": print "=================="