moparisthebest/SickRage
1
0
Fork 0
mirror of https://github.com/moparisthebest/SickRage synced 2024-12-04 15:12:23 -05:00
Code Issues Releases Wiki Activity

Fixed encoding/decoding issues with WebUI and IndexerAPI's

Browse Source
This commit is contained in:
echel0n 2014-12-14 19:35:47 -08:00
parent 2f4b4b7669
commit 956e16f481
6 changed files with 104 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/tvdb_api/tvdb_api.py
View File

@ -619,7 +619,7 @@ class Tvdb:
raise tvdb_error("Bad zip file received from thetvdb.com, could not read it")
else:
try:
return xmltodict.parse(resp.text.rstrip("\r"), postprocessor=process)
return xmltodict.parse(resp.content.decode('utf-8'), postprocessor=process)
except:
return dict([(u'data', None)])

2
lib/tvrage_api/tvrage_api.py
View File

@ -498,7 +498,7 @@ class TVRage:
return (key, value)
try:
return xmltodict.parse(resp.text.rstrip("\r"), postprocessor=remap_keys)
return xmltodict.parse(resp.content.decode('utf-8'), postprocessor=remap_keys)
except:
return dict([(u'data', None)])

63
sickbeard/encodingKludge.py
View File

@ -26,60 +26,32 @@ import unicodedata
from string import ascii_letters, digits
from sickbeard import logger
from unidecode import unidecode
def toSafeString(original):
valid_chars = "-_.() %s%s" % (ascii_letters, digits)
cleaned_filename = unicodedata.normalize('NFKD', _toUnicode(original)).encode('ASCII', 'ignore')
valid_string = ''.join(c for c in cleaned_filename if c in valid_chars)
return ' '.join(valid_string.split())
def simplifyString(original):
string = stripAccents(original.lower())
string = toSafeString(' '.join(re.split('\W+', string)))
split = re.split('\W+|_', string.lower())
return _toUnicode(' '.join(split))
def _toUnicode(x):
if isinstance(x, unicode):
try:
if not isinstance(x, unicode):
if chardet.detect(x).get('encoding') == 'utf-8':
x = x.decode('utf-8')
elif isinstance(x, str):
x = x.decode(sickbeard.SYS_ENCODING)
finally:
return x
else:
try:
return six.text_type(x)
except:
try:
if chardet.detect(x).get('encoding') == 'utf-8':
return x.decode('utf-8')
if isinstance(x, str):
try:
return x.decode(sickbeard.SYS_ENCODING)
except UnicodeDecodeError:
raise
return x
except:
return x
def ss(x):
u_x = _toUnicode(x)
x = _toUnicode(x)
try:
u_x_encoded = u_x.encode(sickbeard.SYS_ENCODING, 'xmlcharrefreplace')
except:
try:
u_x_encoded = u_x.encode(sickbeard.SYS_ENCODING)
except:
try:
u_x_encoded = u_x.encode(sickbeard.SYS_ENCODING, 'replace')
x = x.encode(sickbeard.SYS_ENCODING)
except:
try:
u_x_encoded = u_x.encode('utf-8', 'replace')
except:
try:
u_x_encoded = str(x)
except:
u_x_encoded = x
return u_x_encoded
x = x.encode(sickbeard.SYS_ENCODING, 'ignore')
except:
x = x.encode('utf-8', 'ignore')
finally:
return x
def fixListEncodings(x):
if not isinstance(x, (list, tuple)):
@ -99,7 +71,4 @@ def ek(func, *args, **kwargs):
elif isinstance(result, str):
return _toUnicode(result)
else:
return result
def stripAccents(s):
return ''.join((c for c in unicodedata.normalize('NFD', _toUnicode(s)) if unicodedata.category(c) != 'Mn'))
return result

3
sickbeard/logger.py
View File

@ -283,13 +283,14 @@ class SBRotatingLogHandler(object):
meThread = threading.currentThread().getName()
message = meThread + u" :: " + toLog
out_line = ek.ss(message)
out_line = message
sb_logger = logging.getLogger('sickbeard')
sub_logger = logging.getLogger('subliminal')
imdb_logger = logging.getLogger('imdbpy')
tornado_logger = logging.getLogger('tornado')
feedcache_logger = logging.getLogger('feedcache')
setattr(sb_logger, 'db', lambda *args: sb_logger.log(DB, *args))
# filtering

99
sickbeard/webserve.py
View File

@ -67,7 +67,8 @@ try:
except ImportError:
import xml.etree.ElementTree as etree
from Cheetah.Template import Template
from Cheetah.Template import Template as CheetahTemplate
from Cheetah.Filters import Filter as CheetahFilter
from tornado.routes import route
from tornado.web import RequestHandler, HTTPError, authenticated, asynchronous
@ -78,10 +79,23 @@ from concurrent.futures import ThreadPoolExecutor
route_locks = {}
class html_entities(CheetahFilter):
def filter(self, val, **dummy_kw):
"""Filter incoming strings so they use HTML entity characters"""
if isinstance(val, unicode):
filtered = val.encode('ascii', 'xmlcharrefreplace')
elif val is None:
filtered = ''
elif isinstance(val, str):
filtered = val.decode('utf-8').encode('ascii', 'xmlcharrefreplace')
else:
filtered = self.filter(str(val))
return filtered
class PageTemplate(Template):
class PageTemplate(CheetahTemplate):
def __init__(self, rh, *args, **kwargs):
kwargs['file'] = os.path.join(sickbeard.PROG_DIR, "gui/" + sickbeard.GUI_NAME + "/interfaces/default/", kwargs['file'])
kwargs['filter'] = html_entities
super(PageTemplate, self).__init__(*args, **kwargs)
self.sbRoot = sickbeard.WEB_ROOT
@ -127,7 +141,6 @@ class PageTemplate(Template):
kwargs['cacheDirForModuleFiles'] = os.path.join(sickbeard.CACHE_DIR, 'cheetah')
return super(PageTemplate, self).compile(*args, **kwargs)
class BaseHandler(RequestHandler):
def __init__(self, *args, **kwargs):
super(BaseHandler, self).__init__(*args, **kwargs)
@ -222,8 +235,6 @@ class WebHandler(BaseHandler):
def async_done(self, results):
try:
if results:
results = ek.ss(results)
self.finish(results)
except:
logger.log('Failed sending webui reponse: %s' % (traceback.format_exc()), logger.DEBUG)
@ -337,7 +348,7 @@ class WebRoot(WebHandler):
else:
t.apikey = "api key not generated"
return t
return t.respond()
def showPoster(self, show=None, which=None):
# Redirect initial poster/banner thumb to default images
@ -513,7 +524,7 @@ class WebRoot(WebHandler):
else:
t.layout = sickbeard.COMING_EPS_LAYOUT
return t
return t.respond()
# Raw iCalendar implementation by Pedro Jose Pereira Vieito (@pvieito).
#
@ -642,7 +653,7 @@ class Home(WebRoot):
t.submenu = self.HomeMenu()
t.subject = subject
t.message = message
return t
return t.respond()
def _getEpisode(self, show, season=None, episode=None, absolute=None):
if show is None:
@ -682,7 +693,7 @@ class Home(WebRoot):
t.submenu = self.HomeMenu()
return t
return t.respond()
def is_alive(self, *args, **kwargs):
if 'callback' in kwargs and '_' in kwargs:
@ -1004,7 +1015,7 @@ class Home(WebRoot):
# restart
sickbeard.events.put(sickbeard.events.SystemEvent.RESTART)
return t
return t.respond()
def updateCheck(self, pid=None):
if str(pid) != str(sickbeard.PID):
@ -1023,7 +1034,7 @@ class Home(WebRoot):
sickbeard.events.put(sickbeard.events.SystemEvent.RESTART)
t = PageTemplate(rh=self, file="restart_bare.tmpl")
return t
return t.respond()
else:
return self._genericMessage("Update Failed",
"Update wasn't successful, not restarting. Check your log for more information.")
@ -1155,7 +1166,7 @@ class Home(WebRoot):
t.scene_absolute_numbering = get_scene_absolute_numbering_for_show(indexerid, indexer)
t.xem_absolute_numbering = get_xem_absolute_numbering_for_show(indexerid, indexer)
return t
return t.respond()
def plotDetails(self, show, season, episode):
@ -1235,7 +1246,7 @@ class Home(WebRoot):
t.show = showObj
t.scene_exceptions = get_scene_exceptions(showObj.indexerid)
return t
return t.respond()
flatten_folders = config.checkbox_to_value(flatten_folders)
dvdorder = config.checkbox_to_value(dvdorder)
@ -1691,7 +1702,7 @@ class Home(WebRoot):
t.ep_obj_list = ep_obj_rename_list
t.show = showObj
return t
return t.respond()
def doRename(self, show=None, eps=None):
@ -1960,7 +1971,7 @@ class HomePostProcess(Home):
def index(self):
t = PageTemplate(rh=self, file="home_postprocess.tmpl")
t.submenu = self.HomeMenu()
return t
return t.respond()
def processEpisode(self, dir=None, nzbName=None, jobName=None, quiet=None, process_method=None, force=None,
is_priority=None, failed="0", type="auto", *args, **kwargs):
@ -2000,7 +2011,7 @@ class HomeAddShows(Home):
def index(self):
t = PageTemplate(rh=self, file="home_addShows.tmpl")
t.submenu = self.HomeMenu()
return t
return t.respond()
def getIndexerLanguages(self):
result = sickbeard.indexerApi().config['valid_languages']
@ -2132,7 +2143,7 @@ class HomeAddShows(Home):
t.dirList = dir_list
return t
return t.respond()
def newShow(self, show_to_add=None, other_shows=None):
@ -2177,7 +2188,7 @@ class HomeAddShows(Home):
t.provided_indexer = int(indexer or sickbeard.INDEXER_DEFAULT)
t.indexers = sickbeard.indexerApi().indexers
return t
return t.respond()
def recommendedShows(self):
"""
@ -2187,7 +2198,7 @@ class HomeAddShows(Home):
t = PageTemplate(rh=self, file="home_recommendedShows.tmpl")
t.submenu = self.HomeMenu()
return t
return t.respond()
def getRecommendedShows(self):
final_results = []
@ -2237,7 +2248,7 @@ class HomeAddShows(Home):
t = PageTemplate(rh=self, file="home_trendingShows.tmpl")
t.submenu = self.HomeMenu()
return t
return t.respond()
def getTrendingShows(self):
"""
@ -2263,7 +2274,7 @@ class HomeAddShows(Home):
except (traktException, traktAuthException, traktServerBusy) as e:
logger.log(u"Could not connect to Trakt service: %s" % ex(e), logger.WARNING)
return t
return t.respond()
def existingShows(self):
"""
@ -2272,7 +2283,7 @@ class HomeAddShows(Home):
t = PageTemplate(rh=self, file="home_addExistingShow.tmpl")
t.submenu = self.HomeMenu()
return t
return t.respond()
def addTraktShow(self, indexer_id, showName):
if helpers.findCertainShow(sickbeard.showList, int(indexer_id)):
@ -2524,7 +2535,7 @@ class Manage(Home, WebRoot):
def index(self):
t = PageTemplate(rh=self, file="manage.tmpl")
t.submenu = self.ManageMenu()
return t
return t.respond()
def showEpisodeStatuses(self, indexer_id, whichStatus):
@ -2565,7 +2576,7 @@ class Manage(Home, WebRoot):
# if we have no status then this is as far as we need to go
if not status_list:
return t
return t.respond()
myDB = db.DBConnection()
status_results = myDB.select(
@ -2591,7 +2602,7 @@ class Manage(Home, WebRoot):
t.show_names = show_names
t.ep_counts = ep_counts
t.sorted_show_ids = sorted_show_ids
return t
return t.respond()
def changeEpisodeStatuses(self, oldStatus, newStatus, *args, **kwargs):
@ -2672,7 +2683,7 @@ class Manage(Home, WebRoot):
t.whichSubs = whichSubs
if not whichSubs:
return t
return t.respond()
myDB = db.DBConnection()
status_results = myDB.select(
@ -2702,7 +2713,7 @@ class Manage(Home, WebRoot):
t.show_names = show_names
t.ep_counts = ep_counts
t.sorted_show_ids = sorted_show_ids
return t
return t.respond()
def downloadSubtitleMissed(self, *args, **kwargs):
@ -2789,7 +2800,7 @@ class Manage(Home, WebRoot):
t.showCats = showCats
t.showSQLResults = showSQLResults
return t
return t.respond()
def massEdit(self, toEdit=None):
@ -2913,7 +2924,7 @@ class Manage(Home, WebRoot):
t.air_by_date_value = last_air_by_date if air_by_date_all_same else None
t.root_dir_list = root_dir_list
return t
return t.respond()
def massEditSubmit(self, archive_firstmatch=None, paused=None, anime=None, sports=None, scene=None,
@ -3165,7 +3176,7 @@ class Manage(Home, WebRoot):
else:
t.info_download_station = '<p>To have a better experience please set the Download Station alias as <code>download</code>, you can check this setting in the Synology DSM <b>Control Panel</b> > <b>Application Portal</b>. Make sure you allow DSM to be embedded with iFrames too in <b>Control Panel</b> > <b>DSM Settings</b> > <b>Security</b>.</p><br/><p>There is more information about this available <a href="https://github.com/midgetspy/Sick-Beard/pull/338">here</a>.</p><br/>'
return t
return t.respond()
def failedDownloads(self, limit=100, toRemove=None):
@ -3190,7 +3201,7 @@ class Manage(Home, WebRoot):
t.limit = limit
t.submenu = self.ManageMenu()
return t
return t.respond()
@route('/manage/manageSearches(/?.*)')
@ -3209,7 +3220,7 @@ class ManageSearches(Manage):
t.submenu = self.ManageMenu()
return t
return t.respond()
def forceBacklog(self):
# force it to run the next time it looks
@ -3324,7 +3335,7 @@ class History(WebRoot):
{'title': 'Trim History', 'path': 'history/trimHistory'},
]
return t
return t.respond()
def clearHistory(self):
@ -3369,7 +3380,7 @@ class Config(WebRoot):
t = PageTemplate(rh=self, file="config.tmpl")
t.submenu = self.ConfigMenu()
return t
return t.respond()
@route('/config/general(/?.*)')
@ -3380,7 +3391,7 @@ class ConfigGeneral(Config):
def index(self):
t = PageTemplate(rh=self, file="config_general.tmpl")
t.submenu = self.ConfigMenu()
return t
return t.respond()
def generateApiKey(self):
@ -3518,7 +3529,7 @@ class ConfigBackupRestore(Config):
def index(self):
t = PageTemplate(rh=self, file="config_backuprestore.tmpl")
t.submenu = self.ConfigMenu()
return t
return t.respond()
def backup(self, backupDir=None):
@ -3569,7 +3580,7 @@ class ConfigSearch(Config):
def index(self):
t = PageTemplate(rh=self, file="config_search.tmpl")
t.submenu = self.ConfigMenu()
return t
return t.respond()
def saveSearch(self, use_nzbs=None, use_torrents=None, nzb_dir=None, sab_username=None, sab_password=None,
@ -3664,7 +3675,7 @@ class ConfigPostProcessing(Config):
def index(self):
t = PageTemplate(rh=self, file="config_postProcessing.tmpl")
t.submenu = self.ConfigMenu()
return t
return t.respond()
def savePostProcessing(self, naming_pattern=None, naming_multi_ep=None,
@ -3866,7 +3877,7 @@ class ConfigProviders(Config):
def index(self):
t = PageTemplate(rh=self, file="config_providers.tmpl")
t.submenu = self.ConfigMenu()
return t
return t.respond()
def canAddNewznabProvider(self, name):
@ -4308,7 +4319,7 @@ class ConfigNotifications(Config):
def index(self):
t = PageTemplate(rh=self, file="config_notifications.tmpl")
t.submenu = self.ConfigMenu()
return t
return t.respond()
def saveNotifications(self, use_kodi=None, kodi_always_on=None, kodi_notify_onsnatch=None,
@ -4521,7 +4532,7 @@ class ConfigSubtitles(Config):
def index(self):
t = PageTemplate(rh=self, file="config_subtitles.tmpl")
t.submenu = self.ConfigMenu()
return t
return t.respond()
def saveSubtitles(self, use_subtitles=None, subtitles_plugins=None, subtitles_languages=None, subtitles_dir=None,
@ -4589,7 +4600,7 @@ class ConfigAnime(Config):
t = PageTemplate(rh=self, file="config_anime.tmpl")
t.submenu = self.ConfigMenu()
return t
return t.respond()
def saveAnime(self, use_anidb=None, anidb_username=None, anidb_password=None, anidb_use_mylist=None,
@ -4634,7 +4645,7 @@ class ErrorLogs(WebRoot):
t = PageTemplate(rh=self, file="errorlogs.tmpl")
t.submenu = self.ErrorLogsMenu()
return t
return t.respond()
def clearerrors(self):
@ -4693,4 +4704,4 @@ class ErrorLogs(WebRoot):
t.logLines = result
t.minLevel = minLevel
return t
return t.respond()

35
tests/encoding_tests.py
View File

@ -1,4 +1,4 @@
# coding=utf-8
import locale
import unittest
import sys, os.path
@ -8,17 +8,40 @@ sys.path.append(os.path.abspath('../lib'))
import sickbeard
from sickbeard import encodingKludge as ek
from sickbeard.exceptions import ex
sickbeard.SYS_ENCODING = 'UTF-8'
from sickbeard.helpers import sanitizeFileName
class EncodingTests(unittest.TestCase):
def test_encoding(self):
strings = [u'Les Enfants De La Télé']
rootDir = 'C:\\Temp\\TV'
strings = [u'Les Enfants De La T\xe9l\xe9']
sickbeard.SYS_ENCODING = None
try:
locale.setlocale(locale.LC_ALL, "")
sickbeard.SYS_ENCODING = locale.getpreferredencoding()
except (locale.Error, IOError):
pass
# For OSes that are poorly configured I'll just randomly force UTF-8
if not sickbeard.SYS_ENCODING or sickbeard.SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
sickbeard.SYS_ENCODING = 'UTF-8'
if not hasattr(sys, "setdefaultencoding"):
reload(sys)
try:
# pylint: disable=E1101
# On non-unicode builds this will raise an AttributeError, if encoding type is not valid it throws a LookupError
sys.setdefaultencoding(sickbeard.SYS_ENCODING)
except:
sys.exit("Sorry, you MUST add the SickRage folder to the PYTHONPATH environment variable\n" +
"or find another way to force Python to use " + sickbeard.SYS_ENCODING + " for string encoding.")
for s in strings:
try:
x = ek.ss(s)
assert isinstance(x, unicode)
show_dir = ek.ek(os.path.join, rootDir, sanitizeFileName(s))
self.assertTrue(isinstance(show_dir, unicode))
except Exception, e:
ex(e)