diff --git a/sickbeard/encodingKludge.py b/sickbeard/encodingKludge.py index 1f60633e..18500f61 100644 --- a/sickbeard/encodingKludge.py +++ b/sickbeard/encodingKludge.py @@ -18,48 +18,53 @@ import os import traceback - +import re import sickbeard import six import chardet +import unicodedata +from string import ascii_letters, digits from sickbeard import logger -# This module tries to deal with the apparently random behavior of python when dealing with unicode <-> utf-8 -# encodings. It tries to just use unicode, but if that fails then it tries forcing it to utf-8. Any functions -# which return something should always return unicode. +def toSafeString(original): + valid_chars = "-_.() %s%s" % (ascii_letters, digits) + cleaned_filename = unicodedata.normalize('NFKD', _toUnicode(original)).encode('ASCII', 'ignore') + valid_string = ''.join(c for c in cleaned_filename if c in valid_chars) + return ' '.join(valid_string.split()) + + +def simplifyString(original): + string = stripAccents(original.lower()) + string = toSafeString(' '.join(re.split('\W+', string))) + split = re.split('\W+|_', string.lower()) + return _toUnicode(' '.join(split)) def _toUnicode(x): - try: - if isinstance(x, unicode): - return x - else: + if isinstance(x, unicode): + return x + else: + try: + return six.text_type(x) + except: try: - return six.text_type(x) + if chardet.detect(x).get('encoding') == 'utf-8': + return x.decode('utf-8') + if isinstance(x, str): + try: + return x.decode(sickbeard.SYS_ENCODING) + except UnicodeDecodeError: + raise + return x except: - try: - if chardet.detect(x).get('encoding') == 'utf-8': - return x.decode('utf-8') - if isinstance(x, str): - try: - return x.decode(sickbeard.SYS_ENCODING) - except UnicodeDecodeError: - raise - return x - except: - raise - except: - logger.log('Unable to decode value "%s..." : %s ' % (repr(x)[:20], traceback.format_exc()), logger.WARNING) - ascii_text = str(x).encode('string_escape') - return _toUnicode(ascii_text) + return x def ss(x): u_x = _toUnicode(x) try: return u_x.encode(sickbeard.SYS_ENCODING) - except Exception as e: - logger.log('Failed ss encoding char, force UTF8: %s' % e, logger.WARNING) + except: try: return u_x.encode(sickbeard.SYS_ENCODING, 'replace') except: @@ -84,3 +89,6 @@ def ek(func, *args, **kwargs): return _toUnicode(result) else: return result + +def stripAccents(s): + return ''.join((c for c in unicodedata.normalize('NFD', _toUnicode(s)) if unicodedata.category(c) != 'Mn')) \ No newline at end of file diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index 8a6fa016..85d53a0c 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -1431,3 +1431,5 @@ def get_size(start_path='.'): total_size += ek.ek(os.path.getsize, fp) return total_size +def md5(text): + return hashlib.md5(ek.ss(text)).hexdigest() \ No newline at end of file diff --git a/sickbeard/providers/animezb.py b/sickbeard/providers/animezb.py index 93bf7701..121943cc 100644 --- a/sickbeard/providers/animezb.py +++ b/sickbeard/providers/animezb.py @@ -79,7 +79,7 @@ class Animezb(generic.NZBProvider): logger.log(u"Search url: " + search_url, logger.DEBUG) results = [] - for curItem in self.cache.getRSSFeed(search_url): + for curItem in self.cache.getRSSFeed(search_url, items=['entries']) or []: (title, url) = self._get_title_and_url(curItem) if title and url: @@ -134,6 +134,6 @@ class AnimezbCache(tvcache.TVCache): logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG) - return self.getRSSFeed(rss_url) + return self.getRSSFeed(rss_url, items=['entries', 'feed']) provider = Animezb() diff --git a/sickbeard/providers/bitsoup.py b/sickbeard/providers/bitsoup.py index 8df8ece7..303b69a5 100644 --- a/sickbeard/providers/bitsoup.py +++ b/sickbeard/providers/bitsoup.py @@ -157,7 +157,7 @@ class BitSoupProvider(generic.TorrentProvider): items = {'Season': [], 'Episode': [], 'RSS': []} if not self._doLogin(): - return [] + return results for mode in search_params.keys(): for search_string in search_params[mode]: @@ -273,7 +273,7 @@ class BitSoupCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['']} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} provider = BitSoupProvider() diff --git a/sickbeard/providers/btn.py b/sickbeard/providers/btn.py index d6c31ff9..912f3050 100644 --- a/sickbeard/providers/btn.py +++ b/sickbeard/providers/btn.py @@ -92,7 +92,7 @@ class BTNProvider(generic.TorrentProvider): parsedJSON = self._api_call(apikey, params) if not parsedJSON: logger.log(u"No data returned from " + self.name, logger.ERROR) - return [] + return results if self._checkAuthFromData(parsedJSON): @@ -311,7 +311,7 @@ class BTNCache(tvcache.TVCache): logger.WARNING) seconds_since_last_update = 86400 - return self.provider._doSearch(search_params=None, age=seconds_since_last_update) + return {'entries': self.provider._doSearch(search_params=None, age=seconds_since_last_update)} provider = BTNProvider() diff --git a/sickbeard/providers/ezrss.py b/sickbeard/providers/ezrss.py index 34bc170b..86c88d7b 100644 --- a/sickbeard/providers/ezrss.py +++ b/sickbeard/providers/ezrss.py @@ -123,7 +123,7 @@ class EZRSSProvider(generic.TorrentProvider): logger.log(u"Search string: " + search_url, logger.DEBUG) results = [] - for curItem in self.cache.getRSSFeed(search_url): + for curItem in self.cache.getRSSFeed(search_url, items=['entries']) or []: (title, url) = self._get_title_and_url(curItem) @@ -172,6 +172,6 @@ class EZRSSCache(tvcache.TVCache): rss_url = self.provider.url + 'feed/' logger.log(self.provider.name + " cache update URL: " + rss_url, logger.DEBUG) - return self.getRSSFeed(rss_url) + return self.getRSSFeed(rss_url, items=['entries', 'feed']) provider = EZRSSProvider() diff --git a/sickbeard/providers/fanzub.py b/sickbeard/providers/fanzub.py index 2f416334..dcdcfe27 100644 --- a/sickbeard/providers/fanzub.py +++ b/sickbeard/providers/fanzub.py @@ -74,7 +74,7 @@ class Fanzub(generic.NZBProvider): logger.log(u"Search url: " + search_url, logger.DEBUG) results = [] - for curItem in self.cache.getRSSFeed(search_url): + for curItem in self.cache.getRSSFeed(search_url, items=['entries']) or []: (title, url) = self._get_title_and_url(curItem) if title and url: @@ -129,6 +129,6 @@ class FanzubCache(tvcache.TVCache): logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG) - return self.getRSSFeed(rss_url) + return self.getRSSFeed(rss_url, items=['entries', 'feed']) provider = Fanzub() diff --git a/sickbeard/providers/freshontv.py b/sickbeard/providers/freshontv.py index b16fb8cb..f2bd2c47 100755 --- a/sickbeard/providers/freshontv.py +++ b/sickbeard/providers/freshontv.py @@ -309,6 +309,6 @@ class FreshOnTVCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['']} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} provider = FreshOnTVProvider() \ No newline at end of file diff --git a/sickbeard/providers/hdbits.py b/sickbeard/providers/hdbits.py index 119d4849..60431a3b 100644 --- a/sickbeard/providers/hdbits.py +++ b/sickbeard/providers/hdbits.py @@ -210,11 +210,12 @@ class HDBitsCache(tvcache.TVCache): try: parsedJSON = self.provider.getURL(self.provider.rss_url, post_data=self.provider._make_post_data_JSON(), json=True) + if self.provider._checkAuthFromData(parsedJSON): results = parsedJSON['data'] except: pass - return results + return {'entries': results} provider = HDBitsProvider() diff --git a/sickbeard/providers/hdtorrents.py b/sickbeard/providers/hdtorrents.py index d191ad50..405981d9 100644 --- a/sickbeard/providers/hdtorrents.py +++ b/sickbeard/providers/hdtorrents.py @@ -342,7 +342,7 @@ class HDTorrentsCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': []} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} provider = HDTorrentsProvider() diff --git a/sickbeard/providers/iptorrents.py b/sickbeard/providers/iptorrents.py index ad5dceea..19bd80bb 100644 --- a/sickbeard/providers/iptorrents.py +++ b/sickbeard/providers/iptorrents.py @@ -279,7 +279,7 @@ class IPTorrentsCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['']} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} provider = IPTorrentsProvider() diff --git a/sickbeard/providers/kat.py b/sickbeard/providers/kat.py index 485c88d0..7bb51c61 100644 --- a/sickbeard/providers/kat.py +++ b/sickbeard/providers/kat.py @@ -357,6 +357,6 @@ class KATCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['rss']} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} provider = KATProvider() diff --git a/sickbeard/providers/newznab.py b/sickbeard/providers/newznab.py index b9bd3fba..7e740ee2 100755 --- a/sickbeard/providers/newznab.py +++ b/sickbeard/providers/newznab.py @@ -238,11 +238,10 @@ class NewznabProvider(generic.NZBProvider): def _checkAuthFromData(self, data): - if not data: + if not (data.entries and data.feed): return self._checkAuth() if data.feed.get('error', None): - code = data.feed.error.get('code', None) if code == '100': @@ -297,12 +296,12 @@ class NewznabProvider(generic.NZBProvider): while (total >= offset) and (offset < 1000): search_url = self.url + 'api?' + urllib.urlencode(params) logger.log(u"Search url: " + search_url, logger.DEBUG) - data = self.cache.getRSSFeed(search_url) - if not data or not self._checkAuthFromData(data): + data = self.cache.getRSSFeed(search_url, items=['entries', 'feed']) + if not self._checkAuthFromData(data): break - for item in data.entries: + for item in data.entries or []: (title, url) = self._get_title_and_url(item) @@ -422,56 +421,13 @@ class NewznabCache(tvcache.TVCache): logger.log(self.provider.name + " cache update URL: " + rss_url, logger.DEBUG) - return self.getRSSFeed(rss_url) + return self.getRSSFeed(rss_url, items=['entries', 'feed']) def _checkAuth(self, data): return self.provider._checkAuthFromData(data) - def updateCache(self): - if not self.shouldUpdate(): - return - - try: - if self._checkAuth(None): - data = self._getRSSData() - if not data or not len(data) > 0: - return - - # clear cache - self._clearCache() - - # set updated - self.setLastUpdate() - - try: - items = data.get('entries', []) - except: - items = data - - if self._checkAuth(items): - cl = [] - for item in items: - ci = self._parseItem(item) - if ci is not None: - cl.append(ci) - - if len(cl) > 0: - myDB = self._getDB() - myDB.mass_action(cl) - - else: - raise AuthException( - u"Your authentication credentials for " + self.provider.name + " are incorrect, check your config") - except AuthException, e: - logger.log(u"Authentication error: " + ex(e), logger.ERROR) - except Exception, e: - logger.log(u"Error while searching " + self.provider.name + ", skipping: " + ex(e), logger.ERROR) - logger.log(traceback.format_exc(), logger.DEBUG) - - # overwrite method with that parses the rageid from the newznab feed def _parseItem(self, item): - title = item.title - url = item.link + (title, url) = self._get_title_and_url(item) attrs = item.newznab_attr if not isinstance(attrs, list): diff --git a/sickbeard/providers/nextgen.py b/sickbeard/providers/nextgen.py index 7bcbfd39..06d61bac 100644 --- a/sickbeard/providers/nextgen.py +++ b/sickbeard/providers/nextgen.py @@ -320,7 +320,7 @@ class NextGenCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['']} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} provider = NextGenProvider() diff --git a/sickbeard/providers/nyaatorrents.py b/sickbeard/providers/nyaatorrents.py index f1a94243..41ee3783 100644 --- a/sickbeard/providers/nyaatorrents.py +++ b/sickbeard/providers/nyaatorrents.py @@ -80,8 +80,7 @@ class NyaaProvider(generic.TorrentProvider): logger.log(u"Search string: " + searchURL, logger.DEBUG) results = [] - for curItem in self.cache.getRSSFeed(searchURL): - + for curItem in self.cache.getRSSFeed(searchURL, items=['entries']) or []: (title, url) = self._get_title_and_url(curItem) if title and url: @@ -126,6 +125,6 @@ class NyaaCache(tvcache.TVCache): logger.log(u"NyaaTorrents cache update URL: " + url, logger.DEBUG) - return self.getRSSFeed(url) + return self.getRSSFeed(url, items=['entries', 'feed']) provider = NyaaProvider() diff --git a/sickbeard/providers/omgwtfnzbs.py b/sickbeard/providers/omgwtfnzbs.py index 8dc02b12..b0c5cca9 100644 --- a/sickbeard/providers/omgwtfnzbs.py +++ b/sickbeard/providers/omgwtfnzbs.py @@ -184,6 +184,6 @@ class OmgwtfnzbsCache(tvcache.TVCache): logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG) - return self.getRSSFeed(rss_url) + return self.getRSSFeed(rss_url, items=['entries', 'feed']) provider = OmgwtfnzbsProvider() diff --git a/sickbeard/providers/rsstorrent.py b/sickbeard/providers/rsstorrent.py index cb7b53b8..6d19ead4 100644 --- a/sickbeard/providers/rsstorrent.py +++ b/sickbeard/providers/rsstorrent.py @@ -107,11 +107,11 @@ class TorrentRssProvider(generic.TorrentProvider): if not cookie_validator.match(self.cookies): return (False, 'Cookie is not correctly formatted: ' + self.cookies) - data = self.cache._getRSSData() - if not data or not len(data) > 0: + data = self.cache._getRSSData()['entries'] + if not data: return (False, 'No items found in the RSS feed ' + self.url) - (title, url) = self._get_title_and_url(data.entries[0]) + (title, url) = self._get_title_and_url(data[0]) if not title: return (False, 'Unable to get title from first item') @@ -168,4 +168,4 @@ class TorrentRssCache(tvcache.TVCache): if self.provider.cookies: request_headers = {'Cookie': self.provider.cookies} - return self.getRSSFeed(self.provider.url, request_headers=request_headers) \ No newline at end of file + return self.getRSSFeed(self.provider.url, request_headers=request_headers, items=['entries', 'feed']) \ No newline at end of file diff --git a/sickbeard/providers/scc.py b/sickbeard/providers/scc.py index b670c158..a713ca56 100644 --- a/sickbeard/providers/scc.py +++ b/sickbeard/providers/scc.py @@ -305,6 +305,6 @@ class SCCCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['']} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} provider = SCCProvider() diff --git a/sickbeard/providers/speedcd.py b/sickbeard/providers/speedcd.py index 6fbf1cb0..f01973bd 100644 --- a/sickbeard/providers/speedcd.py +++ b/sickbeard/providers/speedcd.py @@ -254,7 +254,7 @@ class SpeedCDCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['']} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} provider = SpeedCDProvider() diff --git a/sickbeard/providers/t411.py b/sickbeard/providers/t411.py index 7ea4bbc7..2b8ec250 100644 --- a/sickbeard/providers/t411.py +++ b/sickbeard/providers/t411.py @@ -260,7 +260,7 @@ class T411Cache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['']} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} provider = T411Provider() diff --git a/sickbeard/providers/thepiratebay.py b/sickbeard/providers/thepiratebay.py index dd2423e0..0448b7a8 100644 --- a/sickbeard/providers/thepiratebay.py +++ b/sickbeard/providers/thepiratebay.py @@ -340,7 +340,7 @@ class ThePirateBayCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['rss']} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} class ThePirateBayWebproxy: def __init__(self): diff --git a/sickbeard/providers/tokyotoshokan.py b/sickbeard/providers/tokyotoshokan.py index 2782bbc4..f8dde424 100644 --- a/sickbeard/providers/tokyotoshokan.py +++ b/sickbeard/providers/tokyotoshokan.py @@ -164,7 +164,7 @@ class TokyoToshokanCache(tvcache.TVCache): logger.log(u"TokyoToshokan cache update URL: " + url, logger.DEBUG) - return self.getRSSFeed(url) + return self.getRSSFeed(url, items=['entries', 'feed']) provider = TokyoToshokanProvider() diff --git a/sickbeard/providers/torrentbytes.py b/sickbeard/providers/torrentbytes.py index e82e3736..2ef871bf 100644 --- a/sickbeard/providers/torrentbytes.py +++ b/sickbeard/providers/torrentbytes.py @@ -276,7 +276,7 @@ class TorrentBytesCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['']} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} provider = TorrentBytesProvider() diff --git a/sickbeard/providers/torrentday.py b/sickbeard/providers/torrentday.py index 782645cf..79482c9f 100644 --- a/sickbeard/providers/torrentday.py +++ b/sickbeard/providers/torrentday.py @@ -282,8 +282,6 @@ class TorrentDayCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['']} - return self.provider._doSearch(search_params) - - + return {'entries': self.provider._doSearch(search_params)} provider = TorrentDayProvider() diff --git a/sickbeard/providers/torrentleech.py b/sickbeard/providers/torrentleech.py index e82403ed..55be7100 100644 --- a/sickbeard/providers/torrentleech.py +++ b/sickbeard/providers/torrentleech.py @@ -277,7 +277,7 @@ class TorrentLeechCache(tvcache.TVCache): def _getRSSData(self): search_params = {'RSS': ['']} - return self.provider._doSearch(search_params) + return {'entries': self.provider._doSearch(search_params)} provider = TorrentLeechProvider() diff --git a/sickbeard/providers/tvtorrents.py b/sickbeard/providers/tvtorrents.py index 06a4717f..deed3e53 100644 --- a/sickbeard/providers/tvtorrents.py +++ b/sickbeard/providers/tvtorrents.py @@ -60,11 +60,12 @@ class TvTorrentsProvider(generic.TorrentProvider): return True def _checkAuthFromData(self, data): - if not data: + if not (data.entries and data.feed): return self._checkAuth() - if "User can't be found" in data.feed.get('title', None) or "Invalid Hash" in data.feed.get('title', None): - logger.log(u"Incorrect authentication credentials for " + self.name + " : " + str(data.feed.title), + title = data.feed.get('title', None) + if "User can't be found" in title or "Invalid Hash" in title: + logger.log(u"Incorrect authentication credentials for " + self.name + " : " + str(title), logger.DEBUG) raise AuthException( u"Your authentication credentials for " + self.name + " are incorrect, check your config") @@ -89,7 +90,7 @@ class TvTorrentsCache(tvcache.TVCache): rss_url = self.provider.url + 'RssServlet?digest=' + provider.digest + '&hash=' + provider.hash + '&fname=true&exclude=(' + ignore_regex + ')' logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG) - return self.getRSSFeed(rss_url) + return self.getRSSFeed(rss_url, items=['entries', 'feed']) def _checkAuth(self, data): return self.provider._checkAuthFromData(data) diff --git a/sickbeard/providers/womble.py b/sickbeard/providers/womble.py index e28344e9..505713cc 100644 --- a/sickbeard/providers/womble.py +++ b/sickbeard/providers/womble.py @@ -43,30 +43,31 @@ class WombleCache(tvcache.TVCache): self.minTime = 15 def updateCache(self): - - # delete anything older then 7 days - self._clearCache() - + # check if we should update if not self.shouldUpdate(): return + # clear cache + self._clearCache() + + # set updated + self.setLastUpdate() + cl = [] for url in [self.provider.url + 'rss/?sec=tv-sd&fr=false', self.provider.url + 'rss/?sec=tv-hd&fr=false']: logger.log(u"Womble's Index cache update URL: " + url, logger.DEBUG) - # By now we know we've got data and no auth errors, all we need to do is put it in the database - for item in self.getRSSFeed(url).get('entries', []): - ci = self._parseItem(item.title, item.url) + for item in self.getRSSFeed(url, items=['entries', 'feed'])['entries'] or []: + ci = self._parseItem(item) if ci is not None: cl.append(ci) if len(cl) > 0: myDB = self._getDB() myDB.mass_action(cl) - self.setLastUpdate() def _checkAuth(self, data): - return data.feed.get('title', None) != 'Invalid Link' + return data if data.feed.title != 'Invalid Link' else None provider = WombleProvider() diff --git a/sickbeard/rssfeeds.py b/sickbeard/rssfeeds.py index 20368dbf..218ef922 100644 --- a/sickbeard/rssfeeds.py +++ b/sickbeard/rssfeeds.py @@ -33,7 +33,7 @@ class RSSFeeds: finally: self.rssDB.close() - def getFeed(self, url, post_data=None, request_headers=None): + def getFeed(self, url, post_data=None, request_headers=None, items=[]): parsed = list(urlparse.urlparse(url)) parsed[2] = re.sub("/{2,}", "/", parsed[2]) # replace two or more / with one @@ -42,8 +42,15 @@ class RSSFeeds: try: fc = Cache(self.rssDB) - feed = fc.fetch(url, False, False, request_headers) + resp = fc.fetch(url, False, False, request_headers) - return feed + data = {} + for item in items: + try: + data[item] = resp[item] + except: + data[item] = None + + return data finally: self.rssDB.close() \ No newline at end of file diff --git a/sickbeard/search.py b/sickbeard/search.py index 6b001095..9e7de3ff 100644 --- a/sickbeard/search.py +++ b/sickbeard/search.py @@ -375,7 +375,6 @@ def searchForNeededEpisodes(): providers = [x for x in sickbeard.providers.sortedProviderList(sickbeard.RANDOMIZE_PROVIDERS) if x.isActive() and x.enable_daily] for curProvider in providers: - # spawn separate threads for each provider so we don't need to wait for providers with slow network operation threads += [threading.Thread(target=curProvider.cache.updateCache, name=origThreadName + " :: [" + curProvider.name + "]")] # start the thread we just created @@ -393,13 +392,6 @@ def searchForNeededEpisodes(): # pick a single result for each episode, respecting existing results for curEp in curFoundResults: - - if curEp.show.paused: - logger.log( - u"Show " + curEp.show.name + " is paused, ignoring all RSS items for " + curEp.prettyName(), - logger.DEBUG) - continue - # find the best result for the current episode bestResult = None for curResult in curFoundResults[curEp]: @@ -442,6 +434,7 @@ def searchProviders(show, episodes, manualSearch=False): finalResults = [] didSearch = False + threads = [] # build name cache for show sickbeard.name_cache.buildNameCache(show) @@ -449,6 +442,18 @@ def searchProviders(show, episodes, manualSearch=False): origThreadName = threading.currentThread().name providers = [x for x in sickbeard.providers.sortedProviderList(sickbeard.RANDOMIZE_PROVIDERS) if x.isActive() and x.enable_backlog] + for curProvider in providers: + threads += [threading.Thread(target=curProvider.cache.updateCache, + name=origThreadName + " :: [" + curProvider.name + "]")] + + # start the thread we just created + for t in threads: + t.start() + + # wait for all threads to finish + for t in threads: + t.join() + for providerNum, curProvider in enumerate(providers): if curProvider.anime_only and not show.is_anime: logger.log(u"" + str(show.name) + " is not an anime, skiping", logger.DEBUG) @@ -470,7 +475,6 @@ def searchProviders(show, episodes, manualSearch=False): logger.log(u"Performing season pack search for " + show.name) try: - curProvider.cache.updateCache() searchResults = curProvider.findSearchResults(show, episodes, search_mode, manualSearch) except exceptions.AuthException, e: logger.log(u"Authentication error: " + ex(e), logger.ERROR) diff --git a/sickbeard/tvcache.py b/sickbeard/tvcache.py index b8a1b73f..f9fc6842 100644 --- a/sickbeard/tvcache.py +++ b/sickbeard/tvcache.py @@ -96,12 +96,10 @@ class TVCache(): myDB.action("DELETE FROM [" + self.providerID + "] WHERE 1") def _get_title_and_url(self, item): - # override this in the provider if daily search has a different data layout to backlog searches return self.provider._get_title_and_url(item) def _getRSSData(self): - data = None - return data + return None def _checkAuth(self, data): return True @@ -110,49 +108,37 @@ class TVCache(): return True def updateCache(self): + # check if we should update if not self.shouldUpdate(): return try: - if self._checkAuth(None): - data = self._getRSSData() - if not data or not len(data) > 0: - return - + data = self._getRSSData() + if self._checkAuth(data): # clear cache self._clearCache() # set updated self.setLastUpdate() - try: - items = data.get('entries', []) - except: - items = data + cl = [] + for item in data['entries']: + ci = self._parseItem(item) + if ci is not None: + cl.append(ci) - if self._checkAuth(items): - cl = [] - for item in items: - title, url = self._get_title_and_url(item) - ci = self._parseItem(title, url) - if ci is not None: - cl.append(ci) + if len(cl) > 0: + myDB = self._getDB() + myDB.mass_action(cl) - if len(cl) > 0: - myDB = self._getDB() - myDB.mass_action(cl) - - else: - raise AuthException( - u"Your authentication credentials for " + self.provider.name + " are incorrect, check your config") except AuthException, e: logger.log(u"Authentication error: " + ex(e), logger.ERROR) except Exception, e: logger.log(u"Error while searching " + self.provider.name + ", skipping: " + ex(e), logger.ERROR) logger.log(traceback.format_exc(), logger.DEBUG) - def getRSSFeed(self, url, post_data=None, request_headers=None): - return RSSFeeds(self.providerID).getFeed(url, post_data, request_headers) + def getRSSFeed(self, url, post_data=None, request_headers=None, items=[]): + return RSSFeeds(self.providerID).getFeed(url, post_data, request_headers, items) def _translateTitle(self, title): return u'' + title.replace(' ', '.') @@ -160,7 +146,8 @@ class TVCache(): def _translateLinkURL(self, url): return url.replace('&', '&') - def _parseItem(self, title, url): + def _parseItem(self, item): + title, url = self._get_title_and_url(item) self._checkItemAuth(title, url) diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index fbae7cf5..24831e75 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -19,11 +19,9 @@ from __future__ import with_statement import base64 -import inspect +import functools import traceback - import os - import time import urllib import re @@ -59,8 +57,6 @@ from sickbeard.scene_numbering import get_scene_numbering, set_scene_numbering, from sickbeard.blackandwhitelist import BlackAndWhiteList -from browser import WebFileBrowser - from lib.dateutil import tz from lib.unrar2 import RarFile @@ -80,97 +76,247 @@ except ImportError: import xml.etree.ElementTree as etree from Cheetah.Template import Template -from tornado.web import RequestHandler, HTTPError, asynchronous + +from tornado.routes import route +from tornado.web import RequestHandler from bug_tracker import BugTracker +def check_basic_auth(username, password): + # verify username and password are correct + if username == sickbeard.WEB_USERNAME and password == sickbeard.WEB_PASSWORD: + return True -def authenticated(handler_class): - def wrap_execute(handler_execute): - def basicauth(handler, transforms, *args, **kwargs): - def _request_basic_auth(handler): - handler.set_status(401) - handler.set_header('WWW-Authenticate', 'Basic realm="SickRage"') - handler._transforms = [] - handler.finish() - return False +def basic_auth(checkfunc, realm="Authentication Required!"): + def wrap(method): + def request_auth(self): + self.set_header('WWW-Authenticate', 'Basic realm=%s' % realm) + self.set_status(401) + self.finish() + return False + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + if not (sickbeard.WEB_USERNAME and sickbeard.WEB_PASSWORD): + return method(self, *args, **kwargs) + + auth = self.request.headers.get('Authorization') + if auth is None or not auth.startswith('Basic '): + return request_auth(self) + auth = auth[6:] try: - if not (sickbeard.WEB_USERNAME and sickbeard.WEB_PASSWORD): - return True - elif (handler.request.uri.startswith(sickbeard.WEB_ROOT + '/api') and - sickbeard.WEB_ROOT + '/api/builder' not in handler.request.uri): - return True - elif (handler.request.uri.startswith(sickbeard.WEB_ROOT + '/calendar') and - sickbeard.CALENDAR_UNPROTECTED): - return True + username, password = base64.decodestring(auth).split(':', 2) + except: + return request_auth(self) - auth_hdr = handler.request.headers.get('Authorization') + if checkfunc(username, password): + self.request.basic_auth = (username, password) + return method(self, *args, **kwargs) + else: + return request_auth(self) - if auth_hdr is None: - return _request_basic_auth(handler) - if not auth_hdr.startswith('Basic '): - return _request_basic_auth(handler) + return wrapper - auth_decoded = base64.decodestring(auth_hdr[6:]) - username, password = auth_decoded.split(':', 2) + return wrap - if username != sickbeard.WEB_USERNAME or password != sickbeard.WEB_PASSWORD: - return _request_basic_auth(handler) - except Exception, e: - return _request_basic_auth(handler) - return True +def page_not_found(rh): + index_url = sickbeard.WEB_ROOT + url = rh.request.uri[len(index_url):] - def _execute(self, transforms, *args, **kwargs): - if not basicauth(self, transforms, *args, **kwargs): - return False - return handler_execute(self, transforms, *args, **kwargs) + if url[:3] != 'api': + r = index_url + url.lstrip('/') + rh.redirect(r) + else: + rh.set_status(404) + rh.write('Wrong API key used') - return _execute +def _getEpisode(show, season=None, episode=None, absolute=None): + if show is None: + return "Invalid show parameters" - handler_class._execute = wrap_execute(handler_class._execute) - return handler_class + showObj = sickbeard.helpers.findCertainShow(sickbeard.showList, int(show)) + if showObj is None: + return "Invalid show paramaters" -class HTTPRedirect(Exception): - """Exception raised when the request should be redirected.""" + if absolute: + epObj = showObj.getEpisode(absolute_number=int(absolute)) + elif season and episode: + epObj = showObj.getEpisode(int(season), int(episode)) + else: + return "Invalid paramaters" - def __init__(self, url, permanent=False, status=None): - self.url = url - self.permanent = permanent - self.status = status - Exception.__init__(self, self.url, self.permanent, self.status) + if epObj is None: + return "Episode couldn't be retrieved" - def __call__(self): - """Use this exception as a request.handler (raise self).""" - raise self + return epObj +def haveKODI(): + return sickbeard.USE_KODI and sickbeard.KODI_UPDATE_LIBRARY -def redirect(url, permanent=False, status=None): - assert url[0] == '/' - raise HTTPRedirect(sickbeard.WEB_ROOT + url, permanent, status) +def havePLEX(): + return sickbeard.USE_PLEX and sickbeard.PLEX_UPDATE_LIBRARY +def haveTORRENT(): + if sickbeard.USE_TORRENTS and sickbeard.TORRENT_METHOD != 'blackhole' \ + and (sickbeard.ENABLE_HTTPS and sickbeard.TORRENT_HOST[:5] == 'https' + or not sickbeard.ENABLE_HTTPS and sickbeard.TORRENT_HOST[:5] == 'http:'): + return True + else: + return False -@authenticated -class MainHandler(RequestHandler): - def http_error_401_handler(self): - """ Custom handler for 401 error """ - return r''' - -
-