moparisthebest/SickRage
1
0
Fork 0
mirror of https://github.com/moparisthebest/SickRage synced 2024-08-13 16:53:54 -04:00
Code Issues Releases Wiki Activity

Improved RSS feed cache parser code for provider searches and caching of their RSS feeds.

Browse Source
This commit is contained in:
echel0n 2014-12-03 04:10:00 -08:00
parent f6370c6756
commit fa02ff40e7
13 changed files with 105 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.gitignore vendored
View File

@ -1,4 +1,4 @@
# SB User Related # # SR User Related #
###################### ######################
cache/* cache/*
cache.db* cache.db*
@ -11,11 +11,13 @@ server.crt
server.key server.key
restore/ restore/
# SB Test Related # # SR Test Related #
###################### ######################
tests/Logs/* tests/Logs/*
tests/sickbeard.* tests/cache/*
tests/cache.db tests/sickbeard.db*
tests/cache.db*
tests/failed.db
# Compiled source # # Compiled source #
###################### ######################
@ -46,4 +48,4 @@ Thumbs.db
.directory .directory
*~ *~
/.idea/ /.idea/
*.torrent *.torrent

38
sickbeard/providers/animezb.py
View File

@ -78,28 +78,18 @@ class Animezb(generic.NZBProvider):
logger.log(u"Search url: " + search_url, logger.DEBUG) logger.log(u"Search url: " + search_url, logger.DEBUG)
data = self.cache.getRSSFeed(search_url) results = []
if not data: for curItem in self.cache.getRSSFeed(search_url):
return [] (title, url) = self._get_title_and_url(curItem)
if 'entries' in data: if title and url:
results.append(curItem)
else:
logger.log(
u"The data returned from the " + self.name + " is incomplete, this result is unusable",
logger.DEBUG)
items = data.entries return results
results = []
for curItem in items:
(title, url) = self._get_title_and_url(curItem)
if title and url:
results.append(curItem)
else:
logger.log(
u"The data returned from the " + self.name + " is incomplete, this result is unusable",
logger.DEBUG)
return results
return []
def findPropers(self, date=None): def findPropers(self, date=None):
@ -144,12 +134,6 @@ class AnimezbCache(tvcache.TVCache):
logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG) logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG)
data = self.getRSSFeed(rss_url) return self.getRSSFeed(rss_url)
if data and 'entries' in data:
return data.entries
else:
return []
provider = Animezb() provider = Animezb()

13
sickbeard/providers/ezrss.py
View File

@ -122,15 +122,8 @@ class EZRSSProvider(generic.TorrentProvider):
logger.log(u"Search string: " + search_url, logger.DEBUG) logger.log(u"Search string: " + search_url, logger.DEBUG)
data = self.cache.getRSSFeed(search_url)
if not data:
return []
items = data.entries
results = [] results = []
for curItem in items: for curItem in self.cache.getRSSFeed(search_url):
(title, url) = self._get_title_and_url(curItem) (title, url) = self._get_title_and_url(curItem)
@ -179,8 +172,6 @@ class EZRSSCache(tvcache.TVCache):
rss_url = self.provider.url + 'feed/' rss_url = self.provider.url + 'feed/'
logger.log(self.provider.name + " cache update URL: " + rss_url, logger.DEBUG) logger.log(self.provider.name + " cache update URL: " + rss_url, logger.DEBUG)
data = self.getRSSFeed(rss_url) return self.getRSSFeed(rss_url)
return data.entries or []
provider = EZRSSProvider() provider = EZRSSProvider()

38
sickbeard/providers/fanzub.py
View File

@ -73,28 +73,18 @@ class Fanzub(generic.NZBProvider):
logger.log(u"Search url: " + search_url, logger.DEBUG) logger.log(u"Search url: " + search_url, logger.DEBUG)
data = self.cache.getRSSFeed(search_url) results = []
if not data: for curItem in self.cache.getRSSFeed(search_url):
return [] (title, url) = self._get_title_and_url(curItem)
if 'entries' in data: if title and url:
results.append(curItem)
else:
logger.log(
u"The data returned from the " + self.name + " is incomplete, this result is unusable",
logger.DEBUG)
items = data.entries return results
results = []
for curItem in items:
(title, url) = self._get_title_and_url(curItem)
if title and url:
results.append(curItem)
else:
logger.log(
u"The data returned from the " + self.name + " is incomplete, this result is unusable",
logger.DEBUG)
return results
return []
def findPropers(self, date=None): def findPropers(self, date=None):
@ -139,12 +129,6 @@ class FanzubCache(tvcache.TVCache):
logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG) logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG)
data = self.getRSSFeed(rss_url) return self.getRSSFeed(rss_url)
if data and 'entries' in data:
return data.entries
else:
return []
provider = Fanzub() provider = Fanzub()

9
sickbeard/providers/newznab.py
View File

@ -237,11 +237,12 @@ class NewznabProvider(generic.NZBProvider):
def _checkAuthFromData(self, data): def _checkAuthFromData(self, data):
if data is None: if not data.get('entries', None):
return self._checkAuth() return self._checkAuth()
if 'error' in data.feed: if data.feed.get('error', None):
code = data.feed['error']['code']
code = data.feed.error.get('code', None)
if code == '100': if code == '100':
raise AuthException("Your API key for " + self.name + " is incorrect, check your config.") raise AuthException("Your API key for " + self.name + " is incorrect, check your config.")
@ -251,7 +252,7 @@ class NewznabProvider(generic.NZBProvider):
raise AuthException( raise AuthException(
"Your account isn't allowed to use the API on " + self.name + ", contact the administrator") "Your account isn't allowed to use the API on " + self.name + ", contact the administrator")
else: else:
logger.log(u"Unknown error given from " + self.name + ": " + data.feed['error']['description'], logger.log(u"Unknown error given from " + self.name + ": " + data.feed.error.description,
logger.ERROR) logger.ERROR)
return False return False

39
sickbeard/providers/nyaatorrents.py
View File

@ -79,32 +79,21 @@ class NyaaProvider(generic.TorrentProvider):
logger.log(u"Search string: " + searchURL, logger.DEBUG) logger.log(u"Search string: " + searchURL, logger.DEBUG)
data = self.cache.getRSSFeed(searchURL) results = []
if not data: for curItem in self.cache.getRSSFeed(searchURL):
return []
if 'entries' in data: (title, url) = self._get_title_and_url(curItem)
items = data.entries
results = [] if title and url:
results.append(curItem)
else:
logger.log(
u"The data returned from the " + self.name + " is incomplete, this result is unusable",
logger.DEBUG)
for curItem in items: return results
(title, url) = self._get_title_and_url(curItem)
if title and url:
results.append(curItem)
else:
logger.log(
u"The data returned from the " + self.name + " is incomplete, this result is unusable",
logger.DEBUG)
return results
return []
def _get_title_and_url(self, item): def _get_title_and_url(self, item):
return generic.TorrentProvider._get_title_and_url(self, item) return generic.TorrentProvider._get_title_and_url(self, item)
def _extract_name_from_filename(self, filename): def _extract_name_from_filename(self, filename):
@ -137,12 +126,6 @@ class NyaaCache(tvcache.TVCache):
logger.log(u"NyaaTorrents cache update URL: " + url, logger.DEBUG) logger.log(u"NyaaTorrents cache update URL: " + url, logger.DEBUG)
data = self.getRSSFeed(url) return self.getRSSFeed(url)
if data and 'entries' in data:
return data.entries
else:
return []
provider = NyaaProvider() provider = NyaaProvider()

7
sickbeard/providers/omgwtfnzbs.py
View File

@ -184,11 +184,6 @@ class OmgwtfnzbsCache(tvcache.TVCache):
logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG) logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG)
data = self.getRSSFeed(rss_url) return self.getRSSFeed(rss_url)
if data and 'entries' in data:
return data.entries
else:
return []
provider = OmgwtfnzbsProvider() provider = OmgwtfnzbsProvider()

7
sickbeard/providers/rsstorrent.py
View File

@ -169,9 +169,4 @@ class TorrentRssCache(tvcache.TVCache):
if self.provider.cookies: if self.provider.cookies:
request_headers = {'Cookie': self.provider.cookies} request_headers = {'Cookie': self.provider.cookies}
data = self.getRSSFeed(self.provider.url, request_headers=request_headers) return self.getRSSFeed(self.provider.url, request_headers=request_headers)
if data and 'entries' in data:
return data.entries
else:
return []

7
sickbeard/providers/tokyotoshokan.py
View File

@ -164,12 +164,7 @@ class TokyoToshokanCache(tvcache.TVCache):
logger.log(u"TokyoToshokan cache update URL: " + url, logger.DEBUG) logger.log(u"TokyoToshokan cache update URL: " + url, logger.DEBUG)
data = self.getRSSFeed(url) return self.getRSSFeed(url)
if data and 'entries' in data:
return data.entries
else:
return []
provider = TokyoToshokanProvider() provider = TokyoToshokanProvider()

19
sickbeard/providers/tvtorrents.py
View File

@ -54,14 +54,16 @@ class TvTorrentsProvider(generic.TorrentProvider):
return 'tvtorrents.png' return 'tvtorrents.png'
def _checkAuth(self): def _checkAuth(self):
if not self.digest or not self.hash: if not self.digest or not self.hash:
raise AuthException("Your authentication credentials for " + self.name + " are missing, check your config.") raise AuthException("Your authentication credentials for " + self.name + " are missing, check your config.")
return True return True
def _checkAuthFromData(self, data): def _checkAuthFromData(self, data):
if "User can't be found" in data.feed.title or "Invalid Hash" in data.feed.title: if not data.get('entries', None):
return self._checkAuth()
if "User can't be found" in data.feed.get('title', None) or "Invalid Hash" in data.feed.get('title', None):
logger.log(u"Incorrect authentication credentials for " + self.name + " : " + str(data.feed.title), logger.log(u"Incorrect authentication credentials for " + self.name + " : " + str(data.feed.title),
logger.DEBUG) logger.DEBUG)
raise AuthException( raise AuthException(
@ -87,16 +89,9 @@ class TvTorrentsCache(tvcache.TVCache):
rss_url = self.provider.url + 'RssServlet?digest=' + provider.digest + '&hash=' + provider.hash + '&fname=true&exclude=(' + ignore_regex + ')' rss_url = self.provider.url + 'RssServlet?digest=' + provider.digest + '&hash=' + provider.hash + '&fname=true&exclude=(' + ignore_regex + ')'
logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG) logger.log(self.provider.name + u" cache update URL: " + rss_url, logger.DEBUG)
data = self.getRSSFeed(rss_url) return self.getRSSFeed(rss_url)
if not self.provider._checkAuthFromData(data):
return []
if data and 'entries' in data:
return data['entries']
else:
return []
def _checkAuth(self, data):
return self.provider._checkAuthFromData(data)
provider = TvTorrentsProvider() provider = TvTorrentsProvider()

18
sickbeard/providers/womble.py
View File

@ -47,38 +47,26 @@ class WombleCache(tvcache.TVCache):
# delete anything older then 7 days # delete anything older then 7 days
self._clearCache() self._clearCache()
data = None
if not self.shouldUpdate(): if not self.shouldUpdate():
return return
cl = [] cl = []
for url in [self.provider.url + 'rss/?sec=tv-sd&fr=false', self.provider.url + 'rss/?sec=tv-hd&fr=false']: for url in [self.provider.url + 'rss/?sec=tv-sd&fr=false', self.provider.url + 'rss/?sec=tv-hd&fr=false']:
logger.log(u"Womble's Index cache update URL: " + url, logger.DEBUG) logger.log(u"Womble's Index cache update URL: " + url, logger.DEBUG)
data = self.getRSSFeed(url)
# As long as we got something from the provider we count it as an update
if not data:
return []
# By now we know we've got data and no auth errors, all we need to do is put it in the database # By now we know we've got data and no auth errors, all we need to do is put it in the database
for item in data.entries: for item in self.getRSSFeed(url).get('entries', []):
ci = self._parseItem(item) ci = self._parseItem(item.title, item.url)
if ci is not None: if ci is not None:
cl.append(ci) cl.append(ci)
if len(cl) > 0: if len(cl) > 0:
myDB = self._getDB() myDB = self._getDB()
myDB.mass_action(cl) myDB.mass_action(cl)
# set last updated
if data:
self.setLastUpdate() self.setLastUpdate()
def _checkAuth(self, data): def _checkAuth(self, data):
return data != 'Invalid Link' return data.feed.get('title', None) != 'Invalid Link'
provider = WombleProvider() provider = WombleProvider()

46
sickbeard/rssfeeds.py
View File

@ -4,6 +4,7 @@ import os
import urllib import urllib
import urlparse import urlparse
import re import re
import collections
import sickbeard import sickbeard
@ -11,28 +12,30 @@ from sickbeard import logger
from sickbeard import encodingKludge as ek from sickbeard import encodingKludge as ek
from sickbeard.exceptions import ex from sickbeard.exceptions import ex
from contextlib import closing from feedcache.cache import Cache
from lib.feedcache import cache
from sqliteshelf import SQLiteShelf from sqliteshelf import SQLiteShelf
class RSSFeeds: class RSSFeeds:
def __init__(self, db_name): def __init__(self, db_name):
db_name = ek.ek(os.path.join, sickbeard.CACHE_DIR, 'rss', db_name) + '.db' self.rssItems = {'entries': None,
if not os.path.exists(os.path.dirname(db_name)): 'feed': None
sickbeard.helpers.makeDir(os.path.dirname(db_name)) }
try: try:
db_name = ek.ek(os.path.join, sickbeard.CACHE_DIR, 'rss', db_name) + '.db'
if not os.path.exists(os.path.dirname(db_name)):
sickbeard.helpers.makeDir(os.path.dirname(db_name))
self.rssDB = SQLiteShelf(db_name) self.rssDB = SQLiteShelf(db_name)
except Exception as e: except Exception as e:
logger.log(u"RSS error: " + ex(e), logger.DEBUG) logger.log(u"RSS error: " + ex(e), logger.DEBUG)
def clearCache(self, age=None): def clearCache(self, age=None):
try: try:
with closing(self.rssDB) as fs: fc = Cache(self.rssDB).purge(age)
fc = cache.Cache(fs) fc.purge(age)
fc.purge(age) finally:
except Exception as e: self.rssDB.close()
logger.log(u"RSS error clearing cache: " + ex(e), logger.DEBUG)
def getFeed(self, url, post_data=None, request_headers=None): def getFeed(self, url, post_data=None, request_headers=None):
parsed = list(urlparse.urlparse(url)) parsed = list(urlparse.urlparse(url))
@ -42,21 +45,12 @@ class RSSFeeds:
url += urllib.urlencode(post_data) url += urllib.urlencode(post_data)
try: try:
with closing(self.rssDB) as fs: fc = Cache(self.rssDB)
fc = cache.Cache(fs) feed = fc.fetch(url, False, False, request_headers)
feed = fc.fetch(url, False, False, request_headers)
if feed: self.rssItems['entries'] = feed.get('entries', None)
if 'entries' in feed: self.rssItems['feed'] = feed.get('feed', None)
return feed finally:
elif 'error' in feed.feed: self.rssDB.close()
err_code = feed.feed['error']['code']
err_desc = feed.feed['error']['description']
logger.log( return self.rssItems
u"RSS ERROR:[%s] CODE:[%s]" % (err_desc, err_code), logger.DEBUG)
else:
logger.log(u"RSS error loading url: " + url, logger.DEBUG)
except Exception as e:
logger.log(u"RSS error: " + ex(e), logger.DEBUG)

47
sickbeard/tvcache.py
View File

@ -102,36 +102,41 @@ class TVCache():
data = None data = None
return data return data
def _checkAuth(self): def _checkAuth(self, data):
return self.provider._checkAuth() return True
def _checkItemAuth(self, title, url): def _checkItemAuth(self, title, url):
return True return True
def updateCache(self): def updateCache(self):
if self.shouldUpdate() and self._checkAuth(): if not self.shouldUpdate():
# as long as the http request worked we count this as an update return
if self._checkAuth(None):
data = self._getRSSData() data = self._getRSSData()
if not data: if data.get('entries', None):
return [] # clear cache
self._clearCache()
# clear cache # set updated
self._clearCache() self.setLastUpdate()
# set updated if self._checkAuth(data):
self.setLastUpdate()
# parse data cl = []
cl = [] for item in data.get('entries', []):
for item in data: title, url = self._get_title_and_url(item)
title, url = self._get_title_and_url(item) ci = self._parseItem(title, url)
ci = self._parseItem(title, url) if ci is not None:
if ci is not None: cl.append(ci)
cl.append(ci)
if len(cl) > 0: if len(cl) > 0:
myDB = self._getDB() myDB = self._getDB()
myDB.mass_action(cl) myDB.mass_action(cl)
else:
raise AuthException(
u"Your authentication credentials for " + self.provider.name + " are incorrect, check your config")
return [] return []
@ -159,8 +164,6 @@ class TVCache():
logger.log( logger.log(
u"The data returned from the " + self.provider.name + " feed is incomplete, this result is unusable", u"The data returned from the " + self.provider.name + " feed is incomplete, this result is unusable",
logger.DEBUG) logger.DEBUG)
return None
def _getLastUpdate(self): def _getLastUpdate(self):
myDB = self._getDB() myDB = self._getDB()