From 0fcd780c2f05d3c602885a0c433cb6d5bb2cb543 Mon Sep 17 00:00:00 2001 From: echel0n Date: Thu, 13 Mar 2014 20:07:15 -0700 Subject: [PATCH] Improved TVRage API code to now include show/episode summaries and fanart. Bugfix in metadata code for XBMC 12+ to display proper episodeguide urls. Misc tweaks here and there. --- lib/tmdb_api/test_tmdb_api.py | 5 +- lib/tvrage_api/tvrage_api.py | 94 +++++------- sickbeard/indexers/indexer_api.py | 12 +- sickbeard/indexers/test.py | 29 ---- sickbeard/indexers/test/__init__.py | 1 + sickbeard/indexers/test/test.py | 51 +++++++ sickbeard/indexers/test/test_lib.py | 227 ++++++++++++++++++++++++++++ sickbeard/metadata/generic.py | 2 +- sickbeard/metadata/xbmc_12plus.py | 2 +- sickbeard/show_queue.py | 2 +- sickbeard/webserve.py | 2 +- tests/tv_tests.py | 1 - 12 files changed, 331 insertions(+), 97 deletions(-) delete mode 100644 sickbeard/indexers/test.py create mode 100644 sickbeard/indexers/test/__init__.py create mode 100644 sickbeard/indexers/test/test.py create mode 100644 sickbeard/indexers/test/test_lib.py diff --git a/lib/tmdb_api/test_tmdb_api.py b/lib/tmdb_api/test_tmdb_api.py index a192a4f7..21f70a27 100644 --- a/lib/tmdb_api/test_tmdb_api.py +++ b/lib/tmdb_api/test_tmdb_api.py @@ -26,8 +26,9 @@ class TVCheck(unittest.TestCase): id = 1396 name = 'UFC' tmdb = TMDB(TMDB_API_KEY) - tv = tmdb.TV(id) - response = tv.images() + find = tmdb.Find(23281) + response = find.info({'external_source': 'tvrage_id'}) + self.assertTrue(hasattr(response, name)) def testTVSearch(self): id = 1396 diff --git a/lib/tvrage_api/tvrage_api.py b/lib/tvrage_api/tvrage_api.py index cdae5bd8..c574d762 100644 --- a/lib/tvrage_api/tvrage_api.py +++ b/lib/tvrage_api/tvrage_api.py @@ -13,6 +13,7 @@ __author__ = "echel0n" __version__ = "1.0" import os +import re import time import urllib import urllib2 @@ -261,6 +262,11 @@ class TVRage: self.config = {} + if apikey is not None: + self.config['apikey'] = apikey + else: + self.config['apikey'] = "Uhewg1Rr0o62fvZvUIZt" # tvdb_api's API key + self.config['debug_enabled'] = debug # show debugging messages self.config['custom_ui'] = custom_ui @@ -330,11 +336,11 @@ class TVRage: # http://tvrage.com/wiki/index.php/Programmers_API self.config['base_url'] = "http://services.tvrage.com" - self.config['url_getSeries'] = u"%(base_url)s/feeds/full_search.php?show=%%s" % self.config + self.config['url_getSeries'] = u"%(base_url)s/myfeeds/search.php?key=%(apikey)s&show=%%s" % self.config - self.config['url_epInfo'] = u"%(base_url)s/feeds/episode_list.php?sid=%%s" % self.config + self.config['url_epInfo'] = u"%(base_url)s/myfeeds/episode_list.php?key=%(apikey)s&sid=%%s" % self.config - self.config['url_seriesInfo'] = u"%(base_url)s/feeds/full_show_info.php?sid=%%s" % self.config + self.config['url_seriesInfo'] = u"%(base_url)s/myfeeds/showinfo.php?key=%(apikey)s&sid=%%s" % self.config def _getTempDir(self): """Returns the [system temp dir]/tvrage_api-u501 (or @@ -422,15 +428,40 @@ class TVRage: def _getetsrc(self, url): """Loads a URL using caching, returns an ElementTree of the source """ + reDict = { + 'showid': 'id', + 'showname': 'seriesname', + 'summary': 'overview', + 'startdate': 'firstaired', + 'genres': 'genre', + 'airtime': 'airs_time', + 'airday': 'airs_dayofweek', + 'image': 'fanart', + 'epnum': 'id', + 'title': 'episodename', + 'airdate': 'firstaired', + 'screencap': 'filename', + 'seasonnum': 'episodenumber', + } + + robj = re.compile('|'.join(reDict.keys())) src = self._loadUrl(url) try: # TVRAGE doesn't sanitize \r (CR) from user input in some fields, # remove it to avoid errors. Change from SickBeard, from will14m - return ElementTree.fromstring(src.rstrip("\r")) + xml = ElementTree.fromstring(src.rstrip("\r")) + tree = ElementTree.ElementTree(xml) + for elm in tree.iter(): + elm.tag = robj.sub(lambda m: reDict[m.group(0)], elm.tag) + return ElementTree.fromstring(ElementTree.tostring(xml)) except SyntaxError: src = self._loadUrl(url, recache=True) try: - return ElementTree.fromstring(src.rstrip("\r")) + xml = ElementTree.fromstring(src.rstrip("\r")) + tree = ElementTree.ElementTree(xml) + for elm in tree.iter(): + elm.tag = robj.sub(lambda m: reDict[m.group(0)], elm.tag) + return ElementTree.fromstring(ElementTree.tostring(xml)) except SyntaxError, exceptionmsg: errormsg = "There was an error with the XML retrieved from tvrage.com:\n%s" % ( exceptionmsg @@ -489,21 +520,6 @@ class TVRage: """This searches tvrage.com for the series name and returns the result list """ - - remap_keys = { - 'showid': 'id', - 'epnum': 'id', - 'started': 'firstaired', - 'airdate': 'firstaired', - 'genres': 'genre', - 'airtime': 'airs_time', - 'name': 'seriesname', - 'image': 'image_type', - 'airday': 'airs_dayofweek', - 'title': 'episodename', - 'seasonnum': 'episodenumber' - } - series = urllib.quote(series.encode("utf-8")) log().debug("Searching for show %s" % series) seriesEt = self._getetsrc(self.config['url_getSeries'] % (series)) @@ -511,10 +527,7 @@ class TVRage: seriesResult = {} for series in seriesEt: for k in series.getchildren(): - if k.tag.lower() in remap_keys: - seriesResult.setdefault(remap_keys[k.tag.lower()], k.text) - else: - seriesResult.setdefault(k.tag.lower(), k.text) + seriesResult.setdefault(k.tag.lower(), k.text) seriesResult['id'] = int(seriesResult['id']) log().debug('Found series %s' % seriesResult['seriesname']) @@ -549,20 +562,6 @@ class TVRage: shows[series_id][season_number][episode_number] """ - remap_keys = { - 'showid': 'id', - 'epnum': 'id', - 'started': 'firstaired', - 'airdate': 'firstaired', - 'genres': 'genre', - 'airtime': 'airs_time', - 'name': 'seriesname', - 'image': 'image_type', - 'airday': 'airs_dayofweek', - 'title': 'episodename', - 'seasonnum': 'episodenumber' - } - # Parse show information log().debug('Getting all series data for %s' % (sid)) seriesInfoEt = self._getetsrc( @@ -570,12 +569,9 @@ class TVRage: ) for curInfo in seriesInfoEt: - if curInfo.tag.lower() in remap_keys: - tag = remap_keys[curInfo.tag.lower()] - else: - tag = curInfo.tag.lower() + tag = curInfo.tag.lower() - if curInfo.tag.lower() in ('started', 'ended') and curInfo.text is not None: + if tag in 'firstaired': try: fixDate = dt.datetime.strptime(curInfo.text,"%b/%d/%Y") value = fixDate.strftime("%Y-%m-%d") @@ -600,10 +596,7 @@ class TVRage: # Parse genre data log().debug('Getting genres of %s' % (sid)) for genre in seriesInfoEt.find('genres'): - if genre.tag in remap_keys: - tag = remap_keys[genre.tag.lower()] - else: - tag = genre.tag.lower() + tag = genre.tag.lower() value = genre.text if value is not None: @@ -623,13 +616,10 @@ class TVRage: try: seas_no = int(cur_seas.attrib['no']) for cur_ep in cur_seas: - ep_no = int(cur_ep.find('seasonnum').text) + ep_no = int(cur_ep.find('episodenumber').text) self._setItem(sid, seas_no, ep_no, 'seasonnumber', seas_no) for cur_item in cur_ep: - if cur_item.tag.lower() in remap_keys: - tag = remap_keys[cur_item.tag.lower()] - else: - tag = cur_item.tag.lower() + tag = cur_item.tag.lower() value = cur_item.text if value is not None: diff --git a/sickbeard/indexers/indexer_api.py b/sickbeard/indexers/indexer_api.py index 21e0d23e..77597b8c 100644 --- a/sickbeard/indexers/indexer_api.py +++ b/sickbeard/indexers/indexer_api.py @@ -37,16 +37,10 @@ class indexerApi(generic.GenericIndexer): self.config['api_parms']['cache'] = os.path.join(sickbeard.CACHE_DIR, indexer) # wrap the indexer API object and return it back - self._wrapped = eval(indexer)(forceConnect=True, *args, **self.config['api_parms']) + self._wrapped = eval(indexer)(*args, **self.config['api_parms']) def __getattr__(self, attr): - try: - return getattr(self._wrapped, attr) - except KeyError: - raise indexer_attributenotfound + return getattr(self._wrapped, attr) def __getitem__(self, attr): - try: - return self._wrapped.__getitem__(attr) - except KeyError: - raise indexer_attributenotfound \ No newline at end of file + return self._wrapped.__getitem__(attr) \ No newline at end of file diff --git a/sickbeard/indexers/test.py b/sickbeard/indexers/test.py deleted file mode 100644 index 9e477e71..00000000 --- a/sickbeard/indexers/test.py +++ /dev/null @@ -1,29 +0,0 @@ -import sys -import logging -import traceback - -from indexer_api import indexerApi -from indexer_exceptions import indexer_exception - -# Set our common indexer_api options here -INDEXER_API_PARMS = {'apikey': '9DAF49C96CBF8DAC', - 'language': 'en', - 'useZip': True} - - -INDEXER_API_PARMS['indexer'] = 'Tvdb' -lindexer_api_parms = INDEXER_API_PARMS.copy() - -try: - imdbid = " " - - t = indexerApi().config['valid_languages'] - t = indexerApi(**lindexer_api_parms) - myEp = t[258171] - - if getattr(myEp, 'seriesname', None) is not None: - print "FOUND" - -except indexer_exception as e: - print e - pass diff --git a/sickbeard/indexers/test/__init__.py b/sickbeard/indexers/test/__init__.py new file mode 100644 index 00000000..1f47cffe --- /dev/null +++ b/sickbeard/indexers/test/__init__.py @@ -0,0 +1 @@ +__author__ = 'Justin' diff --git a/sickbeard/indexers/test/test.py b/sickbeard/indexers/test/test.py new file mode 100644 index 00000000..e73c5bbb --- /dev/null +++ b/sickbeard/indexers/test/test.py @@ -0,0 +1,51 @@ +from __future__ import with_statement + +import unittest + +import sqlite3 + +import sys +import os.path +sys.path.append(os.path.abspath('..')) +sys.path.append(os.path.abspath('../lib')) + +import sickbeard +import shutil + +from sickbeard import encodingKludge as ek, providers, tvcache +from sickbeard import db +from sickbeard.databases import mainDB +from sickbeard.databases import cache_db + + +from indexer_api import indexerApi +from indexer_exceptions import indexer_exception + +class APICheck(unittest.TestCase): + indexer_id = 258171 + indexer = 'Tvdb' + # Set our common indexer_api options here + INDEXER_API_PARMS = {'apikey': '9DAF49C96CBF8DAC', + 'language': 'en', + 'useZip': True} + + + INDEXER_API_PARMS['indexer'] = indexer + lindexer_api_parms = INDEXER_API_PARMS.copy() + + try: + imdbid = " " + showurl = indexerApi(**lindexer_api_parms).config['base_url'] + indexer_id + '/all/en.zip' + t = indexerApi().config['valid_languages'] + t = indexerApi(**lindexer_api_parms) + myEp = t[258171] + + if getattr(myEp, 'seriesname', None) is not None: + print "FOUND" + + except indexer_exception as e: + print e + pass + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/sickbeard/indexers/test/test_lib.py b/sickbeard/indexers/test/test_lib.py new file mode 100644 index 00000000..0357c389 --- /dev/null +++ b/sickbeard/indexers/test/test_lib.py @@ -0,0 +1,227 @@ +# coding=UTF-8 +# Author: Dennis Lutter +# URL: http://code.google.com/p/sickbeard/ +# +# This file is part of Sick Beard. +# +# Sick Beard is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Sick Beard is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Sick Beard. If not, see . + +from __future__ import with_statement + +import unittest + +import sqlite3 + +import sys +import os.path +sys.path.append(os.path.abspath('..')) +sys.path.append(os.path.abspath('../lib')) + +import sickbeard +import shutil + +from sickbeard import encodingKludge as ek, providers, tvcache +from sickbeard import db +from sickbeard.databases import mainDB +from sickbeard.databases import cache_db + +#================= +# test globals +#================= +TESTDIR = os.path.abspath('.') +TESTDBNAME = "sickbeard.db" +TESTCACHEDBNAME = "cache.db" + + +SHOWNAME = u"show name" +SEASON = 4 +EPISODE = 2 +FILENAME = u"show name - s0" + str(SEASON) + "e0" + str(EPISODE) + ".mkv" +FILEDIR = os.path.join(TESTDIR, SHOWNAME) +FILEPATH = os.path.join(FILEDIR, FILENAME) + +SHOWDIR = os.path.join(TESTDIR, SHOWNAME + " final") + +#sickbeard.logger.sb_log_instance = sickbeard.logger.SBRotatingLogHandler(os.path.join(TESTDIR, 'sickbeard.log'), sickbeard.logger.NUM_LOGS, sickbeard.logger.LOG_SIZE) +sickbeard.logger.SBRotatingLogHandler.log_file = os.path.join(os.path.join(TESTDIR, 'Logs'), 'test_sickbeard.log') + + +#================= +# prepare env functions +#================= +def createTestLogFolder(): + if not os.path.isdir(sickbeard.LOG_DIR): + os.mkdir(sickbeard.LOG_DIR) + +# call env functions at appropriate time during sickbeard var setup + +#================= +# sickbeard globals +#================= +sickbeard.SYS_ENCODING = 'UTF-8' +sickbeard.showList = [] +sickbeard.QUALITY_DEFAULT = 4 # hdtv +sickbeard.FLATTEN_FOLDERS_DEFAULT = 0 + +sickbeard.NAMING_PATTERN = '' +sickbeard.NAMING_ABD_PATTERN = '' +sickbeard.NAMING_MULTI_EP = 1 + + +sickbeard.PROVIDER_ORDER = ["sick_beard_index"] +sickbeard.newznabProviderList = providers.getNewznabProviderList("Sick Beard Index|http://lolo.sickbeard.com/|0|5030,5040|0!!!NZBs.org|http://nzbs.org/||5030,5040,5070,5090|0!!!Usenet-Crawler|http://www.usenet-crawler.com/||5030,5040|0") +sickbeard.providerList = providers.makeProviderList() + +sickbeard.PROG_DIR = os.path.abspath('..') +sickbeard.DATA_DIR = sickbeard.PROG_DIR +sickbeard.LOG_DIR = os.path.join(TESTDIR, 'Logs') +createTestLogFolder() +sickbeard.logger.sb_log_instance.initLogging(False) + + +#================= +# dummy functions +#================= +def _dummy_saveConfig(): + return True +# this overrides the sickbeard save_config which gets called during a db upgrade +# this might be considered a hack +mainDB.sickbeard.save_config = _dummy_saveConfig + + +# the real one tries to contact tvdb just stop it from getting more info on the ep +def _fake_specifyEP(self, season, episode): + pass + +sickbeard.tv.TVEpisode.specifyEpisode = _fake_specifyEP + + +#================= +# test classes +#================= +class SickbeardTestDBCase(unittest.TestCase): + def setUp(self): + sickbeard.showList = [] + setUp_test_db() + setUp_test_episode_file() + setUp_test_show_dir() + + def tearDown(self): + sickbeard.showList = [] + tearDown_test_db() + tearDown_test_episode_file() + tearDown_test_show_dir() + + +class TestDBConnection(db.DBConnection, object): + + def __init__(self, dbFileName=TESTDBNAME): + dbFileName = os.path.join(TESTDIR, dbFileName) + super(TestDBConnection, self).__init__(dbFileName) + + +class TestCacheDBConnection(TestDBConnection, object): + + def __init__(self, providerName): + db.DBConnection.__init__(self, os.path.join(TESTDIR, TESTCACHEDBNAME)) + + # Create the table if it's not already there + try: + sql = "CREATE TABLE " + providerName + " (name TEXT, season NUMERIC, episodes TEXT, indexerid NUMERIC, url TEXT, time NUMERIC, quality TEXT);" + self.connection.execute(sql) + self.connection.commit() + except sqlite3.OperationalError, e: + if str(e) != "table " + providerName + " already exists": + raise + + # Create the table if it's not already there + try: + sql = "CREATE TABLE lastUpdate (provider TEXT, time NUMERIC);" + self.connection.execute(sql) + self.connection.commit() + except sqlite3.OperationalError, e: + if str(e) != "table lastUpdate already exists": + raise + +# this will override the normal db connection +sickbeard.db.DBConnection = TestDBConnection +sickbeard.tvcache.CacheDBConnection = TestCacheDBConnection + + +#================= +# test functions +#================= +def setUp_test_db(): + """upgrades the db to the latest version + """ + # upgrading the db + db.upgradeDatabase(db.DBConnection(), mainDB.InitialSchema) + # fix up any db problems + db.sanityCheckDatabase(db.DBConnection(), mainDB.MainSanityCheck) + + #and for cache.b too + db.upgradeDatabase(db.DBConnection("cache.db"), cache_db.InitialSchema) + + +def tearDown_test_db(): + """Deletes the test db + although this seams not to work on my system it leaves me with an zero kb file + """ + # uncomment next line so leave the db intact between test and at the end + #return False + if os.path.exists(os.path.join(TESTDIR, TESTDBNAME)): + os.remove(os.path.join(TESTDIR, TESTDBNAME)) + if os.path.exists(os.path.join(TESTDIR, TESTCACHEDBNAME)): + os.remove(os.path.join(TESTDIR, TESTCACHEDBNAME)) + + +def setUp_test_episode_file(): + if not os.path.exists(FILEDIR): + os.makedirs(FILEDIR) + + try: + with open(FILEPATH, 'w') as f: + f.write("foo bar") + except EnvironmentError: + print "Unable to set up test episode" + raise + + +def tearDown_test_episode_file(): + shutil.rmtree(FILEDIR) + + +def setUp_test_show_dir(): + if not os.path.exists(SHOWDIR): + os.makedirs(SHOWDIR) + + +def tearDown_test_show_dir(): + shutil.rmtree(SHOWDIR) + +tearDown_test_db() + +if __name__ == '__main__': + print "==================" + print "Dont call this directly" + print "==================" + print "you might want to call" + + dirList = os.listdir(TESTDIR) + for fname in dirList: + if (fname.find("_test") > 0) and (fname.find("pyc") < 0): + print "- " + fname + + print "==================" + print "or just call all_tests.py" diff --git a/sickbeard/metadata/generic.py b/sickbeard/metadata/generic.py index 2814a64f..785309d2 100644 --- a/sickbeard/metadata/generic.py +++ b/sickbeard/metadata/generic.py @@ -746,7 +746,7 @@ class GenericMetadata(): if getattr(indexer_show_obj, 'banner', None) is not None: image_url = re.sub('graphical', '_cache/graphical', indexer_show_obj['banner']) else: - if getattr(indexer_show_obj, 'banner', None) is not None: + if getattr(indexer_show_obj, image_type, None) is not None: image_url = indexer_show_obj[image_type] # Try and get posters and fanart from TMDB diff --git a/sickbeard/metadata/xbmc_12plus.py b/sickbeard/metadata/xbmc_12plus.py index 941f65e9..d0b4d5da 100644 --- a/sickbeard/metadata/xbmc_12plus.py +++ b/sickbeard/metadata/xbmc_12plus.py @@ -153,7 +153,7 @@ class XBMC_12PlusMetadata(generic.GenericMetadata): episodeguideurl = etree.SubElement(episodeguide, "url") episodeguideurl2 = etree.SubElement(tv_node, "episodeguideurl") if getattr(myShow, 'id', None) is not None: - showurl = indexer_api.indexerApi(show_obj.indexer).config['base_url'] + myShow["id"] + '/all/en.zip' + showurl = t.config['base_url'] + myShow["id"] + '/all/en.zip' episodeguideurl.text = showurl episodeguideurl2.text = showurl diff --git a/sickbeard/show_queue.py b/sickbeard/show_queue.py index 27ba14c4..375b839a 100644 --- a/sickbeard/show_queue.py +++ b/sickbeard/show_queue.py @@ -347,7 +347,7 @@ class QueueItemAdd(ShowQueueItem): sickbeard.backlogSearchScheduler.action.searchBacklog([self.show]) #@UndefinedVariable self.show.writeMetadata(force=self.refresh) - self.show.populateCache() + self.show.populateCache() self.show.flushEpisodes() diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index ae630ec3..f6c789ac 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -2266,7 +2266,7 @@ class NewHomeAddShows: if indexer is not None and indexer_id is not None: # add the show - sickbeard.showQueueScheduler.action.addShow(indexer, indexer_id, show_dir, SKIPPED, sickbeard.QUALITY_DEFAULT, sickbeard.FLATTEN_FOLDERS_DEFAULT, sickbeard.SUBTITLES_DEFAULT, refresh=True) # @UndefinedVariable + sickbeard.showQueueScheduler.action.addShow(indexer, indexer_id, show_dir, SKIPPED, sickbeard.QUALITY_DEFAULT, sickbeard.FLATTEN_FOLDERS_DEFAULT, sickbeard.SUBTITLES_DEFAULT, refresh=True) num_added += 1 if num_added: diff --git a/tests/tv_tests.py b/tests/tv_tests.py index ce957337..1eb9f014 100644 --- a/tests/tv_tests.py +++ b/tests/tv_tests.py @@ -23,7 +23,6 @@ import test_lib as test import sickbeard from sickbeard.tv import TVEpisode, TVShow - class TVShowTests(test.SickbeardTestDBCase): def setUp(self):