diff --git a/sickbeard/dailysearcher.py b/sickbeard/dailysearcher.py index 967d3a47..ac528d1e 100644 --- a/sickbeard/dailysearcher.py +++ b/sickbeard/dailysearcher.py @@ -107,9 +107,6 @@ class DailySearcher(): for show in todaysEps: segment = todaysEps[show] - # remove show from name cache if marked invalid - sickbeard.name_cache.clearCache(show) - dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem(show, segment) sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item) else: diff --git a/sickbeard/db.py b/sickbeard/db.py index 92f6d917..c42c675c 100644 --- a/sickbeard/db.py +++ b/sickbeard/db.py @@ -164,6 +164,8 @@ class DBConnection(object): logger.log(u"Fatal error executing query: " + ex(e), logger.ERROR) raise + time.sleep(0.05) + return sqlResult def action(self, query, args=None, fetchall=False, fetchone=False): @@ -199,6 +201,8 @@ class DBConnection(object): logger.log(u"Fatal error executing query: " + ex(e), logger.ERROR) raise + time.sleep(0.05) + return sqlResult def select(self, query, args=None): diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index 18f1133a..e1b943a0 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -730,7 +730,6 @@ def sanitizeSceneName(name, ezrss=False): """ if name: - if not ezrss: bad_chars = u",:()'!?\u2019" # ezrss leaves : and ! in their show names as far as I can tell @@ -1066,37 +1065,24 @@ def _check_against_names(nameInQuestion, show, season=-1): def get_show_by_name(name, useIndexer=False): - name = full_sanitizeSceneName(name) - try: # check cache for show showObj = sickbeard.name_cache.retrieveShowFromCache(name) if showObj: return showObj - if not showObj and sickbeard.showList: - db_indexerid = searchDBForShow(name) - if db_indexerid: - showObj = findCertainShow(sickbeard.showList, db_indexerid) - - if not showObj: - scene_indexerid, scene_season = sickbeard.scene_exceptions.get_scene_exception_by_name(name) - if scene_indexerid: - showObj = findCertainShow(sickbeard.showList, scene_indexerid) - - if useIndexer and not showObj: - (sn, idx, id) = searchIndexerForShowID(name, ui=classes.ShowListUI) - if id: - showObj = findCertainShow(sickbeard.showList, int(id)) + if useIndexer and sickbeard.showList and not showObj: + (sn, idx, id) = searchIndexerForShowID(full_sanitizeSceneName(name), ui=classes.ShowListUI) + if id: + showObj = findCertainShow(sickbeard.showList, int(id)) # add show to cache if showObj: sickbeard.name_cache.addNameToCache(name, showObj.indexerid) + + return showObj except: - showObj = None - - return showObj - + pass def is_hidden_folder(folder): """ diff --git a/sickbeard/maintenance.py b/sickbeard/maintenance.py index 9bce6481..ccc3f871 100644 --- a/sickbeard/maintenance.py +++ b/sickbeard/maintenance.py @@ -23,7 +23,7 @@ import sickbeard from sickbeard import scene_exceptions from sickbeard import failed_history from sickbeard import network_timezones - +from sickbeard import name_cache class Maintenance(): def __init__(self): @@ -31,15 +31,18 @@ class Maintenance(): self.amActive = False - def __del__(self): - pass - def run(self, force=False): self.amActive = True + # clear internal name cache + name_cache.clearCache() + # get and update scene exceptions lists scene_exceptions.retrieve_exceptions() + # build internal name cache for searches and parsing + name_cache.buildNameCache() + # refresh network timezones network_timezones.update_network_dict() diff --git a/sickbeard/name_cache.py b/sickbeard/name_cache.py index cdcd351a..6661b89d 100644 --- a/sickbeard/name_cache.py +++ b/sickbeard/name_cache.py @@ -22,6 +22,8 @@ from sickbeard import db from sickbeard.helpers import sanitizeSceneName from sickbeard import logger +nameCache = None + def addNameToCache(name, indexer_id=0): """ Adds the show & tvdb id to the scene_names table in cache.db. @@ -29,11 +31,15 @@ def addNameToCache(name, indexer_id=0): name: The show name to cache indexer_id: the TVDB and TVRAGE id that this show should be cached with (can be None/0 for unknown) """ + global nameCache + + cacheDB = db.DBConnection('cache.db') # standardize the name we're using to account for small differences in providers - name = sanitizeSceneName(name) - myDB = db.DBConnection('cache.db') - myDB.action("INSERT INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name]) + name = sickbeard.helpers.full_sanitizeSceneName(name) + if name not in nameCache: + nameCache[name] = int(indexer_id) + cacheDB.action("INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name]) def retrieveNameFromCache(name): @@ -44,33 +50,66 @@ def retrieveNameFromCache(name): Returns: the TVDB and TVRAGE id that resulted from the cache lookup or None if the show wasn't found in the cache """ + global nameCache - cache_results = None - - # standardize the name we're using to account for small differences in providers - name = sanitizeSceneName(name) - - myDB = db.DBConnection('cache.db') - if myDB.hasTable('scene_names'): - cache_results = myDB.select("SELECT * FROM scene_names WHERE name = ?", [name]) - - if cache_results: - return int(cache_results[0]["indexer_id"]) + name = sickbeard.helpers.full_sanitizeSceneName(name) + if name in nameCache: + return int(nameCache[name]) def retrieveShowFromCache(name): - indexerid = retrieveNameFromCache(name) - if indexerid: - return sickbeard.helpers.findCertainShow(sickbeard.showList, int(indexerid)) + global nameCache -def clearCache(show=None, season=-1, indexer_id=0): + indexer_id = retrieveNameFromCache(name) + if indexer_id: + return sickbeard.helpers.findCertainShow(sickbeard.showList, int(indexer_id)) + +def clearCache(): """ Deletes all "unknown" entries from the cache (names with indexer_id of 0). """ + global nameCache - myDB = db.DBConnection('cache.db') - if show: - showNames = sickbeard.show_name_helpers.allPossibleShowNames(show, season=season) - for showName in showNames: - myDB.action("DELETE FROM scene_names WHERE name = ? and indexer_id = ?", [showName, indexer_id]) - else: - myDB.action("DELETE FROM scene_names WHERE indexer_id = ?", [indexer_id]) \ No newline at end of file + # init name cache + if not nameCache: + nameCache = {} + + cacheDB = db.DBConnection('cache.db') + cacheDB.action("DELETE FROM scene_names WHERE indexer_id = ?", [0]) + + toRemove = [key for key, value in nameCache.iteritems() if value == 0] + for key in toRemove: + del nameCache[key] + +def saveNameCacheToDb(): + cacheDB = db.DBConnection('cache.db') + + for name, indexer_id in nameCache.items(): + cacheDB.action("INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name]) + +def buildNameCache(): + global nameCache + + # init name cache + if not nameCache: + nameCache = {} + + # clear internal name cache + clearCache() + + logger.log(u"Updating internal name cache", logger.MESSAGE) + + cacheDB = db.DBConnection('cache.db') + cache_results = cacheDB.select("SELECT * FROM scene_names") + for cache_result in cache_results: + name = sickbeard.helpers.full_sanitizeSceneName(cache_result["name"]) + indexer_id = int(cache_result["indexer_id"]) + nameCache[name] = indexer_id + + for show in sickbeard.showList: + for curSeason in [-1] + sickbeard.scene_exceptions.get_scene_seasons(show.indexerid): + nameCache[sickbeard.helpers.full_sanitizeSceneName(show.name)] = show.indexerid + for name in sickbeard.scene_exceptions.get_scene_exceptions(show.indexerid, season=curSeason): + nameCache[sickbeard.helpers.full_sanitizeSceneName(name)] = show.indexerid + + logger.log(u"Updated internal name cache", logger.MESSAGE) + logger.log(u"Internal name cache set to: " + str(nameCache), logger.DEBUG) \ No newline at end of file diff --git a/sickbeard/providers/btn.py b/sickbeard/providers/btn.py index 5dca36ea..71c69c48 100644 --- a/sickbeard/providers/btn.py +++ b/sickbeard/providers/btn.py @@ -343,8 +343,6 @@ class BTNCache(tvcache.TVCache): if ci is not None: cl.append(ci) - - if cl: myDB = self._getDB() myDB.mass_action(cl) diff --git a/sickbeard/scene_exceptions.py b/sickbeard/scene_exceptions.py index 5168548d..f59370b7 100644 --- a/sickbeard/scene_exceptions.py +++ b/sickbeard/scene_exceptions.py @@ -27,7 +27,8 @@ from sickbeard import name_cache from sickbeard import logger from sickbeard import db -scene_lock = threading.Lock() +exceptionsCache = None +exceptionsSeasonCache = None def shouldRefresh(list): MAX_REFRESH_AGE_SECS = 86400 # 1 day @@ -51,14 +52,21 @@ def get_scene_exceptions(indexer_id, season=-1): """ Given a indexer_id, return a list of all the scene exceptions. """ - + global exceptionsCache exceptionsList = [] - myDB = db.DBConnection('cache.db') - exceptions = myDB.select("SELECT show_name FROM scene_exceptions WHERE indexer_id = ? and season = ?", - [indexer_id, season]) - if exceptions: - exceptionsList = list(set([cur_exception["show_name"] for cur_exception in exceptions])) + if indexer_id not in exceptionsCache or season not in exceptionsCache[indexer_id]: + myDB = db.DBConnection('cache.db') + exceptions = myDB.select("SELECT show_name FROM scene_exceptions WHERE indexer_id = ? and season = ?", + [indexer_id, season]) + if exceptions: + exceptionsList = list(set([cur_exception["show_name"] for cur_exception in exceptions])) + + if not indexer_id in exceptionsCache: + exceptionsCache[indexer_id] = {} + exceptionsCache[indexer_id][season] = exceptionsList + else: + exceptionsList = exceptionsCache[indexer_id][season] if season == 1: # if we where looking for season 1 we can add generic names exceptionsList += get_scene_exceptions(indexer_id, season=-1) @@ -84,14 +92,24 @@ def get_scene_seasons(indexer_id): """ return a list of season numbers that have scene exceptions """ + global exceptionsSeasonCache + exceptionsSeasonList = [] - myDB = db.DBConnection('cache.db') - sqlResults = myDB.select("SELECT DISTINCT(season) as season FROM scene_exceptions WHERE indexer_id = ?", - [indexer_id]) + if indexer_id not in exceptionsSeasonCache: + myDB = db.DBConnection('cache.db') + sqlResults = myDB.select("SELECT DISTINCT(season) as season FROM scene_exceptions WHERE indexer_id = ?", + [indexer_id]) + if sqlResults: + exceptionsSeasonList = list(set([int(x["season"]) for x in sqlResults])) - if sqlResults: - return [int(x["season"]) for x in sqlResults] + if not indexer_id in exceptionsSeasonCache: + exceptionsSeasonCache[indexer_id] = {} + exceptionsSeasonCache[indexer_id] = exceptionsSeasonList + else: + exceptionsSeasonList = exceptionsSeasonCache[indexer_id] + + return exceptionsSeasonList def get_scene_exception_by_name(show_name): return get_scene_exception_by_name_multiple(show_name)[0] @@ -136,7 +154,10 @@ def retrieve_exceptions(): Looks up the exceptions on github, parses them into a dict, and inserts them into the scene_exceptions table in cache.db. Also clears the scene name cache. """ + global exceptionsCache, exceptionsSeasonCache + exceptionsCache = {} + exceptionsSeasonCache = {} exception_dict = {} # exceptions are stored on github pages @@ -209,7 +230,6 @@ def retrieve_exceptions(): # since this could invalidate the results of the cache we clear it out after updating if changed_exceptions: logger.log(u"Updated scene exceptions") - name_cache.clearCache() else: logger.log(u"No scene exceptions update needed") @@ -230,8 +250,6 @@ def update_scene_exceptions(indexer_id, scene_exceptions): myDB.action("INSERT INTO scene_exceptions (indexer_id, show_name, season, custom) VALUES (?,?,?,?)", [indexer_id, cur_exception, cur_season, 1]) - name_cache.clearCache() - def _retrieve_anidb_mainnames(): anidb_mainNames = {} diff --git a/sickbeard/show_queue.py b/sickbeard/show_queue.py index 35e9f13b..f85f39e2 100644 --- a/sickbeard/show_queue.py +++ b/sickbeard/show_queue.py @@ -279,9 +279,6 @@ class QueueItemAdd(ShowQueueItem): return try: - # clear the name cache - name_cache.clearCache() - newShow = TVShow(self.indexer, self.indexer_id, self.lang) newShow.loadFromIndexer() @@ -361,6 +358,9 @@ class QueueItemAdd(ShowQueueItem): # before we parse local files lets update exceptions sickbeard.scene_exceptions.retrieve_exceptions() + # update internal name cache + name_cache.buildNameCache() + try: self.show.loadEpisodesFromDir() except Exception, e: diff --git a/sickbeard/tvcache.py b/sickbeard/tvcache.py index da350e0b..d3905fbc 100644 --- a/sickbeard/tvcache.py +++ b/sickbeard/tvcache.py @@ -128,8 +128,6 @@ class TVCache(): if ci is not None: cl.append(ci) - time.sleep(.2) - if cl: myDB = self._getDB() myDB.mass_action(cl)