mirror of
https://github.com/moparisthebest/SickRage
synced 2024-08-13 16:53:54 -04:00
SickRage now uses a internal name cache comprised of (showlist, scene exceptions, scene names) used to search and create show objects from which is needed for parsing search results.
Scene exceptions now uses a internal cache for scene exceptions and scene season exceptions, helps reduce overhead to DB and performs faster lookups when making scene exception requests.
This commit is contained in:
parent
f0146f728e
commit
56e2c28bad
@ -107,9 +107,6 @@ class DailySearcher():
|
|||||||
for show in todaysEps:
|
for show in todaysEps:
|
||||||
segment = todaysEps[show]
|
segment = todaysEps[show]
|
||||||
|
|
||||||
# remove show from name cache if marked invalid
|
|
||||||
sickbeard.name_cache.clearCache(show)
|
|
||||||
|
|
||||||
dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem(show, segment)
|
dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem(show, segment)
|
||||||
sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item)
|
sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item)
|
||||||
else:
|
else:
|
||||||
|
@ -164,6 +164,8 @@ class DBConnection(object):
|
|||||||
logger.log(u"Fatal error executing query: " + ex(e), logger.ERROR)
|
logger.log(u"Fatal error executing query: " + ex(e), logger.ERROR)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
time.sleep(0.05)
|
||||||
|
|
||||||
return sqlResult
|
return sqlResult
|
||||||
|
|
||||||
def action(self, query, args=None, fetchall=False, fetchone=False):
|
def action(self, query, args=None, fetchall=False, fetchone=False):
|
||||||
@ -199,6 +201,8 @@ class DBConnection(object):
|
|||||||
logger.log(u"Fatal error executing query: " + ex(e), logger.ERROR)
|
logger.log(u"Fatal error executing query: " + ex(e), logger.ERROR)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
time.sleep(0.05)
|
||||||
|
|
||||||
return sqlResult
|
return sqlResult
|
||||||
|
|
||||||
def select(self, query, args=None):
|
def select(self, query, args=None):
|
||||||
|
@ -730,7 +730,6 @@ def sanitizeSceneName(name, ezrss=False):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
if name:
|
if name:
|
||||||
|
|
||||||
if not ezrss:
|
if not ezrss:
|
||||||
bad_chars = u",:()'!?\u2019"
|
bad_chars = u",:()'!?\u2019"
|
||||||
# ezrss leaves : and ! in their show names as far as I can tell
|
# ezrss leaves : and ! in their show names as far as I can tell
|
||||||
@ -1066,37 +1065,24 @@ def _check_against_names(nameInQuestion, show, season=-1):
|
|||||||
|
|
||||||
|
|
||||||
def get_show_by_name(name, useIndexer=False):
|
def get_show_by_name(name, useIndexer=False):
|
||||||
name = full_sanitizeSceneName(name)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# check cache for show
|
# check cache for show
|
||||||
showObj = sickbeard.name_cache.retrieveShowFromCache(name)
|
showObj = sickbeard.name_cache.retrieveShowFromCache(name)
|
||||||
if showObj:
|
if showObj:
|
||||||
return showObj
|
return showObj
|
||||||
|
|
||||||
if not showObj and sickbeard.showList:
|
if useIndexer and sickbeard.showList and not showObj:
|
||||||
db_indexerid = searchDBForShow(name)
|
(sn, idx, id) = searchIndexerForShowID(full_sanitizeSceneName(name), ui=classes.ShowListUI)
|
||||||
if db_indexerid:
|
|
||||||
showObj = findCertainShow(sickbeard.showList, db_indexerid)
|
|
||||||
|
|
||||||
if not showObj:
|
|
||||||
scene_indexerid, scene_season = sickbeard.scene_exceptions.get_scene_exception_by_name(name)
|
|
||||||
if scene_indexerid:
|
|
||||||
showObj = findCertainShow(sickbeard.showList, scene_indexerid)
|
|
||||||
|
|
||||||
if useIndexer and not showObj:
|
|
||||||
(sn, idx, id) = searchIndexerForShowID(name, ui=classes.ShowListUI)
|
|
||||||
if id:
|
if id:
|
||||||
showObj = findCertainShow(sickbeard.showList, int(id))
|
showObj = findCertainShow(sickbeard.showList, int(id))
|
||||||
|
|
||||||
# add show to cache
|
# add show to cache
|
||||||
if showObj:
|
if showObj:
|
||||||
sickbeard.name_cache.addNameToCache(name, showObj.indexerid)
|
sickbeard.name_cache.addNameToCache(name, showObj.indexerid)
|
||||||
except:
|
|
||||||
showObj = None
|
|
||||||
|
|
||||||
return showObj
|
return showObj
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
def is_hidden_folder(folder):
|
def is_hidden_folder(folder):
|
||||||
"""
|
"""
|
||||||
|
@ -23,7 +23,7 @@ import sickbeard
|
|||||||
from sickbeard import scene_exceptions
|
from sickbeard import scene_exceptions
|
||||||
from sickbeard import failed_history
|
from sickbeard import failed_history
|
||||||
from sickbeard import network_timezones
|
from sickbeard import network_timezones
|
||||||
|
from sickbeard import name_cache
|
||||||
|
|
||||||
class Maintenance():
|
class Maintenance():
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
@ -31,15 +31,18 @@ class Maintenance():
|
|||||||
|
|
||||||
self.amActive = False
|
self.amActive = False
|
||||||
|
|
||||||
def __del__(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def run(self, force=False):
|
def run(self, force=False):
|
||||||
self.amActive = True
|
self.amActive = True
|
||||||
|
|
||||||
|
# clear internal name cache
|
||||||
|
name_cache.clearCache()
|
||||||
|
|
||||||
# get and update scene exceptions lists
|
# get and update scene exceptions lists
|
||||||
scene_exceptions.retrieve_exceptions()
|
scene_exceptions.retrieve_exceptions()
|
||||||
|
|
||||||
|
# build internal name cache for searches and parsing
|
||||||
|
name_cache.buildNameCache()
|
||||||
|
|
||||||
# refresh network timezones
|
# refresh network timezones
|
||||||
network_timezones.update_network_dict()
|
network_timezones.update_network_dict()
|
||||||
|
|
||||||
|
@ -22,6 +22,8 @@ from sickbeard import db
|
|||||||
from sickbeard.helpers import sanitizeSceneName
|
from sickbeard.helpers import sanitizeSceneName
|
||||||
from sickbeard import logger
|
from sickbeard import logger
|
||||||
|
|
||||||
|
nameCache = None
|
||||||
|
|
||||||
def addNameToCache(name, indexer_id=0):
|
def addNameToCache(name, indexer_id=0):
|
||||||
"""
|
"""
|
||||||
Adds the show & tvdb id to the scene_names table in cache.db.
|
Adds the show & tvdb id to the scene_names table in cache.db.
|
||||||
@ -29,11 +31,15 @@ def addNameToCache(name, indexer_id=0):
|
|||||||
name: The show name to cache
|
name: The show name to cache
|
||||||
indexer_id: the TVDB and TVRAGE id that this show should be cached with (can be None/0 for unknown)
|
indexer_id: the TVDB and TVRAGE id that this show should be cached with (can be None/0 for unknown)
|
||||||
"""
|
"""
|
||||||
|
global nameCache
|
||||||
|
|
||||||
|
cacheDB = db.DBConnection('cache.db')
|
||||||
|
|
||||||
# standardize the name we're using to account for small differences in providers
|
# standardize the name we're using to account for small differences in providers
|
||||||
name = sanitizeSceneName(name)
|
name = sickbeard.helpers.full_sanitizeSceneName(name)
|
||||||
myDB = db.DBConnection('cache.db')
|
if name not in nameCache:
|
||||||
myDB.action("INSERT INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name])
|
nameCache[name] = int(indexer_id)
|
||||||
|
cacheDB.action("INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name])
|
||||||
|
|
||||||
|
|
||||||
def retrieveNameFromCache(name):
|
def retrieveNameFromCache(name):
|
||||||
@ -44,33 +50,66 @@ def retrieveNameFromCache(name):
|
|||||||
|
|
||||||
Returns: the TVDB and TVRAGE id that resulted from the cache lookup or None if the show wasn't found in the cache
|
Returns: the TVDB and TVRAGE id that resulted from the cache lookup or None if the show wasn't found in the cache
|
||||||
"""
|
"""
|
||||||
|
global nameCache
|
||||||
|
|
||||||
cache_results = None
|
name = sickbeard.helpers.full_sanitizeSceneName(name)
|
||||||
|
if name in nameCache:
|
||||||
# standardize the name we're using to account for small differences in providers
|
return int(nameCache[name])
|
||||||
name = sanitizeSceneName(name)
|
|
||||||
|
|
||||||
myDB = db.DBConnection('cache.db')
|
|
||||||
if myDB.hasTable('scene_names'):
|
|
||||||
cache_results = myDB.select("SELECT * FROM scene_names WHERE name = ?", [name])
|
|
||||||
|
|
||||||
if cache_results:
|
|
||||||
return int(cache_results[0]["indexer_id"])
|
|
||||||
|
|
||||||
def retrieveShowFromCache(name):
|
def retrieveShowFromCache(name):
|
||||||
indexerid = retrieveNameFromCache(name)
|
global nameCache
|
||||||
if indexerid:
|
|
||||||
return sickbeard.helpers.findCertainShow(sickbeard.showList, int(indexerid))
|
|
||||||
|
|
||||||
def clearCache(show=None, season=-1, indexer_id=0):
|
indexer_id = retrieveNameFromCache(name)
|
||||||
|
if indexer_id:
|
||||||
|
return sickbeard.helpers.findCertainShow(sickbeard.showList, int(indexer_id))
|
||||||
|
|
||||||
|
def clearCache():
|
||||||
"""
|
"""
|
||||||
Deletes all "unknown" entries from the cache (names with indexer_id of 0).
|
Deletes all "unknown" entries from the cache (names with indexer_id of 0).
|
||||||
"""
|
"""
|
||||||
|
global nameCache
|
||||||
|
|
||||||
myDB = db.DBConnection('cache.db')
|
# init name cache
|
||||||
if show:
|
if not nameCache:
|
||||||
showNames = sickbeard.show_name_helpers.allPossibleShowNames(show, season=season)
|
nameCache = {}
|
||||||
for showName in showNames:
|
|
||||||
myDB.action("DELETE FROM scene_names WHERE name = ? and indexer_id = ?", [showName, indexer_id])
|
cacheDB = db.DBConnection('cache.db')
|
||||||
else:
|
cacheDB.action("DELETE FROM scene_names WHERE indexer_id = ?", [0])
|
||||||
myDB.action("DELETE FROM scene_names WHERE indexer_id = ?", [indexer_id])
|
|
||||||
|
toRemove = [key for key, value in nameCache.iteritems() if value == 0]
|
||||||
|
for key in toRemove:
|
||||||
|
del nameCache[key]
|
||||||
|
|
||||||
|
def saveNameCacheToDb():
|
||||||
|
cacheDB = db.DBConnection('cache.db')
|
||||||
|
|
||||||
|
for name, indexer_id in nameCache.items():
|
||||||
|
cacheDB.action("INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name])
|
||||||
|
|
||||||
|
def buildNameCache():
|
||||||
|
global nameCache
|
||||||
|
|
||||||
|
# init name cache
|
||||||
|
if not nameCache:
|
||||||
|
nameCache = {}
|
||||||
|
|
||||||
|
# clear internal name cache
|
||||||
|
clearCache()
|
||||||
|
|
||||||
|
logger.log(u"Updating internal name cache", logger.MESSAGE)
|
||||||
|
|
||||||
|
cacheDB = db.DBConnection('cache.db')
|
||||||
|
cache_results = cacheDB.select("SELECT * FROM scene_names")
|
||||||
|
for cache_result in cache_results:
|
||||||
|
name = sickbeard.helpers.full_sanitizeSceneName(cache_result["name"])
|
||||||
|
indexer_id = int(cache_result["indexer_id"])
|
||||||
|
nameCache[name] = indexer_id
|
||||||
|
|
||||||
|
for show in sickbeard.showList:
|
||||||
|
for curSeason in [-1] + sickbeard.scene_exceptions.get_scene_seasons(show.indexerid):
|
||||||
|
nameCache[sickbeard.helpers.full_sanitizeSceneName(show.name)] = show.indexerid
|
||||||
|
for name in sickbeard.scene_exceptions.get_scene_exceptions(show.indexerid, season=curSeason):
|
||||||
|
nameCache[sickbeard.helpers.full_sanitizeSceneName(name)] = show.indexerid
|
||||||
|
|
||||||
|
logger.log(u"Updated internal name cache", logger.MESSAGE)
|
||||||
|
logger.log(u"Internal name cache set to: " + str(nameCache), logger.DEBUG)
|
@ -343,8 +343,6 @@ class BTNCache(tvcache.TVCache):
|
|||||||
if ci is not None:
|
if ci is not None:
|
||||||
cl.append(ci)
|
cl.append(ci)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if cl:
|
if cl:
|
||||||
myDB = self._getDB()
|
myDB = self._getDB()
|
||||||
myDB.mass_action(cl)
|
myDB.mass_action(cl)
|
||||||
|
@ -27,7 +27,8 @@ from sickbeard import name_cache
|
|||||||
from sickbeard import logger
|
from sickbeard import logger
|
||||||
from sickbeard import db
|
from sickbeard import db
|
||||||
|
|
||||||
scene_lock = threading.Lock()
|
exceptionsCache = None
|
||||||
|
exceptionsSeasonCache = None
|
||||||
|
|
||||||
def shouldRefresh(list):
|
def shouldRefresh(list):
|
||||||
MAX_REFRESH_AGE_SECS = 86400 # 1 day
|
MAX_REFRESH_AGE_SECS = 86400 # 1 day
|
||||||
@ -51,15 +52,22 @@ def get_scene_exceptions(indexer_id, season=-1):
|
|||||||
"""
|
"""
|
||||||
Given a indexer_id, return a list of all the scene exceptions.
|
Given a indexer_id, return a list of all the scene exceptions.
|
||||||
"""
|
"""
|
||||||
|
global exceptionsCache
|
||||||
exceptionsList = []
|
exceptionsList = []
|
||||||
|
|
||||||
|
if indexer_id not in exceptionsCache or season not in exceptionsCache[indexer_id]:
|
||||||
myDB = db.DBConnection('cache.db')
|
myDB = db.DBConnection('cache.db')
|
||||||
exceptions = myDB.select("SELECT show_name FROM scene_exceptions WHERE indexer_id = ? and season = ?",
|
exceptions = myDB.select("SELECT show_name FROM scene_exceptions WHERE indexer_id = ? and season = ?",
|
||||||
[indexer_id, season])
|
[indexer_id, season])
|
||||||
if exceptions:
|
if exceptions:
|
||||||
exceptionsList = list(set([cur_exception["show_name"] for cur_exception in exceptions]))
|
exceptionsList = list(set([cur_exception["show_name"] for cur_exception in exceptions]))
|
||||||
|
|
||||||
|
if not indexer_id in exceptionsCache:
|
||||||
|
exceptionsCache[indexer_id] = {}
|
||||||
|
exceptionsCache[indexer_id][season] = exceptionsList
|
||||||
|
else:
|
||||||
|
exceptionsList = exceptionsCache[indexer_id][season]
|
||||||
|
|
||||||
if season == 1: # if we where looking for season 1 we can add generic names
|
if season == 1: # if we where looking for season 1 we can add generic names
|
||||||
exceptionsList += get_scene_exceptions(indexer_id, season=-1)
|
exceptionsList += get_scene_exceptions(indexer_id, season=-1)
|
||||||
|
|
||||||
@ -84,14 +92,24 @@ def get_scene_seasons(indexer_id):
|
|||||||
"""
|
"""
|
||||||
return a list of season numbers that have scene exceptions
|
return a list of season numbers that have scene exceptions
|
||||||
"""
|
"""
|
||||||
|
global exceptionsSeasonCache
|
||||||
|
exceptionsSeasonList = []
|
||||||
|
|
||||||
|
if indexer_id not in exceptionsSeasonCache:
|
||||||
myDB = db.DBConnection('cache.db')
|
myDB = db.DBConnection('cache.db')
|
||||||
sqlResults = myDB.select("SELECT DISTINCT(season) as season FROM scene_exceptions WHERE indexer_id = ?",
|
sqlResults = myDB.select("SELECT DISTINCT(season) as season FROM scene_exceptions WHERE indexer_id = ?",
|
||||||
[indexer_id])
|
[indexer_id])
|
||||||
|
|
||||||
if sqlResults:
|
if sqlResults:
|
||||||
return [int(x["season"]) for x in sqlResults]
|
exceptionsSeasonList = list(set([int(x["season"]) for x in sqlResults]))
|
||||||
|
|
||||||
|
if not indexer_id in exceptionsSeasonCache:
|
||||||
|
exceptionsSeasonCache[indexer_id] = {}
|
||||||
|
|
||||||
|
exceptionsSeasonCache[indexer_id] = exceptionsSeasonList
|
||||||
|
else:
|
||||||
|
exceptionsSeasonList = exceptionsSeasonCache[indexer_id]
|
||||||
|
|
||||||
|
return exceptionsSeasonList
|
||||||
|
|
||||||
def get_scene_exception_by_name(show_name):
|
def get_scene_exception_by_name(show_name):
|
||||||
return get_scene_exception_by_name_multiple(show_name)[0]
|
return get_scene_exception_by_name_multiple(show_name)[0]
|
||||||
@ -136,7 +154,10 @@ def retrieve_exceptions():
|
|||||||
Looks up the exceptions on github, parses them into a dict, and inserts them into the
|
Looks up the exceptions on github, parses them into a dict, and inserts them into the
|
||||||
scene_exceptions table in cache.db. Also clears the scene name cache.
|
scene_exceptions table in cache.db. Also clears the scene name cache.
|
||||||
"""
|
"""
|
||||||
|
global exceptionsCache, exceptionsSeasonCache
|
||||||
|
|
||||||
|
exceptionsCache = {}
|
||||||
|
exceptionsSeasonCache = {}
|
||||||
exception_dict = {}
|
exception_dict = {}
|
||||||
|
|
||||||
# exceptions are stored on github pages
|
# exceptions are stored on github pages
|
||||||
@ -209,7 +230,6 @@ def retrieve_exceptions():
|
|||||||
# since this could invalidate the results of the cache we clear it out after updating
|
# since this could invalidate the results of the cache we clear it out after updating
|
||||||
if changed_exceptions:
|
if changed_exceptions:
|
||||||
logger.log(u"Updated scene exceptions")
|
logger.log(u"Updated scene exceptions")
|
||||||
name_cache.clearCache()
|
|
||||||
else:
|
else:
|
||||||
logger.log(u"No scene exceptions update needed")
|
logger.log(u"No scene exceptions update needed")
|
||||||
|
|
||||||
@ -230,8 +250,6 @@ def update_scene_exceptions(indexer_id, scene_exceptions):
|
|||||||
myDB.action("INSERT INTO scene_exceptions (indexer_id, show_name, season, custom) VALUES (?,?,?,?)",
|
myDB.action("INSERT INTO scene_exceptions (indexer_id, show_name, season, custom) VALUES (?,?,?,?)",
|
||||||
[indexer_id, cur_exception, cur_season, 1])
|
[indexer_id, cur_exception, cur_season, 1])
|
||||||
|
|
||||||
name_cache.clearCache()
|
|
||||||
|
|
||||||
def _retrieve_anidb_mainnames():
|
def _retrieve_anidb_mainnames():
|
||||||
|
|
||||||
anidb_mainNames = {}
|
anidb_mainNames = {}
|
||||||
|
@ -279,9 +279,6 @@ class QueueItemAdd(ShowQueueItem):
|
|||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# clear the name cache
|
|
||||||
name_cache.clearCache()
|
|
||||||
|
|
||||||
newShow = TVShow(self.indexer, self.indexer_id, self.lang)
|
newShow = TVShow(self.indexer, self.indexer_id, self.lang)
|
||||||
newShow.loadFromIndexer()
|
newShow.loadFromIndexer()
|
||||||
|
|
||||||
@ -361,6 +358,9 @@ class QueueItemAdd(ShowQueueItem):
|
|||||||
# before we parse local files lets update exceptions
|
# before we parse local files lets update exceptions
|
||||||
sickbeard.scene_exceptions.retrieve_exceptions()
|
sickbeard.scene_exceptions.retrieve_exceptions()
|
||||||
|
|
||||||
|
# update internal name cache
|
||||||
|
name_cache.buildNameCache()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.show.loadEpisodesFromDir()
|
self.show.loadEpisodesFromDir()
|
||||||
except Exception, e:
|
except Exception, e:
|
||||||
|
@ -128,8 +128,6 @@ class TVCache():
|
|||||||
if ci is not None:
|
if ci is not None:
|
||||||
cl.append(ci)
|
cl.append(ci)
|
||||||
|
|
||||||
time.sleep(.2)
|
|
||||||
|
|
||||||
if cl:
|
if cl:
|
||||||
myDB = self._getDB()
|
myDB = self._getDB()
|
||||||
myDB.mass_action(cl)
|
myDB.mass_action(cl)
|
||||||
|
Loading…
Reference in New Issue
Block a user