moparisthebest
/
SickRage
mirror of https://github.com/moparisthebest/SickRage
1
0
Fork 0
Code Issues Releases Wiki Activity

SickRage now uses a internal name cache comprised of (showlist, scene exceptions, scene names) used to search and create show objects from which is needed for parsing search results. 

Scene exceptions now uses a internal cache for scene exceptions and scene season exceptions, helps reduce overhead to DB and performs faster lookups when making scene exception requests.
This commit is contained in:
echel0n 2014-07-03 10:30:15 -07:00
parent f0146f728e
commit 56e2c28bad
9 changed files with 118 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
sickbeard/dailysearcher.py
View File

@ -107,9 +107,6 @@ class DailySearcher():
for show in todaysEps:
segment = todaysEps[show]
# remove show from name cache if marked invalid
sickbeard.name_cache.clearCache(show)
dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem(show, segment)
sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item)
else:

4
sickbeard/db.py
View File

@ -164,6 +164,8 @@ class DBConnection(object):
logger.log(u"Fatal error executing query: " + ex(e), logger.ERROR)
raise
time.sleep(0.05)
return sqlResult
def action(self, query, args=None, fetchall=False, fetchone=False):
@ -199,6 +201,8 @@ class DBConnection(object):
logger.log(u"Fatal error executing query: " + ex(e), logger.ERROR)
raise
time.sleep(0.05)
return sqlResult
def select(self, query, args=None):

28
sickbeard/helpers.py
View File

@ -730,7 +730,6 @@ def sanitizeSceneName(name, ezrss=False):
"""
if name:
if not ezrss:
bad_chars = u",:()'!?\u2019"
# ezrss leaves : and ! in their show names as far as I can tell
@ -1066,37 +1065,24 @@ def _check_against_names(nameInQuestion, show, season=-1):
def get_show_by_name(name, useIndexer=False):
name = full_sanitizeSceneName(name)
try:
# check cache for show
showObj = sickbeard.name_cache.retrieveShowFromCache(name)
if showObj:
return showObj
if not showObj and sickbeard.showList:
db_indexerid = searchDBForShow(name)
if db_indexerid:
showObj = findCertainShow(sickbeard.showList, db_indexerid)
if not showObj:
scene_indexerid, scene_season = sickbeard.scene_exceptions.get_scene_exception_by_name(name)
if scene_indexerid:
showObj = findCertainShow(sickbeard.showList, scene_indexerid)
if useIndexer and not showObj:
(sn, idx, id) = searchIndexerForShowID(name, ui=classes.ShowListUI)
if id:
showObj = findCertainShow(sickbeard.showList, int(id))
if useIndexer and sickbeard.showList and not showObj:
(sn, idx, id) = searchIndexerForShowID(full_sanitizeSceneName(name), ui=classes.ShowListUI)
if id:
showObj = findCertainShow(sickbeard.showList, int(id))
# add show to cache
if showObj:
sickbeard.name_cache.addNameToCache(name, showObj.indexerid)
return showObj
except:
showObj = None
return showObj
pass
def is_hidden_folder(folder):
"""

11
sickbeard/maintenance.py
View File

@ -23,7 +23,7 @@ import sickbeard
from sickbeard import scene_exceptions
from sickbeard import failed_history
from sickbeard import network_timezones
from sickbeard import name_cache
class Maintenance():
def __init__(self):
@ -31,15 +31,18 @@ class Maintenance():
self.amActive = False
def __del__(self):
pass
def run(self, force=False):
self.amActive = True
# clear internal name cache
name_cache.clearCache()
# get and update scene exceptions lists
scene_exceptions.retrieve_exceptions()
# build internal name cache for searches and parsing
name_cache.buildNameCache()
# refresh network timezones
network_timezones.update_network_dict()

89
sickbeard/name_cache.py
View File

@ -22,6 +22,8 @@ from sickbeard import db
from sickbeard.helpers import sanitizeSceneName
from sickbeard import logger
nameCache = None
def addNameToCache(name, indexer_id=0):
"""
Adds the show & tvdb id to the scene_names table in cache.db.
@ -29,11 +31,15 @@ def addNameToCache(name, indexer_id=0):
name: The show name to cache
indexer_id: the TVDB and TVRAGE id that this show should be cached with (can be None/0 for unknown)
"""
global nameCache
cacheDB = db.DBConnection('cache.db')
# standardize the name we're using to account for small differences in providers
name = sanitizeSceneName(name)
myDB = db.DBConnection('cache.db')
myDB.action("INSERT INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name])
name = sickbeard.helpers.full_sanitizeSceneName(name)
if name not in nameCache:
nameCache[name] = int(indexer_id)
cacheDB.action("INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name])
def retrieveNameFromCache(name):
@ -44,33 +50,66 @@ def retrieveNameFromCache(name):
Returns: the TVDB and TVRAGE id that resulted from the cache lookup or None if the show wasn't found in the cache
"""
global nameCache
cache_results = None
# standardize the name we're using to account for small differences in providers
name = sanitizeSceneName(name)
myDB = db.DBConnection('cache.db')
if myDB.hasTable('scene_names'):
cache_results = myDB.select("SELECT * FROM scene_names WHERE name = ?", [name])
if cache_results:
return int(cache_results[0]["indexer_id"])
name = sickbeard.helpers.full_sanitizeSceneName(name)
if name in nameCache:
return int(nameCache[name])
def retrieveShowFromCache(name):
indexerid = retrieveNameFromCache(name)
if indexerid:
return sickbeard.helpers.findCertainShow(sickbeard.showList, int(indexerid))
global nameCache
def clearCache(show=None, season=-1, indexer_id=0):
indexer_id = retrieveNameFromCache(name)
if indexer_id:
return sickbeard.helpers.findCertainShow(sickbeard.showList, int(indexer_id))
def clearCache():
"""
Deletes all "unknown" entries from the cache (names with indexer_id of 0).
"""
global nameCache
myDB = db.DBConnection('cache.db')
if show:
showNames = sickbeard.show_name_helpers.allPossibleShowNames(show, season=season)
for showName in showNames:
myDB.action("DELETE FROM scene_names WHERE name = ? and indexer_id = ?", [showName, indexer_id])
else:
myDB.action("DELETE FROM scene_names WHERE indexer_id = ?", [indexer_id])
# init name cache
if not nameCache:
nameCache = {}
cacheDB = db.DBConnection('cache.db')
cacheDB.action("DELETE FROM scene_names WHERE indexer_id = ?", [0])
toRemove = [key for key, value in nameCache.iteritems() if value == 0]
for key in toRemove:
del nameCache[key]
def saveNameCacheToDb():
cacheDB = db.DBConnection('cache.db')
for name, indexer_id in nameCache.items():
cacheDB.action("INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name])
def buildNameCache():
global nameCache
# init name cache
if not nameCache:
nameCache = {}
# clear internal name cache
clearCache()
logger.log(u"Updating internal name cache", logger.MESSAGE)
cacheDB = db.DBConnection('cache.db')
cache_results = cacheDB.select("SELECT * FROM scene_names")
for cache_result in cache_results:
name = sickbeard.helpers.full_sanitizeSceneName(cache_result["name"])
indexer_id = int(cache_result["indexer_id"])
nameCache[name] = indexer_id
for show in sickbeard.showList:
for curSeason in [-1] + sickbeard.scene_exceptions.get_scene_seasons(show.indexerid):
nameCache[sickbeard.helpers.full_sanitizeSceneName(show.name)] = show.indexerid
for name in sickbeard.scene_exceptions.get_scene_exceptions(show.indexerid, season=curSeason):
nameCache[sickbeard.helpers.full_sanitizeSceneName(name)] = show.indexerid
logger.log(u"Updated internal name cache", logger.MESSAGE)
logger.log(u"Internal name cache set to: " + str(nameCache), logger.DEBUG)

2
sickbeard/providers/btn.py
View File

@ -343,8 +343,6 @@ class BTNCache(tvcache.TVCache):
if ci is not None:
cl.append(ci)
if cl:
myDB = self._getDB()
myDB.mass_action(cl)

48
sickbeard/scene_exceptions.py
View File

@ -27,7 +27,8 @@ from sickbeard import name_cache
from sickbeard import logger
from sickbeard import db
scene_lock = threading.Lock()
exceptionsCache = None
exceptionsSeasonCache = None
def shouldRefresh(list):
MAX_REFRESH_AGE_SECS = 86400 # 1 day
@ -51,14 +52,21 @@ def get_scene_exceptions(indexer_id, season=-1):
"""
Given a indexer_id, return a list of all the scene exceptions.
"""
global exceptionsCache
exceptionsList = []
myDB = db.DBConnection('cache.db')
exceptions = myDB.select("SELECT show_name FROM scene_exceptions WHERE indexer_id = ? and season = ?",
[indexer_id, season])
if exceptions:
exceptionsList = list(set([cur_exception["show_name"] for cur_exception in exceptions]))
if indexer_id not in exceptionsCache or season not in exceptionsCache[indexer_id]:
myDB = db.DBConnection('cache.db')
exceptions = myDB.select("SELECT show_name FROM scene_exceptions WHERE indexer_id = ? and season = ?",
[indexer_id, season])
if exceptions:
exceptionsList = list(set([cur_exception["show_name"] for cur_exception in exceptions]))
if not indexer_id in exceptionsCache:
exceptionsCache[indexer_id] = {}
exceptionsCache[indexer_id][season] = exceptionsList
else:
exceptionsList = exceptionsCache[indexer_id][season]
if season == 1: # if we where looking for season 1 we can add generic names
exceptionsList += get_scene_exceptions(indexer_id, season=-1)
@ -84,14 +92,24 @@ def get_scene_seasons(indexer_id):
"""
return a list of season numbers that have scene exceptions
"""
global exceptionsSeasonCache
exceptionsSeasonList = []
myDB = db.DBConnection('cache.db')
sqlResults = myDB.select("SELECT DISTINCT(season) as season FROM scene_exceptions WHERE indexer_id = ?",
[indexer_id])
if indexer_id not in exceptionsSeasonCache:
myDB = db.DBConnection('cache.db')
sqlResults = myDB.select("SELECT DISTINCT(season) as season FROM scene_exceptions WHERE indexer_id = ?",
[indexer_id])
if sqlResults:
exceptionsSeasonList = list(set([int(x["season"]) for x in sqlResults]))
if sqlResults:
return [int(x["season"]) for x in sqlResults]
if not indexer_id in exceptionsSeasonCache:
exceptionsSeasonCache[indexer_id] = {}
exceptionsSeasonCache[indexer_id] = exceptionsSeasonList
else:
exceptionsSeasonList = exceptionsSeasonCache[indexer_id]
return exceptionsSeasonList
def get_scene_exception_by_name(show_name):
return get_scene_exception_by_name_multiple(show_name)[0]
@ -136,7 +154,10 @@ def retrieve_exceptions():
Looks up the exceptions on github, parses them into a dict, and inserts them into the
scene_exceptions table in cache.db. Also clears the scene name cache.
"""
global exceptionsCache, exceptionsSeasonCache
exceptionsCache = {}
exceptionsSeasonCache = {}
exception_dict = {}
# exceptions are stored on github pages
@ -209,7 +230,6 @@ def retrieve_exceptions():
# since this could invalidate the results of the cache we clear it out after updating
if changed_exceptions:
logger.log(u"Updated scene exceptions")
name_cache.clearCache()
else:
logger.log(u"No scene exceptions update needed")
@ -230,8 +250,6 @@ def update_scene_exceptions(indexer_id, scene_exceptions):
myDB.action("INSERT INTO scene_exceptions (indexer_id, show_name, season, custom) VALUES (?,?,?,?)",
[indexer_id, cur_exception, cur_season, 1])
name_cache.clearCache()
def _retrieve_anidb_mainnames():
anidb_mainNames = {}

6
sickbeard/show_queue.py
View File

@ -279,9 +279,6 @@ class QueueItemAdd(ShowQueueItem):
return
try:
# clear the name cache
name_cache.clearCache()
newShow = TVShow(self.indexer, self.indexer_id, self.lang)
newShow.loadFromIndexer()
@ -361,6 +358,9 @@ class QueueItemAdd(ShowQueueItem):
# before we parse local files lets update exceptions
sickbeard.scene_exceptions.retrieve_exceptions()
# update internal name cache
name_cache.buildNameCache()
try:
self.show.loadEpisodesFromDir()
except Exception, e:

2
sickbeard/tvcache.py
View File

@ -128,8 +128,6 @@ class TVCache():
if ci is not None:
cl.append(ci)
time.sleep(.2)
if cl:
myDB = self._getDB()
myDB.mass_action(cl)