From 5ac389560025e06fa37acef4aa60823404e1c746 Mon Sep 17 00:00:00 2001 From: adam Date: Sat, 20 Sep 2014 20:03:48 +0800 Subject: [PATCH] Fixes daily search and speed improvements --- sickbeard/dailysearcher.py | 10 ++----- sickbeard/db.py | 6 ++-- sickbeard/providers/generic.py | 9 ++---- sickbeard/search.py | 55 ++++++++++++++++++++++++++++++---- sickbeard/search_queue.py | 17 ++++++----- sickbeard/tv.py | 2 ++ sickbeard/tvcache.py | 30 +++++++++++-------- 7 files changed, 87 insertions(+), 42 deletions(-) diff --git a/sickbeard/dailysearcher.py b/sickbeard/dailysearcher.py index 61e4dafc..216e017d 100644 --- a/sickbeard/dailysearcher.py +++ b/sickbeard/dailysearcher.py @@ -50,13 +50,11 @@ class DailySearcher(): sql_l = [] show = None - wantedEp = {} for sqlEp in sqlResults: try: if not show or int(sqlEp["showid"]) != show.indexerid: show = helpers.findCertainShow(sickbeard.showList, int(sqlEp["showid"])) - wantedEp[show] = [] except exceptions.MultipleShowObjectsException: logger.log(u"ERROR: expected to find a single show matching " + sqlEp["showid"]) @@ -68,7 +66,6 @@ class DailySearcher(): ep.status = common.SKIPPED else: ep.status = common.WANTED - wantedEp[show].append(ep) sql_l.append(ep.get_sql()) else: @@ -78,9 +75,8 @@ class DailySearcher(): myDB = db.DBConnection() myDB.mass_action(sql_l) - for show, episode in wantedEp.items(): - # queue episode for daily search - dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem(show, episode) - sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item) + # queue episode for daily search + dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem() + sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item) self.amActive = False \ No newline at end of file diff --git a/sickbeard/db.py b/sickbeard/db.py index fa48b6a1..b629a7d1 100644 --- a/sickbeard/db.py +++ b/sickbeard/db.py @@ -118,7 +118,7 @@ class DBConnection(object): else: return 0 - def mass_action(self, querylist, logTransaction=False): + def mass_action(self, querylist, logTransaction=False, fetchall=False): with db_lock: # remove None types @@ -136,11 +136,11 @@ class DBConnection(object): if len(qu) == 1: if logTransaction: logger.log(qu[0], logger.DEBUG) - sqlResult.append(self.execute(qu[0])) + sqlResult.append(self.execute(qu[0], fetchall=fetchall)) elif len(qu) > 1: if logTransaction: logger.log(qu[0] + " with args " + str(qu[1]), logger.DEBUG) - sqlResult.append(self.execute(qu[0], qu[1])) + sqlResult.append(self.execute(qu[0], qu[1], fetchall=fetchall)) logger.log(u"Transaction with " + str(len(querylist)) + u" queries executed", logger.DEBUG) diff --git a/sickbeard/providers/generic.py b/sickbeard/providers/generic.py index ec13997f..7c4b637f 100644 --- a/sickbeard/providers/generic.py +++ b/sickbeard/providers/generic.py @@ -203,13 +203,8 @@ class GenericProvider: return True - def searchRSS(self, episode): - results = {} - - for ep in episode: - results.update(self.cache.findNeededEpisodes(ep)) - - return results + def searchRSS(self, episodes): + return self.cache.findNeededEpisodes(episodes) def getQuality(self, item, anime=False): """ diff --git a/sickbeard/search.py b/sickbeard/search.py index 86ae61e4..bc2c8bf8 100644 --- a/sickbeard/search.py +++ b/sickbeard/search.py @@ -41,6 +41,7 @@ from sickbeard import failed_history from sickbeard.exceptions import ex from sickbeard.providers.generic import GenericProvider from sickbeard.blackandwhitelist import BlackAndWhiteList +from sickbeard import common def _downloadResult(result): """ @@ -319,16 +320,60 @@ def isFirstBestMatch(result): return False -def searchForNeededEpisodes(show, episode): +def wantedEpisodes(show, fromDate): + anyQualities, bestQualities = common.Quality.splitQuality(show.quality) # @UnusedVariable + allQualities = list(set(anyQualities + bestQualities)) + + logger.log(u"Seeing if we need anything from " + show.name) + myDB = db.DBConnection() + + if show.air_by_date: + sqlResults = myDB.select( + "SELECT ep.status, ep.season, ep.episode FROM tv_episodes ep, tv_shows show WHERE season != 0 AND ep.showid = show.indexer_id AND show.paused = 0 AND ep.airdate > ? AND ep.showid = ? AND show.air_by_date = 1", + [fromDate.toordinal(), show.indexerid]) + else: + sqlResults = myDB.select( + "SELECT status, season, episode FROM tv_episodes WHERE showid = ? AND season > 0 and airdate > ?", + [show.indexerid, fromDate.toordinal()]) + + # check through the list of statuses to see if we want any + wanted = [] + for result in sqlResults: + curCompositeStatus = int(result["status"]) + curStatus, curQuality = common.Quality.splitCompositeStatus(curCompositeStatus) + + if bestQualities: + highestBestQuality = max(allQualities) + else: + highestBestQuality = 0 + + # if we need a better one then say yes + if (curStatus in (common.DOWNLOADED, common.SNATCHED, common.SNATCHED_PROPER, + common.SNATCHED_BEST) and curQuality < highestBestQuality) or curStatus == common.WANTED: + + epObj = show.getEpisode(int(result["season"]), int(result["episode"])) + epObj.wantedQuality = [i for i in allQualities if (i > curQuality and i != common.Quality.UNKNOWN)] + wanted.append(epObj) + + return wanted + +def searchForNeededEpisodes(): foundResults = {} didSearch = False - # build name cache for show - sickbeard.name_cache.buildNameCache(show) - origThreadName = threading.currentThread().name + show_list = sickbeard.showList + fromDate = datetime.date.fromordinal(1) + episodes = [] + + for curShow in show_list: + if curShow.paused: + continue + + episodes.extend(wantedEpisodes(curShow, fromDate)) + providers = [x for x in sickbeard.providers.sortedProviderList() if x.isActive() and x.enable_daily] for curProvider in providers: @@ -336,7 +381,7 @@ def searchForNeededEpisodes(show, episode): try: curProvider.cache.updateCache() - curFoundResults = curProvider.searchRSS(episode) + curFoundResults = curProvider.searchRSS(episodes) except exceptions.AuthException, e: logger.log(u"Authentication error: " + ex(e), logger.ERROR) continue diff --git a/sickbeard/search_queue.py b/sickbeard/search_queue.py index 34bdcf8e..62aa6589 100644 --- a/sickbeard/search_queue.py +++ b/sickbeard/search_queue.py @@ -116,8 +116,11 @@ class SearchQueue(generic_queue.GenericQueue): def add_item(self, item): - if isinstance(item, (DailySearchQueueItem, BacklogQueueItem)) and not self.is_in_queue(item.show, item.segment): - # daily and backlog searches + if isinstance(item, DailySearchQueueItem): + # daily searches + generic_queue.GenericQueue.add_item(self, item) + elif isinstance(item, BacklogQueueItem) and not self.is_in_queue(item.show, item.segment): + # backlog searches generic_queue.GenericQueue.add_item(self, item) elif isinstance(item, (ManualSearchQueueItem, FailedQueueItem)) and not self.is_ep_in_queue(item.segment): # manual and failed searches @@ -126,20 +129,18 @@ class SearchQueue(generic_queue.GenericQueue): logger.log(u"Not adding item, it's already in the queue", logger.DEBUG) class DailySearchQueueItem(generic_queue.QueueItem): - def __init__(self, show, segment): + def __init__(self): generic_queue.QueueItem.__init__(self, 'Daily Search', DAILY_SEARCH) - self.show = show - self.segment = segment def run(self): generic_queue.QueueItem.run(self) try: - logger.log("Beginning daily search for: [" + self.show.name + "]") - foundResults = search.searchForNeededEpisodes(self.show, self.segment) + logger.log("Beginning daily search for new episodes") + foundResults = search.searchForNeededEpisodes() if not len(foundResults): - logger.log(u"No needed episodes found during daily search for: [" + self.show.name + "]") + logger.log(u"No needed episodes found") else: for result in foundResults: # just use the first result for now diff --git a/sickbeard/tv.py b/sickbeard/tv.py index 862838f3..e9735b23 100644 --- a/sickbeard/tv.py +++ b/sickbeard/tv.py @@ -1305,6 +1305,8 @@ class TVEpisode(object): self.checkForMetaFiles() + self.wantedQuality = [] + name = property(lambda self: self._name, dirty_setter("_name")) season = property(lambda self: self._season, dirty_setter("_season")) episode = property(lambda self: self._episode, dirty_setter("_episode")) diff --git a/sickbeard/tvcache.py b/sickbeard/tvcache.py index f257a10d..0d790d4f 100644 --- a/sickbeard/tvcache.py +++ b/sickbeard/tvcache.py @@ -32,6 +32,7 @@ from sickbeard.exceptions import AuthException from name_parser.parser import NameParser, InvalidNameException, InvalidShowException from sickbeard.rssfeeds import RSSFeeds from sickbeard import clients +import itertools class CacheDBConnection(db.DBConnection): def __init__(self, providerName): @@ -279,7 +280,10 @@ class TVCache(): def searchCache(self, episode, manualSearch=False): neededEps = self.findNeededEpisodes(episode, manualSearch) - return neededEps[episode] + if len(neededEps) > 0: + return neededEps[episode] + else: + return [] def listPropers(self, date=None, delimiter="."): myDB = self._getDB() @@ -291,19 +295,24 @@ class TVCache(): return filter(lambda x: x['indexerid'] != 0, myDB.select(sql)) - def findNeededEpisodes(self, episode=None, manualSearch=False): + def findNeededEpisodes(self, episode, manualSearch=False): neededEps = {} - - if episode: - neededEps[episode] = [] + cl = [] myDB = self._getDB() - if not episode: - sqlResults = myDB.select("SELECT * FROM [" + self.providerID + "]") - else: + if type(episode) != list: sqlResults = myDB.select( "SELECT * FROM [" + self.providerID + "] WHERE indexerid = ? AND season = ? AND episodes LIKE ?", [episode.show.indexerid, episode.season, "%|" + str(episode.episode) + "|%"]) + else: + for epObj in episode: + cl.append([ + "SELECT * FROM [" + self.providerID + "] WHERE indexerid = ? AND season = ? AND episodes LIKE ? " + "AND quality IN (" + ",".join([str(x) for x in epObj.wantedQuality]) + ")", + [epObj.show.indexerid, epObj.season, "%|" + str(epObj.episode) + "|%"]]) + + sqlResults = myDB.mass_action(cl, fetchall=True) + sqlResults = list(itertools.chain(*sqlResults)) # for each cache entry for curResult in sqlResults: @@ -341,10 +350,7 @@ class TVCache(): Quality.qualityStrings[curQuality], logger.DEBUG) continue - if episode: - epObj = episode - else: - epObj = showObj.getEpisode(curSeason, curEp) + epObj = showObj.getEpisode(curSeason, curEp) # build a result object title = curResult["name"]