Re-wrote daily searcher to search for unaired and wanted episodes going back as far as 1 week, also moved it so it queue's its items now.

Cleaned up description of new per-provider season search options.

gui/slick/interfaces/default/config_providers.tmpl

@@ -339,8 +339,8 @@ var show_nzb_providers = #if $sickbeard.USE_NZBS then "true" else "false"#;
                         <div class="field-pair">
                             <input type="checkbox" name="${curTorrentProvider.getID()}_search_fallback" id="${curTorrentProvider.getID()}_search_fallback" #if $curTorrentProvider.search_fallback then "checked=\"checked\"" else ""#/>
                             <label class="clearfix" for="${curTorrentProvider.getID()}_search_fallback">
-                                <span class="component-title">Search Fallback</span>
+                                <span class="component-title">Season Search Fallback</span>
-                                <span class="component-desc">This will restart the search using the alternative method from Search Mode if first attempt fails to return any results to snatch.</span>
+                                <span class="component-desc">When searching for a complete season depending on search mode you may return no results, this helps by restarting the search using the opposite search mode</span>
                             </label>
                         </div>
                         #end if
@@ -348,8 +348,8 @@ var show_nzb_providers = #if $sickbeard.USE_NZBS then "true" else "false"#;
                         #if $hasattr($curTorrentProvider, 'search_mode'):
                         <div class="field-pair">
                             <label class="nocheck clearfix">
-                                <span class="component-title">Search Mode</span>
+                                <span class="component-title">Season Search Mode</span>
-                                <span class="component-desc">
+                                <span class="component-desc">When searching for complete seasons you can choose to have it look for season packs ONLY or choose to have it build a complete season from just single episodes</span>
                                     <input type="radio" name="${curTorrentProvider.getID()}_search_mode" id="${curTorrentProvider.getID()}_search_mode_sponly" value="sponly" class="radio" #if $curTorrentProvider.search_mode=="sponly" then "checked=\"checked\"" else ""# />Season Packs ONLY!<br />
                                 </span>
                             </label>

sickbeard/dailysearcher.py

@@ -32,7 +32,6 @@ from sickbeard.exceptions import ex
 from sickbeard.search import pickBestResult, snatchEpisode
 from sickbeard import generic_queue
 
-
 class DailySearcher():
     def __init__(self):
         self.lock = threading.Lock()
@@ -40,33 +39,18 @@ class DailySearcher():
         self.amActive = False
 
     def run(self):
-        self.amActive = True
-        self._changeUnairedEpisodes()
-
         # remove names from cache that link back to active shows that we watch
         sickbeard.name_cache.syncNameCache()
 
-        logger.log(u"Starting Daily Searcher ...")
+        logger.log(u"Checking to see if any shows have wanted episodes available for the last week ...")
-        foundResults = self.searchForNeededEpisodes()
 
-        if not len(foundResults):
+        curDate = datetime.date.today() - datetime.timedelta(weeks=1)
-            logger.log(u"No needed episodes found on the RSS feeds")
-        else:
-            for curResult in foundResults:
-                snatchEpisode(curResult)
-
-        self.amActive = False
-
-    def _changeUnairedEpisodes(self):
-
-        logger.log(u"Setting todays new releases to status WANTED")
-
-        curDate = datetime.date.today().toordinal()
 
         myDB = db.DBConnection()
-        sqlResults = myDB.select("SELECT * FROM tv_episodes WHERE status = ? AND airdate < ?",
+        sqlResults = myDB.select("SELECT * FROM tv_episodes WHERE (status = ? OR status = ?) AND airdate < ?",
-                                 [common.UNAIRED, curDate])
+                                 [common.UNAIRED, common.WANTED, curDate.toordinal()])
 
+        todaysEps = {}
         for sqlEp in sqlResults:
 
             try:
@@ -85,72 +69,21 @@ class DailySearcher():
                 if ep.show.paused:
                     ep.status = common.SKIPPED
                 else:
-                    ep.status = common.WANTED
+                    if ep.status == common.UNAIRED:
+                        ep.status = common.WANTED
+
                 ep.saveToDB()
 
-    def searchForNeededEpisodes(self):
+                if ep.status == common.WANTED:
+                    if show not in todaysEps:
+                        todaysEps[show] = [ep]
+                    else:
+                        todaysEps[show].append(ep)
 
-        foundResults = {}
+        if len(todaysEps):
-
+            for show in todaysEps:
-        didSearch = False
+                segment = todaysEps[show]
-
+                dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem(show, segment)
-        # ask all providers for any episodes it finds
+                sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item)  #@UndefinedVariable
-        threadName = threading.currentThread().name
+        else:
-        providers = [x for x in sickbeard.providers.sortedProviderList() if x.isActive()]
+            logger.log(u"Could not find any wanted show episodes going back 1 week at this current time ...")
-        for curProviderCount, curProvider in enumerate(providers):
-            threading.currentThread().name = threadName + ":[" + curProvider.name + "]"
-
-            try:
-                logger.log(u"Updating RSS cache ...")
-                curProvider.cache.updateCache()
-
-                logger.log(u"Searching RSS cache ...")
-                curFoundResults = curProvider.searchRSS()
-            except exceptions.AuthException, e:
-                logger.log(u"Authentication error: " + ex(e), logger.ERROR)
-                if curProviderCount != len(providers):
-                    continue
-                break
-            except Exception, e:
-                logger.log(u"Error while searching " + curProvider.name + ", skipping: " + ex(e), logger.ERROR)
-                logger.log(traceback.format_exc(), logger.DEBUG)
-                if curProviderCount != len(providers):
-                    continue
-                break
-
-            didSearch = True
-
-            # pick a single result for each episode, respecting existing results
-            for curEp in curFoundResults:
-
-                if curEp.show.paused:
-                    logger.log(
-                        u"Show " + curEp.show.name + " is paused, ignoring all RSS items for " + curEp.prettyName(),
-                        logger.DEBUG)
-                    continue
-
-                # find the best result for the current episode
-                bestResult = None
-                for curResult in curFoundResults[curEp]:
-                    if not bestResult or bestResult.quality < curResult.quality:
-                        bestResult = curResult
-
-                bestResult = pickBestResult(curFoundResults[curEp], curEp.show)
-
-                # if all results were rejected move on to the next episode
-                if not bestResult:
-                    logger.log(u"All found results for " + curEp.prettyName() + " were rejected.", logger.DEBUG)
-                    continue
-
-                # if it's already in the list (from another provider) and the newly found quality is no better then skip it
-                if curEp in foundResults and bestResult.quality <= foundResults[curEp].quality:
-                    continue
-
-                foundResults[curEp] = bestResult
-
-        if not didSearch:
-            logger.log(
-                u"No NZB/Torrent providers found or enabled in the sickbeard config. Please check your settings.",
-                logger.ERROR)
-
-        return foundResults.values() if len(foundResults) else {}

sickbeard/providers/generic.py

@@ -184,8 +184,8 @@ class GenericProvider:
 
         return True
 
-    def searchRSS(self):
+    def searchRSS(self, episodes):
-        return self.cache.findNeededEpisodes()
+        return self.cache.findNeededEpisodes(episodes)
 
     def getQuality(self, item):
         """
@@ -252,7 +252,7 @@ class GenericProvider:
                         u"Incomplete Indexer <-> Scene mapping detected for " + epObj.prettyName() + ", skipping search!")
                     continue
 
-            cacheResult = self.cache.searchCache(epObj, manualSearch)
+            cacheResult = self.cache.searchCache([epObj], manualSearch)
             if len(cacheResult):
                 results.update({epObj.episode:cacheResult[epObj]})
                 continue

sickbeard/search.py

@@ -316,9 +316,73 @@ def filterSearchResults(show, results):
     return foundResults
 
 
-def searchProviders(queueItem, show, season, episodes, manualSearch=False):
+def searchForNeededEpisodes(queueItem):
-    threadName = threading.currentThread().name
+    foundResults = {}
 
+    didSearch = False
+
+    # ask all providers for any episodes it finds
+    providers = [x for x in sickbeard.providers.sortedProviderList() if x.isActive()]
+    for curProviderCount, curProvider in enumerate(providers):
+        threading.currentThread().name = queueItem.thread_name + "[" + curProvider.name + "]"
+
+        try:
+            logger.log(u"Updating RSS cache ...")
+            curProvider.cache.updateCache()
+
+            logger.log(u"Searching RSS cache ...")
+            curFoundResults = curProvider.searchRSS(queueItem.segment)
+        except exceptions.AuthException, e:
+            logger.log(u"Authentication error: " + ex(e), logger.ERROR)
+            if curProviderCount != len(providers):
+                continue
+            break
+        except Exception, e:
+            logger.log(u"Error while searching " + curProvider.name + ", skipping: " + ex(e), logger.ERROR)
+            logger.log(traceback.format_exc(), logger.DEBUG)
+            if curProviderCount != len(providers):
+                continue
+            break
+
+        didSearch = True
+
+        # pick a single result for each episode, respecting existing results
+        for curEp in curFoundResults:
+
+            if curEp.show.paused:
+                logger.log(
+                    u"Show " + curEp.show.name + " is paused, ignoring all RSS items for " + curEp.prettyName(),
+                    logger.DEBUG)
+                continue
+
+            # find the best result for the current episode
+            bestResult = None
+            for curResult in curFoundResults[curEp]:
+                if not bestResult or bestResult.quality < curResult.quality:
+                    bestResult = curResult
+
+            bestResult = pickBestResult(curFoundResults[curEp], curEp.show)
+
+            # if all results were rejected move on to the next episode
+            if not bestResult:
+                logger.log(u"All found results for " + curEp.prettyName() + " were rejected.", logger.DEBUG)
+                continue
+
+            # if it's already in the list (from another provider) and the newly found quality is no better then skip it
+            if curEp in foundResults and bestResult.quality <= foundResults[curEp].quality:
+                continue
+
+            foundResults[curEp] = bestResult
+
+    if not didSearch:
+        logger.log(
+            u"No NZB/Torrent providers found or enabled in the sickbeard config. Please check your settings.",
+            logger.ERROR)
+
+    return foundResults.values() if len(foundResults) else {}
+
+
+def searchProviders(queueItem, show, season, episodes, manualSearch=False):
     # check if we want to search for season packs instead of just season/episode
     seasonSearch = False
     seasonEps = show.getAllEpisodes(season)
@@ -334,7 +398,7 @@ def searchProviders(queueItem, show, season, episodes, manualSearch=False):
 
     foundResults = {}
     for providerNum, provider in enumerate(providers):
-        threading.currentThread().name = threadName + ":[" + provider.name + "]"
+        threading.currentThread().name = queueItem.thread_name + ":[" + provider.name + "]"
         foundResults.setdefault(provider.name, {})
         searchCount = 0
 

sickbeard/search_queue.py

@@ -27,10 +27,13 @@ from sickbeard import db, logger, common, exceptions, helpers
 from sickbeard import generic_queue, scheduler
 from sickbeard import search, failed_history, history
 from sickbeard import ui
+from sickbeard.exceptions import ex
+from sickbeard.search import pickBestResult
 
 search_queue_lock = threading.Lock()
 
 BACKLOG_SEARCH = 10
+DAILY_SEARCH = 20
 FAILED_SEARCH = 30
 MANUAL_SEARCH = 30
 
@@ -66,7 +69,9 @@ class SearchQueue(generic_queue.GenericQueue):
 
     def add_item(self, item):
 
-        if isinstance(item, BacklogQueueItem) and not self.is_in_queue(item.show, item.segment):
+        if isinstance(item, DailySearchQueueItem) and not self.is_in_queue(item.show, item.segment):
+            generic_queue.GenericQueue.add_item(self, item)
+        elif isinstance(item, BacklogQueueItem) and not self.is_in_queue(item.show, item.segment):
             generic_queue.GenericQueue.add_item(self, item)
         elif isinstance(item, ManualSearchQueueItem) and not self.is_in_queue(item.show, item.segment):
             generic_queue.GenericQueue.add_item(self, item)
@@ -80,8 +85,33 @@ class SearchQueue(generic_queue.GenericQueue):
             # just use the first result for now
             logger.log(u"Downloading " + result.name + " from " + result.provider.name)
             item.success =  search.snatchEpisode(result)
+
             time.sleep(2)
-            generic_queue.QueueItem.finish(item)
+
+        return item
+
+class DailySearchQueueItem(generic_queue.QueueItem):
+    def __init__(self, show, segment):
+        generic_queue.QueueItem.__init__(self, 'Daily Search', DAILY_SEARCH)
+        self.priority = generic_queue.QueuePriorities.HIGH
+        self.thread_name = 'DAILYSEARCH-' + str(show.indexerid) + '-'
+        self.show = show
+        self.segment = segment
+        self.results = []
+
+    def execute(self):
+        generic_queue.QueueItem.execute(self)
+
+        logger.log("Beginning daily search for [" + self.show.name + "]")
+        foundResults = search.searchForNeededEpisodes(self)
+
+        if not len(foundResults):
+            logger.log(u"No needed episodes found during daily search for [" + self.show.name + "]")
+        else:
+            for curResult in foundResults:
+                SearchQueue().snatch_item(curResult)
+
+        generic_queue.QueueItem.finish(self)
 
 class ManualSearchQueueItem(generic_queue.QueueItem):
     def __init__(self, show, segment):
@@ -96,12 +126,14 @@ class ManualSearchQueueItem(generic_queue.QueueItem):
     def execute(self):
         generic_queue.QueueItem.execute(self)
 
+        queueItem = self
+
         try:
             logger.log("Beginning manual search for [" + self.segment.prettyName() + "]")
-            searchResult = search.searchProviders(self, self.show, self.segment.season, [self.segment], True)
+            searchResult = search.searchProviders(queueItem, self.show, self.segment.season, [self.segment], True)
 
             if searchResult:
-                SearchQueue().snatch_item(searchResult)
+                queueItem = SearchQueue().snatch_item(searchResult)
             else:
                 ui.notifications.message('No downloads were found',
                                          "Couldn't find a download for <i>%s</i>" % self.segment.prettyName())
@@ -111,7 +143,7 @@ class ManualSearchQueueItem(generic_queue.QueueItem):
         except Exception:
             logger.log(traceback.format_exc(), logger.DEBUG)
 
-        self.finish()
+        generic_queue.QueueItem.finish(queueItem)
 
 class BacklogQueueItem(generic_queue.QueueItem):
     def __init__(self, show, segment):
@@ -138,7 +170,7 @@ class BacklogQueueItem(generic_queue.QueueItem):
                 if searchResult:
                     SearchQueue().snatch_item(searchResult)
                 else:
-                    logger.log(u"No needed episodes found during backlog search")
+                    logger.log(u"No needed episodes found during backlog search for [" + self.show.name + "]")
 
             except Exception:
                 logger.log(traceback.format_exc(), logger.DEBUG)

sickbeard/tvcache.py

@@ -85,7 +85,9 @@ class TVCache():
 
         myDB = self._getDB()
 
-        myDB.action("DELETE FROM [" + self.providerID + "] WHERE 1")
+        curDate = datetime.date.today() - datetime.timedelta(weeks=1)
+
+        myDB.action("DELETE FROM [" + self.providerID + "] WHERE time < ?", [curDate.toordinal()])
 
     def _getRSSData(self):
 
@@ -345,8 +347,8 @@ class TVCache():
                 [name, season, episodeText, indexerid, url, curTimestamp, quality]]
 
 
-    def searchCache(self, episode, manualSearch=False):
+    def searchCache(self, episodes, manualSearch=False):
-        neededEps = self.findNeededEpisodes(episode, manualSearch)
+        neededEps = self.findNeededEpisodes(episodes, manualSearch)
         return neededEps
 
     def listPropers(self, date=None, delimiter="."):
@@ -360,74 +362,72 @@ class TVCache():
         return filter(lambda x: x['indexerid'] != 0, myDB.select(sql))
 
 
-    def findNeededEpisodes(self, epObj=None, manualSearch=False):
+    def findNeededEpisodes(self, episodes, manualSearch=False):
         neededEps = {}
 
         cacheDB = self._getDB()
 
-        if not epObj:
+        for epObj in episodes:
-            sqlResults = cacheDB.select("SELECT * FROM [" + self.providerID + "]")
-        else:
             sqlResults = cacheDB.select(
                 "SELECT * FROM [" + self.providerID + "] WHERE indexerid = ? AND season = ? AND episodes LIKE ?",
                 [epObj.show.indexerid, epObj.season, "%|" + str(epObj.episode) + "|%"])
 
-        # for each cache entry
+            # for each cache entry
-        for curResult in sqlResults:
+            for curResult in sqlResults:
 
-            time.sleep(cpu_presets[sickbeard.CPU_PRESET])
+                time.sleep(cpu_presets[sickbeard.CPU_PRESET])
 
-            # skip non-tv crap (but allow them for Newzbin cause we assume it's filtered well)
+                # skip non-tv crap (but allow them for Newzbin cause we assume it's filtered well)
-            if self.providerID != 'newzbin' and not show_name_helpers.filterBadReleases(curResult["name"]):
+                if self.providerID != 'newzbin' and not show_name_helpers.filterBadReleases(curResult["name"]):
-                continue
+                    continue
 
-            # get the show object, or if it's not one of our shows then ignore it
+                # get the show object, or if it's not one of our shows then ignore it
-            try:
+                try:
-                showObj = helpers.findCertainShow(sickbeard.showList, int(curResult["indexerid"]))
+                    showObj = helpers.findCertainShow(sickbeard.showList, int(curResult["indexerid"]))
-            except MultipleShowObjectsException:
+                except MultipleShowObjectsException:
-                showObj = None
+                    showObj = None
 
-            if not showObj:
+                if not showObj:
-                continue
+                    continue
 
-            # get season and ep data (ignoring multi-eps for now)
+                # get season and ep data (ignoring multi-eps for now)
-            curSeason = int(curResult["season"])
+                curSeason = int(curResult["season"])
-            if curSeason == -1:
+                if curSeason == -1:
-                continue
+                    continue
-            curEp = curResult["episodes"].split("|")[1]
+                curEp = curResult["episodes"].split("|")[1]
-            if not curEp:
+                if not curEp:
-                continue
+                    continue
-            curEp = int(curEp)
+                curEp = int(curEp)
-            curQuality = int(curResult["quality"])
+                curQuality = int(curResult["quality"])
 
-            # if the show says we want that episode then add it to the list
+                # if the show says we want that episode then add it to the list
-            if not showObj.wantEpisode(curSeason, curEp, curQuality, manualSearch):
+                if not showObj.wantEpisode(curSeason, curEp, curQuality, manualSearch):
-                logger.log(u"Skipping " + curResult["name"] + " because we don't want an episode that's " +
+                    logger.log(u"Skipping " + curResult["name"] + " because we don't want an episode that's " +
-                           Quality.qualityStrings[curQuality], logger.DEBUG)
+                               Quality.qualityStrings[curQuality], logger.DEBUG)
-            else:
-
-                if not epObj:
-                    epObj = showObj.getEpisode(curSeason, curEp)
-
-                # build a result object
-                title = curResult["name"]
-                url = curResult["url"]
-
-                logger.log(u"Found result " + title + " at " + url)
-
-                result = self.provider.getResult([epObj])
-                result.url = url
-                result.name = title
-                result.quality = curQuality
-                result.content = self.provider.getURL(url) \
-                    if self.provider.providerType == sickbeard.providers.generic.GenericProvider.TORRENT \
-                    and not url.startswith('magnet') else None
-
-                # add it to the list
-                if epObj not in neededEps:
-                    neededEps[epObj] = [result]
                 else:
-                    neededEps[epObj].append(result)
+
+                    if not epObj:
+                        epObj = showObj.getEpisode(curSeason, curEp)
+
+                    # build a result object
+                    title = curResult["name"]
+                    url = curResult["url"]
+
+                    logger.log(u"Found result " + title + " at " + url)
+
+                    result = self.provider.getResult([epObj])
+                    result.url = url
+                    result.name = title
+                    result.quality = curQuality
+                    result.content = self.provider.getURL(url) \
+                        if self.provider.providerType == sickbeard.providers.generic.GenericProvider.TORRENT \
+                        and not url.startswith('magnet') else None
+
+                    # add it to the list
+                    if epObj not in neededEps:
+                        neededEps[epObj] = [result]
+                    else:
+                        neededEps[epObj].append(result)
 
         # datetime stamp this search so cache gets cleared
         self.setLastSearch()