Complete re-write of backlog search.

Fixed air-by-date backlog searches.

sickbeard/search.py

@@ -387,12 +387,8 @@ def searchProviders(queueItem, show, season, episodes, seasonSearch=False, manua
                 else:
                     anyWanted = True
 
-            # if we need every ep in the season check if single episode releases should be preferred over season releases (missing single episode releases will be picked individually from season release)
-            preferSingleEpisodesOverSeasonReleases = sickbeard.PREFER_EPISODE_RELEASES
-            logger.log(u"Prefer single episodes over season releases: " + str(preferSingleEpisodesOverSeasonReleases),
-                       logger.DEBUG)
             # if we need every ep in the season and there's nothing better then just download this and be done with it (unless single episodes are preferred)
-            if allWanted and bestSeasonNZB.quality == highest_quality_overall and not preferSingleEpisodesOverSeasonReleases:
+            if allWanted and bestSeasonNZB.quality == highest_quality_overall:
                 logger.log(
                     u"Every ep in this season is needed, downloading the whole " + bestSeasonNZB.provider.providerType + " " + bestSeasonNZB.name)
                 epObjs = []

sickbeard/searchBacklog.py

@@ -27,7 +27,7 @@ from sickbeard import db, scheduler
 from sickbeard import search_queue
 from sickbeard import logger
 from sickbeard import ui
-#from sickbeard.common import *
+from sickbeard import common
 
 class BacklogSearchScheduler(scheduler.Scheduler):
     def forceSearch(self):
@@ -90,46 +90,17 @@ class BacklogSearcher:
         self.amActive = True
         self.amPaused = False
 
-        #myDB = db.DBConnection()
-        #numSeasonResults = myDB.select("SELECT DISTINCT(season), showid FROM tv_episodes ep, tv_shows show WHERE season != 0 AND ep.showid = show.indexer_id AND show.paused = 0 AND ep.airdate > ?", [fromDate.toordinal()])
-
-        # get separate lists of the season/date shows
-        #season_shows = [x for x in show_list if not x.air_by_date]
-        air_by_date_shows = [x for x in show_list if x.air_by_date]
-
-        # figure out how many segments of air by date shows we're going to do
-        air_by_date_segments = []
-        for cur_id in [x.indexerid for x in air_by_date_shows]:
-            air_by_date_segments += self._get_air_by_date_segments(cur_id, fromDate)
-
-        logger.log(u"Air-by-date segments: " + str(air_by_date_segments), logger.DEBUG)
-
-        #totalSeasons = float(len(numSeasonResults) + len(air_by_date_segments))
-        #numSeasonsDone = 0.0
-
         # go through non air-by-date shows and see if they need any episodes
         for curShow in show_list:
 
             if curShow.paused:
                 continue
 
-            if curShow.air_by_date:
+            segments = self._get_segments(curShow, fromDate)
-                segments = [x[1] for x in self._get_air_by_date_segments(curShow.indexerid, fromDate)]
-            else:
-                segments = self._get_segments(curShow.indexerid, fromDate)
 
-            for cur_segment in segments:
+            if len(segments):
-
+                backlog_queue_item = search_queue.BacklogQueueItem(curShow, segments)
-                self.currentSearchInfo = {'title': curShow.name + " Season " + str(cur_segment)}
+                sickbeard.searchQueueScheduler.action.add_item(backlog_queue_item)  #@UndefinedVariable
-
-                backlog_queue_item = search_queue.BacklogQueueItem(curShow, cur_segment)
-
-                if backlog_queue_item.wantedEpisodes:
-                    sickbeard.searchQueueScheduler.action.add_item(backlog_queue_item)  #@UndefinedVariable
-                else:
-                    logger.log(
-                        u"Nothing in season " + str(cur_segment) + " needs to be downloaded, skipping this season",
-                        logger.DEBUG)
 
         # don't consider this an actual backlog search if we only did recent eps
         # or if we only did certain shows
@@ -158,34 +129,42 @@ class BacklogSearcher:
         self._lastBacklog = lastBacklog
         return self._lastBacklog
 
-    def _get_segments(self, indexer_id, fromDate):
+    def _get_segments(self, show, fromDate):
+        anyQualities, bestQualities = common.Quality.splitQuality(show.quality)  #@UnusedVariable
+
         myDB = db.DBConnection()
-        sqlResults = myDB.select(
+        if show.air_by_date:
-            "SELECT DISTINCT(season) as season FROM tv_episodes WHERE showid = ? AND season > 0 and airdate > ?",
+            sqlResults = myDB.select(
-            [indexer_id, fromDate.toordinal()])
+                "SELECT ep.status, ep.season, ep.episode FROM tv_episodes ep, tv_shows show WHERE season != 0 AND ep.showid = show.indexer_id AND show.paused = 0 ANd ep.airdate > ? AND ep.showid = ? AND show.air_by_date = 1",
+                [fromDate.toordinal(), show.indexerid])
+        else:
+            sqlResults = myDB.select(
+                "SELECT status, season, episode FROM tv_episodes WHERE showid = ? AND season > 0 and airdate > ?",
+                [show.indexerid, fromDate.toordinal()])
 
-        return [int(x["season"]) for x in sqlResults]
+        # check through the list of statuses to see if we want any
+        wanted = {}
+        for result in sqlResults:
+            curCompositeStatus = int(result["status"])
+            curStatus, curQuality = common.Quality.splitCompositeStatus(curCompositeStatus)
 
-    def _get_air_by_date_segments(self, indexer_id, fromDate):
+            if bestQualities:
-        # query the DB for all dates for this show
+                highestBestQuality = max(bestQualities)
-        myDB = db.DBConnection()
+            else:
-        num_air_by_date_results = myDB.select(
+                highestBestQuality = 0
-            "SELECT airdate, showid FROM tv_episodes ep, tv_shows show WHERE season != 0 AND ep.showid = show.indexer_id AND show.paused = 0 ANd ep.airdate > ? AND ep.showid = ? AND show.air_by_date = 1",
-            [fromDate.toordinal(), indexer_id])
 
-        # break them apart into month/year strings
+            # if we need a better one then say yes
-        air_by_date_segments = []
+            if (curStatus in (common.DOWNLOADED, common.SNATCHED, common.SNATCHED_PROPER,
-        for cur_result in num_air_by_date_results:
+                              common.SNATCHED_BEST) and curQuality < highestBestQuality) or curStatus == common.WANTED:
-            cur_date = datetime.date.fromordinal(int(cur_result["airdate"]))
-            cur_date_str = str(cur_date)[:7]
-            cur_indexer_id = int(cur_result["showid"])
 
-            cur_result_tuple = (cur_indexer_id, cur_date_str)
+                epObj = show.getEpisode(int(result["season"]), int(result["episode"]))
-            if cur_result_tuple not in air_by_date_segments:
-                air_by_date_segments.append(cur_result_tuple)
 
-        return air_by_date_segments
+                if epObj.season in wanted:
+                    wanted[epObj.season].append(epObj)
+                else:
+                    wanted[epObj.season] = [epObj]
 
+        return wanted
 
     def _set_lastBacklog(self, when):
 

sickbeard/search_queue.py

@@ -42,16 +42,8 @@ class SearchQueue(generic_queue.GenericQueue):
     def is_in_queue(self, show, segment):
         queue =  [x for x in self.queue.queue] + [self.currentItem]
         for cur_item in queue:
-            with search_queue_lock:
+            if cur_item:
-                if isinstance(cur_item, BacklogQueueItem) and cur_item.show == show and cur_item.segment == segment:
+                if cur_item.show == show and cur_item.segment == segment:
-                    return True
-        return False
-
-    def is_ep_in_queue(self, ep_obj):
-        queue = [x for x in self.queue.queue] + [self.currentItem]
-        for cur_item in queue:
-            with search_queue_lock:
-                if isinstance(cur_item, ManualSearchQueueItem) and cur_item.ep_obj == ep_obj:
                     return True
         return False
 
@@ -76,9 +68,9 @@ class SearchQueue(generic_queue.GenericQueue):
 
         if isinstance(item, BacklogQueueItem) and not self.is_in_queue(item.show, item.segment):
             generic_queue.GenericQueue.add_item(self, item)
-        elif isinstance(item, ManualSearchQueueItem) and not self.is_ep_in_queue(item.ep_obj):
+        elif isinstance(item, ManualSearchQueueItem) and not self.is_in_queue(item.show, item.segment):
             generic_queue.GenericQueue.add_item(self, item)
-        elif isinstance(item, FailedQueueItem) and not self.is_in_queue(item.show, item.episodes):
+        elif isinstance(item, FailedQueueItem) and not self.is_in_queue(item.show, item.segment):
             generic_queue.GenericQueue.add_item(self, item)
         else:
             logger.log(u"Not adding item, it's already in the queue", logger.DEBUG)
@@ -92,29 +84,29 @@ class SearchQueue(generic_queue.GenericQueue):
             generic_queue.QueueItem.finish(item)
 
 class ManualSearchQueueItem(generic_queue.QueueItem):
-    def __init__(self, ep_obj):
+    def __init__(self, show, segment):
         generic_queue.QueueItem.__init__(self, 'Manual Search', MANUAL_SEARCH)
         self.priority = generic_queue.QueuePriorities.HIGH
-        self.thread_name = 'MANUAL-' + str(ep_obj.show.indexerid) + '-'
+        self.thread_name = 'MANUAL-' + str(show.indexerid) + '-'
         self.success = None
-        self.show = ep_obj.show
+        self.show = show
-        self.ep_obj = ep_obj
+        self.segment = segment
         self.results = []
 
     def execute(self):
         generic_queue.QueueItem.execute(self)
 
         try:
-            logger.log("Beginning manual search for [" + self.ep_obj.prettyName() + "]")
+            logger.log("Beginning manual search for [" + self.segment.prettyName() + "]")
-            searchResult = search.searchProviders(self, self.show, self.ep_obj.season, [self.ep_obj],False,True)
+            searchResult = search.searchProviders(self, self.show, self.segment.season, [self.segment],False,True)
 
             if searchResult:
                 SearchQueue().snatch_item(searchResult)
             else:
                 ui.notifications.message('No downloads were found',
-                                         "Couldn't find a download for <i>%s</i>" % self.ep_obj.prettyName())
+                                         "Couldn't find a download for <i>%s</i>" % self.segment.prettyName())
 
-                logger.log(u"Unable to find a download for " + self.ep_obj.prettyName())
+                logger.log(u"Unable to find a download for " + self.segment.prettyName())
 
         except Exception:
             logger.log(traceback.format_exc(), logger.DEBUG)
@@ -129,86 +121,41 @@ class BacklogQueueItem(generic_queue.QueueItem):
         self.success = None
         self.show = show
         self.segment = segment
-        self.wantedEpisodes = []
         self.results = []
 
-        logger.log(u"Seeing if we need any episodes from " + self.show.name + " season " + str(self.segment))
-
-        myDB = db.DBConnection()
-
-        # see if there is anything in this season worth searching for
-        if not self.show.air_by_date:
-            statusResults = myDB.select("SELECT status, episode FROM tv_episodes WHERE showid = ? AND season = ?",
-                                        [self.show.indexerid, self.segment])
-        else:
-            season_year, season_month = map(int, self.segment.split('-'))
-            min_date = datetime.date(season_year, season_month, 1)
-
-            # it's easier to just hard code this than to worry about rolling the year over or making a month length map
-            if season_month == 12:
-                max_date = datetime.date(season_year, 12, 31)
-            else:
-                max_date = datetime.date(season_year, season_month + 1, 1) - datetime.timedelta(days=1)
-
-            statusResults = myDB.select(
-                "SELECT status, episode FROM tv_episodes WHERE showid = ? AND airdate >= ? AND airdate <= ?",
-                [self.show.indexerid, min_date.toordinal(), max_date.toordinal()])
-
-        anyQualities, bestQualities = common.Quality.splitQuality(self.show.quality)  #@UnusedVariable
-        self.wantedEpisodes = self._need_any_episodes(statusResults, bestQualities)
-
     def execute(self):
         generic_queue.QueueItem.execute(self)
 
-        # check if we want to search for season packs instead of just season/episode
+        for season in self.segment:
-        seasonSearch = False
+            wantedEps = self.segment[season]
-        seasonEps = self.show.getAllEpisodes(self.segment)
-        if len(seasonEps) == len(self.wantedEpisodes):
-            seasonSearch = True
 
-        try:
+            # check if we want to search for season packs instead of just season/episode
-            logger.log("Beginning backlog search for episodes from [" + self.show.name + "]  - Season[" + str(self.segment) + "]")
+            seasonSearch = False
-            searchResult = search.searchProviders(self, self.show, self.segment, self.wantedEpisodes, seasonSearch, False)
+            seasonEps = self.show.getAllEpisodes(season)
+            if len(seasonEps) == len(wantedEps) and not sickbeard.PREFER_EPISODE_RELEASES:
+                seasonSearch = True
 
-            if searchResult:
+            try:
-                SearchQueue().snatch_item(searchResult)
+                logger.log("Beginning backlog search for episodes from [" + self.show.name + "]  - Season[" + str(season) + "]")
-            else:
+                searchResult = search.searchProviders(self, self.show, season, wantedEps, seasonSearch, False)
-                logger.log(u"No needed episodes found during backlog search")
 
-        except Exception:
+                if searchResult:
-            logger.log(traceback.format_exc(), logger.DEBUG)
+                    SearchQueue().snatch_item(searchResult)
+                else:
+                    logger.log(u"No needed episodes found during backlog search")
+
+            except Exception:
+                logger.log(traceback.format_exc(), logger.DEBUG)
 
         self.finish()
 
-    def _need_any_episodes(self, statusResults, bestQualities):
-        wantedEpisodes = []
-
-        # check through the list of statuses to see if we want any
-        for curStatusResult in statusResults:
-            curCompositeStatus = int(curStatusResult["status"])
-            curStatus, curQuality = common.Quality.splitCompositeStatus(curCompositeStatus)
-            episode = int(curStatusResult["episode"])
-
-            if bestQualities:
-                highestBestQuality = max(bestQualities)
-            else:
-                highestBestQuality = 0
-
-            # if we need a better one then say yes
-            if (curStatus in (common.DOWNLOADED, common.SNATCHED, common.SNATCHED_PROPER,
-                              common.SNATCHED_BEST) and curQuality < highestBestQuality) or curStatus == common.WANTED:
-                epObj = self.show.getEpisode(self.segment, episode)
-                wantedEpisodes.append(epObj)
-
-        return wantedEpisodes
-
 class FailedQueueItem(generic_queue.QueueItem):
-    def __init__(self, show, episodes):
+    def __init__(self, show, segment):
         generic_queue.QueueItem.__init__(self, 'Retry', FAILED_SEARCH)
         self.priority = generic_queue.QueuePriorities.HIGH
         self.thread_name = 'RETRY-' + str(show.indexerid) + '-'
         self.show = show
-        self.episodes = episodes
+        self.segment = segment
         self.success = None
         self.results = []
 
@@ -216,7 +163,9 @@ class FailedQueueItem(generic_queue.QueueItem):
         generic_queue.QueueItem.execute(self)
 
         failed_episodes = []
-        for i, epObj in enumerate(self.episodes):
+        for season in self.segment:
+            epObj = self.segment[season]
+
             (release, provider) = failed_history.findRelease(epObj)
             if release:
                 logger.log(u"Marking release as bad: " + release)
@@ -226,11 +175,11 @@ class FailedQueueItem(generic_queue.QueueItem):
                 failed_history.revertEpisode(epObj)
                 failed_episodes.append(epObj)
 
+                logger.log(
+                    "Beginning failed download search for [" + epObj.prettyName() + "]")
+
         if len(failed_episodes):
             try:
-                logger.log(
-                    "Beginning failed download search for episodes from Season [" + str(self.episodes[0].season) + "]")
-
                 searchResult = search.searchProviders(self, self.show, failed_episodes[0].season, failed_episodes, False, True)
 
                 if searchResult:

sickbeard/tv.py

@@ -140,20 +140,7 @@ class TVShow(object):
         sql_selection = sql_selection + " FROM tv_episodes tve WHERE showid = " + str(self.indexerid)
 
         if season is not None:
-            if not self.air_by_date:
+            sql_selection = sql_selection + " AND season = " + str(season)
-                sql_selection = sql_selection + " AND season = " + str(season)
-            else:
-                segment_year, segment_month = map(int, str(season).split('-'))
-                min_date = datetime.date(segment_year, segment_month, 1)
-
-                # it's easier to just hard code this than to worry about rolling the year over or making a month length map
-                if segment_month == 12:
-                    max_date = datetime.date(segment_year, 12, 31)
-                else:
-                    max_date = datetime.date(segment_year, segment_month + 1, 1) - datetime.timedelta(days=1)
-
-                sql_selection = sql_selection + " AND airdate >= " + str(
-                    min_date.toordinal()) + " AND airdate <= " + str(max_date.toordinal())
 
         if has_location:
             sql_selection = sql_selection + " AND location != '' "

sickbeard/webserve.py

@@ -3227,8 +3227,7 @@ class Home:
             else:
                 return _genericMessage("Error", errMsg)
 
-        wanted_segments = []
+        segments = {}
-        failed_segments = {}
 
         if eps is not None:
 
@@ -3244,22 +3243,12 @@ class Home:
                 if epObj is None:
                     return _genericMessage("Error", "Episode couldn't be retrieved")
 
-                if int(status) == WANTED:
+                if int(status) in [WANTED, FAILED]:
                     # figure out what episodes are wanted so we can backlog them
-                    if epObj.show.air_by_date or epObj.show.sports:
+                    if epObj in segments:
-                        segment = str(epObj.airdate)[:7]
+                        segments[epObj.season].append(epObj)
                     else:
-                        segment = epObj.season
+                        segments[epObj.season] = [epObj]
-
-                    if segment not in wanted_segments:
-                        wanted_segments.append(segment)
-
-                elif int(status) == FAILED:
-                    # figure out what episodes failed so we can retry them
-                    if epObj.season not in failed_segments:
-                        failed_segments[epObj.season] = []
-                    if epObj.episode not in failed_segments[epObj.season]:
-                        failed_segments[epObj.season].append(epObj.episode)
 
                 with epObj.lock:
                     # don't let them mess up UNAIRED episodes
@@ -3294,7 +3283,7 @@ class Home:
 
         if int(status) == WANTED:
             msg = "Backlog was automatically started for the following seasons of <b>" + showObj.name + "</b>:<br />"
-            for cur_segment in wanted_segments:
+            for cur_segment in segments:
                 msg += "<li>Season " + str(cur_segment) + "</li>"
                 logger.log(u"Sending backlog for " + showObj.name + " season " + str(
                     cur_segment) + " because some eps were set to wanted")
@@ -3302,12 +3291,12 @@ class Home:
                 sickbeard.searchQueueScheduler.action.add_item(cur_backlog_queue_item)  # @UndefinedVariable
             msg += "</ul>"
 
-            if wanted_segments:
+            if segments:
                 ui.notifications.message("Backlog started", msg)
 
         if int(status) == FAILED:
             msg = "Retrying Search was automatically started for the following season of <b>" + showObj.name + "</b>:<br />"
-            for cur_segment in failed_segments:
+            for cur_segment in segments:
                 msg += "<li>Season " + str(cur_segment) + "</li>"
                 logger.log(u"Retrying Search for " + showObj.name + " season " + str(
                     cur_segment) + " because some eps were set to failed")
@@ -3315,7 +3304,7 @@ class Home:
                 sickbeard.searchQueueScheduler.action.add_item(cur_failed_queue_item)  # @UndefinedVariable
             msg += "</ul>"
 
-            if failed_segments:
+            if segments:
                 ui.notifications.message("Retry Search started", msg)
 
         if direct:
@@ -3427,7 +3416,7 @@ class Home:
             return json.dumps({'result': 'failure'})
 
         # make a queue item for it and put it on the queue
-        ep_queue_item = search_queue.ManualSearchQueueItem(ep_obj)
+        ep_queue_item = search_queue.ManualSearchQueueItem(ep_obj.show, ep_obj)
         sickbeard.searchQueueScheduler.action.add_item(ep_queue_item)  # @UndefinedVariable
 
         # wait until the queue item tells us whether it worked or not