SickRage/lib/feedcache/cache.py

#!/usr/bin/env python
#
# Copyright 2007 Doug Hellmann.
#
#
#                         All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#

"""

"""

__module_id__ = "$Id$"

#
# Import system modules
#
from feedparser import feedparser

import logging
import time

#
# Import local modules
#


#
# Module
#

logger = logging.getLogger('feedcache.cache')


class Cache:
    """A class to wrap Mark Pilgrim's Universal Feed Parser module
    (http://www.feedparser.org) so that parameters can be used to
    cache the feed results locally instead of fetching the feed every
    time it is requested. Uses both etag and modified times for
    caching.
    """

    def __init__(self, storage, timeToLiveSeconds=300, userAgent='feedcache'):
        """
        Arguments:

          storage -- Backing store for the cache.  It should follow
          the dictionary API, with URLs used as keys.  It should
          persist data.

          timeToLiveSeconds=300 -- The length of time content should
          live in the cache before an update is attempted.

          userAgent='feedcache' -- User agent string to be used when
          fetching feed contents.

        """
        self.storage = storage
        self.time_to_live = timeToLiveSeconds
        self.user_agent = userAgent
        return

    def purge(self, olderThanSeconds):
        """Remove cached data from the storage if the data is older than the
        date given.  If olderThanSeconds is None, the entire cache is purged.
        """
        if olderThanSeconds is None:
            logger.debug('purging the entire cache')
            for key in self.storage.keys():
                del self.storage[key]
        else:
            now = time.time()
            # Iterate over the keys and load each item one at a time
            # to avoid having the entire cache loaded into memory
            # at one time.
            for url in self.storage.keys():
                (cached_time, cached_data) = self.storage[url]
                age = now - cached_time
                if age >= olderThanSeconds:
                    logger.debug('removing %s with age %d', url, age)
                    del self.storage[url]
        return

    def fetch(self, url, force_update=False, offline=False, request_headers=None, referrer=None):
        """Return the feed at url.

        url - The URL of the feed.

        force_update=False - When True, update the cache whether the
                                           current contents have
                                           exceeded their time-to-live
                                           or not.

        offline=False - When True, only return data from the local
                                 cache and never access the remote
                                 URL.

        request_headers=None - Add addition request headers to request

        referrer=None - Added a referrer to request

        If there is data for that feed in the cache already, check
        the expiration date before accessing the server.  If the
        cached data has not expired, return it without accessing the
        server.

        In cases where the server is accessed, check for updates
        before deciding what to return.  If the server reports a
        status of 304, the previously cached content is returned.

        The cache is only updated if the server returns a status of
        200, to avoid holding redirected data in the cache.
        """
        logger.debug('url="%s"' % url)

        # Convert the URL to a value we can use
        # as a key for the storage backend.
        key = url
        if isinstance(key, unicode):
            key = key.encode('utf-8')

        modified = None
        etag = None
        now = time.time()

        cached_time, cached_content = self.storage.get(key, (None, None))

        # Offline mode support (no networked requests)
        # so return whatever we found in the storage.
        # If there is nothing in the storage, we'll be returning None.
        if offline:
            logger.debug('offline mode')
            return cached_content

        # Does the storage contain a version of the data
        # which is older than the time-to-live?
        logger.debug('cache modified time: %s' % str(cached_time))
        if cached_time is not None and not force_update:
            if self.time_to_live:
                age = now - cached_time
                if age <= self.time_to_live:
                    logger.debug('cache contents still valid')
                    return cached_content
                else:
                    logger.debug('cache contents older than TTL')
            else:
                logger.debug('no TTL value')

            # The cache is out of date, but we have
            # something.  Try to use the etag and modified_time
            # values from the cached content.
            etag = cached_content.get('etag')
            modified = cached_content.get('modified')
            logger.debug('cached etag=%s' % etag)
            logger.debug('cached modified=%s' % str(modified))
        else:
            logger.debug('nothing in the cache, or forcing update')

        # We know we need to fetch, so go ahead and do it.
        logger.debug('fetching...')
        parsed_result = feedparser.parse(url,
                                         agent=self.user_agent,
                                         modified=modified,
                                         etag=etag,
                                         referrer=referrer,
                                         request_headers=request_headers)

        status = parsed_result.get('status', None)
        logger.debug('HTTP status=%s' % status)
        if status == 304:
            # No new data, based on the etag or modified values.
            # We need to update the modified time in the
            # storage, though, so we know that what we have
            # stored is up to date.
            self.storage[key] = (now, cached_content)

            # Return the data from the cache, since
            # the parsed data will be empty.
            parsed_result = cached_content
        elif status == 200:
            # There is new content, so store it unless there was an error.
            error = parsed_result.get('bozo_exception')
            if not error:
                logger.debug('Updating stored data for %s' % url)
                self.storage[key] = (now, parsed_result)
            else:
                logger.warning('Not storing data with exception: %s',
                               error)
        else:
            logger.warning('Not updating cache with HTTP status %s', status)

        return parsed_result