moparisthebest
/
SickRage
mirror of https://github.com/moparisthebest/SickRage
1
0
Fork 0
Code Issues Releases Wiki Activity

Updated cacheHandler to allow us to force cache and set a max-age on the cached content, vastly improved performance with indexers.

This commit is contained in:
echel0n 2014-03-14 15:35:05 -07:00
parent 05410e2aa0
commit 40c69d6a0f
5 changed files with 36 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/cachecontrol/adapter.py
View File

@ -7,10 +7,10 @@ from cachecontrol.cache import DictCache
class CacheControlAdapter(HTTPAdapter):
invalidating_methods = set(['PUT', 'DELETE'])
def __init__(self, cache=None, cache_etags=True, *args, **kw):
def __init__(self, cache=None, cache_etags=True, cache_force=False, *args, **kw):
super(CacheControlAdapter, self).__init__(*args, **kw)
self.cache = cache or DictCache()
self.controller = CacheController(self.cache, cache_etags=cache_etags)
self.controller = CacheController(self.cache, cache_etags=cache_etags, cache_force=cache_force)
def send(self, request, **kw):
"""Send a request. Use the request information to see if it

11
lib/cachecontrol/controller.py
View File

@ -4,6 +4,7 @@ The httplib2 algorithms ported for use with requests.
import re
import calendar
import time
import os
from cachecontrol.cache import DictCache
from cachecontrol.compat import parsedate_tz
@ -24,9 +25,10 @@ def parse_uri(uri):
class CacheController(object):
"""An interface to see if request should cached or not.
"""
def __init__(self, cache=None, cache_etags=True):
def __init__(self, cache=None, cache_etags=True, cache_force=False):
self.cache = cache or DictCache()
self.cache_etags = cache_etags
self.cache_force = cache_force
def _urlnorm(self, uri):
"""Normalize the URL to create a safe key for the cache"""
@ -166,7 +168,7 @@ class CacheController(object):
# return the original handler
return False
def add_headers(self, url):
def add_headers(self, url, resp=None):
resp = self.cache.get(url)
if resp and 'etag' in resp.headers:
return {'If-None-Match': resp.headers['etag']}
@ -212,6 +214,11 @@ class CacheController(object):
if resp.headers['expires']:
self.cache.set(cache_url, resp)
# If the request is for our local cache, it means we should cache it
elif self.cache_force:
resp.headers.update({'cache-control': 'max-age=21600, private'})
self.cache.set(cache_url, resp)
def update_cached_response(self, request, response):
"""On a 304 we will get a new set of headers that we want to
update our cached value with, assuming we have one.

4
lib/cachecontrol/wrapper.py
View File

@ -2,9 +2,9 @@ from cachecontrol.adapter import CacheControlAdapter
from cachecontrol.cache import DictCache
def CacheControl(sess, cache=None, cache_etags=True):
def CacheControl(sess, cache=None, cache_etags=True, cache_force=False):
cache = cache or DictCache()
adapter = CacheControlAdapter(cache, cache_etags=cache_etags)
adapter = CacheControlAdapter(cache, cache_etags=cache_etags, cache_force=cache_force)
sess.mount('http://', adapter)
return sess

44
lib/tvdb_api/tvdb_api.py
View File

@ -494,7 +494,7 @@ class Tvdb:
self.config['params_getSeries'] = {"seriesname": "", "language": "all"}
else:
self.config['url_getSeries'] = u"%(base_url)s/api/GetSeries.php" % self.config
self.config['params_getSeries'] = {"seriesname": "", "language": ""}
self.config['params_getSeries'] = {"seriesname": "", "language": self.config['language']}
self.config['url_epInfo'] = u"%(base_url)s/api/%(apikey)s/series/%%s/all/%%s.xml" % self.config
self.config['url_epInfo_zip'] = u"%(base_url)s/api/%(apikey)s/series/%%s/all/%%s.zip" % self.config
@ -527,7 +527,7 @@ class Tvdb:
# cacheControl
if self.config['cache_enabled']:
sess = CacheControl(requests.Session(), cache=FileCache(self.config['cache_location']))
sess = CacheControl(requests.Session(), cache_force=True, cache=FileCache(self.config['cache_location']))
else:
sess = requests.Session()
@ -538,26 +538,26 @@ class Tvdb:
lastTimeout = datetime.datetime.now()
raise tvdb_error("Could not connect to server: %s" % (e))
# handle gzipped content,
# http://dbr.lighthouseapp.com/projects/13342/tickets/72-gzipped-data-patch
if 'gzip' in resp.headers.get("Content-Encoding", ''):
if gzip:
stream = StringIO.StringIO(resp.content)
gz = gzip.GzipFile(fileobj=stream)
return gz.read()
raise tvdb_error("Received gzip data from thetvdb.com, but could not correctly handle it")
if 'application/zip' in resp.headers.get("Content-Type", ''):
try:
# TODO: The zip contains actors.xml and banners.xml, which are currently ignored [GH-20]
log().debug("We recived a zip file unpacking now ...")
zipdata = StringIO.StringIO()
zipdata.write(resp.content)
myzipfile = zipfile.ZipFile(zipdata)
return myzipfile.read('%s.xml' % language)
except zipfile.BadZipfile:
raise tvdb_error("Bad zip file received from thetvdb.com, could not read it")
## handle gzipped content,
## http://dbr.lighthouseapp.com/projects/13342/tickets/72-gzipped-data-patch
#if 'gzip' in resp.headers.get("Content-Encoding", ''):
# if gzip:
# stream = StringIO.StringIO(resp.content)
# gz = gzip.GzipFile(fileobj=stream)
# return gz.read()
#
# raise tvdb_error("Received gzip data from thetvdb.com, but could not correctly handle it")
#
#if 'application/zip' in resp.headers.get("Content-Type", ''):
# try:
# # TODO: The zip contains actors.xml and banners.xml, which are currently ignored [GH-20]
# log().debug("We recived a zip file unpacking now ...")
# zipdata = StringIO.StringIO()
# zipdata.write(resp.content)
# myzipfile = zipfile.ZipFile(zipdata)
# return myzipfile.read('%s.xml' % language)
# except zipfile.BadZipfile:
# raise tvdb_error("Bad zip file received from thetvdb.com, could not read it")
return resp.content

2
lib/tvrage_api/tvrage_api.py
View File

@ -358,7 +358,7 @@ class TVRage:
# cacheControl
if self.config['cache_enabled']:
sess = CacheControl(requests.Session(), cache=FileCache(self.config['cache_location']))
sess = CacheControl(requests.Session(), cache_force=True, cache=FileCache(self.config['cache_location']))
else:
sess = requests.Session()