mirror of
https://github.com/moparisthebest/SickRage
synced 2024-08-13 16:53:54 -04:00
0d9fbc1ad7
This version of SickBeard uses both TVDB and TVRage to search and gather it's series data from allowing you to now have access to and download shows that you couldn't before because of being locked into only what TheTVDB had to offer. Also this edition is based off the code we used in our XEM editon so it does come with scene numbering support as well as all the other features our XEM edition has to offer. Please before using this with your existing database (sickbeard.db) please make a backup copy of it and delete any other database files such as cache.db and failed.db if present, we HIGHLY recommend starting out with no database files at all to make this a fresh start but the choice is at your own risk! Enjoy!
259 lines
8.8 KiB
Python
259 lines
8.8 KiB
Python
# urllib3/poolmanager.py
|
|
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
|
#
|
|
# This module is part of urllib3 and is released under
|
|
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
|
|
|
import logging
|
|
|
|
try: # Python 3
|
|
from urllib.parse import urljoin
|
|
except ImportError:
|
|
from urlparse import urljoin
|
|
|
|
from ._collections import RecentlyUsedContainer
|
|
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
|
from .connectionpool import port_by_scheme
|
|
from .request import RequestMethods
|
|
from .util import parse_url
|
|
|
|
|
|
__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
|
|
|
|
|
|
pool_classes_by_scheme = {
|
|
'http': HTTPConnectionPool,
|
|
'https': HTTPSConnectionPool,
|
|
}
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
|
|
'ssl_version')
|
|
|
|
|
|
class PoolManager(RequestMethods):
|
|
"""
|
|
Allows for arbitrary requests while transparently keeping track of
|
|
necessary connection pools for you.
|
|
|
|
:param num_pools:
|
|
Number of connection pools to cache before discarding the least
|
|
recently used pool.
|
|
|
|
:param headers:
|
|
Headers to include with all requests, unless other headers are given
|
|
explicitly.
|
|
|
|
:param \**connection_pool_kw:
|
|
Additional parameters are used to create fresh
|
|
:class:`urllib3.connectionpool.ConnectionPool` instances.
|
|
|
|
Example: ::
|
|
|
|
>>> manager = PoolManager(num_pools=2)
|
|
>>> r = manager.request('GET', 'http://google.com/')
|
|
>>> r = manager.request('GET', 'http://google.com/mail')
|
|
>>> r = manager.request('GET', 'http://yahoo.com/')
|
|
>>> len(manager.pools)
|
|
2
|
|
|
|
"""
|
|
|
|
proxy = None
|
|
|
|
def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
|
|
RequestMethods.__init__(self, headers)
|
|
self.connection_pool_kw = connection_pool_kw
|
|
self.pools = RecentlyUsedContainer(num_pools,
|
|
dispose_func=lambda p: p.close())
|
|
|
|
def _new_pool(self, scheme, host, port):
|
|
"""
|
|
Create a new :class:`ConnectionPool` based on host, port and scheme.
|
|
|
|
This method is used to actually create the connection pools handed out
|
|
by :meth:`connection_from_url` and companion methods. It is intended
|
|
to be overridden for customization.
|
|
"""
|
|
pool_cls = pool_classes_by_scheme[scheme]
|
|
kwargs = self.connection_pool_kw
|
|
if scheme == 'http':
|
|
kwargs = self.connection_pool_kw.copy()
|
|
for kw in SSL_KEYWORDS:
|
|
kwargs.pop(kw, None)
|
|
|
|
return pool_cls(host, port, **kwargs)
|
|
|
|
def clear(self):
|
|
"""
|
|
Empty our store of pools and direct them all to close.
|
|
|
|
This will not affect in-flight connections, but they will not be
|
|
re-used after completion.
|
|
"""
|
|
self.pools.clear()
|
|
|
|
def connection_from_host(self, host, port=None, scheme='http'):
|
|
"""
|
|
Get a :class:`ConnectionPool` based on the host, port, and scheme.
|
|
|
|
If ``port`` isn't given, it will be derived from the ``scheme`` using
|
|
``urllib3.connectionpool.port_by_scheme``.
|
|
"""
|
|
|
|
scheme = scheme or 'http'
|
|
|
|
port = port or port_by_scheme.get(scheme, 80)
|
|
|
|
pool_key = (scheme, host, port)
|
|
|
|
with self.pools.lock:
|
|
# If the scheme, host, or port doesn't match existing open
|
|
# connections, open a new ConnectionPool.
|
|
pool = self.pools.get(pool_key)
|
|
if pool:
|
|
return pool
|
|
|
|
# Make a fresh ConnectionPool of the desired type
|
|
pool = self._new_pool(scheme, host, port)
|
|
self.pools[pool_key] = pool
|
|
return pool
|
|
|
|
def connection_from_url(self, url):
|
|
"""
|
|
Similar to :func:`urllib3.connectionpool.connection_from_url` but
|
|
doesn't pass any additional parameters to the
|
|
:class:`urllib3.connectionpool.ConnectionPool` constructor.
|
|
|
|
Additional parameters are taken from the :class:`.PoolManager`
|
|
constructor.
|
|
"""
|
|
u = parse_url(url)
|
|
return self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
|
|
|
|
def urlopen(self, method, url, redirect=True, **kw):
|
|
"""
|
|
Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
|
|
with custom cross-host redirect logic and only sends the request-uri
|
|
portion of the ``url``.
|
|
|
|
The given ``url`` parameter must be absolute, such that an appropriate
|
|
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
|
|
"""
|
|
u = parse_url(url)
|
|
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
|
|
|
|
kw['assert_same_host'] = False
|
|
kw['redirect'] = False
|
|
if 'headers' not in kw:
|
|
kw['headers'] = self.headers
|
|
|
|
if self.proxy is not None and u.scheme == "http":
|
|
response = conn.urlopen(method, url, **kw)
|
|
else:
|
|
response = conn.urlopen(method, u.request_uri, **kw)
|
|
|
|
redirect_location = redirect and response.get_redirect_location()
|
|
if not redirect_location:
|
|
return response
|
|
|
|
# Support relative URLs for redirecting.
|
|
redirect_location = urljoin(url, redirect_location)
|
|
|
|
# RFC 2616, Section 10.3.4
|
|
if response.status == 303:
|
|
method = 'GET'
|
|
|
|
log.info("Redirecting %s -> %s" % (url, redirect_location))
|
|
kw['retries'] = kw.get('retries', 3) - 1 # Persist retries countdown
|
|
kw['redirect'] = redirect
|
|
return self.urlopen(method, redirect_location, **kw)
|
|
|
|
|
|
class ProxyManager(PoolManager):
|
|
"""
|
|
Behaves just like :class:`PoolManager`, but sends all requests through
|
|
the defined proxy, using the CONNECT method for HTTPS URLs.
|
|
|
|
:param poxy_url:
|
|
The URL of the proxy to be used.
|
|
|
|
:param proxy_headers:
|
|
A dictionary contaning headers that will be sent to the proxy. In case
|
|
of HTTP they are being sent with each request, while in the
|
|
HTTPS/CONNECT case they are sent only once. Could be used for proxy
|
|
authentication.
|
|
|
|
Example:
|
|
>>> proxy = urllib3.ProxyManager('http://localhost:3128/')
|
|
>>> r1 = proxy.request('GET', 'http://google.com/')
|
|
>>> r2 = proxy.request('GET', 'http://httpbin.org/')
|
|
>>> len(proxy.pools)
|
|
1
|
|
>>> r3 = proxy.request('GET', 'https://httpbin.org/')
|
|
>>> r4 = proxy.request('GET', 'https://twitter.com/')
|
|
>>> len(proxy.pools)
|
|
3
|
|
|
|
"""
|
|
|
|
def __init__(self, proxy_url, num_pools=10, headers=None,
|
|
proxy_headers=None, **connection_pool_kw):
|
|
|
|
if isinstance(proxy_url, HTTPConnectionPool):
|
|
proxy_url = '%s://%s:%i' % (proxy_url.scheme, proxy_url.host,
|
|
proxy_url.port)
|
|
proxy = parse_url(proxy_url)
|
|
if not proxy.port:
|
|
port = port_by_scheme.get(proxy.scheme, 80)
|
|
proxy = proxy._replace(port=port)
|
|
self.proxy = proxy
|
|
self.proxy_headers = proxy_headers or {}
|
|
assert self.proxy.scheme in ("http", "https"), \
|
|
'Not supported proxy scheme %s' % self.proxy.scheme
|
|
connection_pool_kw['_proxy'] = self.proxy
|
|
connection_pool_kw['_proxy_headers'] = self.proxy_headers
|
|
super(ProxyManager, self).__init__(
|
|
num_pools, headers, **connection_pool_kw)
|
|
|
|
def connection_from_host(self, host, port=None, scheme='http'):
|
|
if scheme == "https":
|
|
return super(ProxyManager, self).connection_from_host(
|
|
host, port, scheme)
|
|
|
|
return super(ProxyManager, self).connection_from_host(
|
|
self.proxy.host, self.proxy.port, self.proxy.scheme)
|
|
|
|
def _set_proxy_headers(self, url, headers=None):
|
|
"""
|
|
Sets headers needed by proxies: specifically, the Accept and Host
|
|
headers. Only sets headers not provided by the user.
|
|
"""
|
|
headers_ = {'Accept': '*/*'}
|
|
|
|
netloc = parse_url(url).netloc
|
|
if netloc:
|
|
headers_['Host'] = netloc
|
|
|
|
if headers:
|
|
headers_.update(headers)
|
|
return headers_
|
|
|
|
def urlopen(self, method, url, redirect=True, **kw):
|
|
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
|
|
u = parse_url(url)
|
|
|
|
if u.scheme == "http":
|
|
# For proxied HTTPS requests, httplib sets the necessary headers
|
|
# on the CONNECT to the proxy. For HTTP, we'll definitely
|
|
# need to set 'Host' at the very least.
|
|
kw['headers'] = self._set_proxy_headers(url, kw.get('headers',
|
|
self.headers))
|
|
|
|
return super(ProxyManager, self).urlopen(method, url, redirect, **kw)
|
|
|
|
|
|
def proxy_from_url(url, **kw):
|
|
return ProxyManager(proxy_url=url, **kw)
|