mirror of
https://github.com/moparisthebest/SickRage
synced 2024-11-12 04:15:09 -05:00
05410e2aa0
Replaced urllib2 with requests for both TVDB and TVRage. Updated cache cleanup code to process both TVDB and TVRage cache folders.
391 lines
14 KiB
Python
391 lines
14 KiB
Python
'''SSL with SNI_-support for Python 2.
|
|
|
|
This needs the following packages installed:
|
|
|
|
* pyOpenSSL (tested with 0.13)
|
|
* ndg-httpsclient (tested with 0.3.2)
|
|
* pyasn1 (tested with 0.1.6)
|
|
|
|
To activate it call :func:`~urllib3.contrib.pyopenssl.inject_into_urllib3`.
|
|
This can be done in a ``sitecustomize`` module, or at any other time before
|
|
your application begins using ``urllib3``, like this::
|
|
|
|
try:
|
|
import urllib3.contrib.pyopenssl
|
|
urllib3.contrib.pyopenssl.inject_into_urllib3()
|
|
except ImportError:
|
|
pass
|
|
|
|
Now you can use :mod:`urllib3` as you normally would, and it will support SNI
|
|
when the required modules are installed.
|
|
|
|
Activating this module also has the positive side effect of disabling SSL/TLS
|
|
encryption in Python 2 (see `CRIME attack`_).
|
|
|
|
If you want to configure the default list of supported cipher suites, you can
|
|
set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.
|
|
|
|
Module Variables
|
|
----------------
|
|
|
|
:var DEFAULT_SSL_CIPHER_LIST: The list of supported SSL/TLS cipher suites.
|
|
Default: ``EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA256
|
|
EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EDH+aRSA EECDH RC4 !aNULL !eNULL !LOW !3DES
|
|
!MD5 !EXP !PSK !SRP !DSS'``
|
|
|
|
.. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication
|
|
.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit)
|
|
|
|
'''
|
|
|
|
from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
|
|
from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName
|
|
import OpenSSL.SSL
|
|
from pyasn1.codec.der import decoder as der_decoder
|
|
from pyasn1.type import univ, constraint
|
|
from socket import _fileobject
|
|
import ssl
|
|
import select
|
|
from cStringIO import StringIO
|
|
|
|
from .. import connection
|
|
from .. import util
|
|
|
|
__all__ = ['inject_into_urllib3', 'extract_from_urllib3']
|
|
|
|
# SNI only *really* works if we can read the subjectAltName of certificates.
|
|
HAS_SNI = SUBJ_ALT_NAME_SUPPORT
|
|
|
|
# Map from urllib3 to PyOpenSSL compatible parameter-values.
|
|
_openssl_versions = {
|
|
ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD,
|
|
ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD,
|
|
ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
|
|
}
|
|
_openssl_verify = {
|
|
ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
|
|
ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
|
|
ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
|
|
+ OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
|
|
}
|
|
|
|
# Default SSL/TLS cipher list.
|
|
# Recommendation by https://community.qualys.com/blogs/securitylabs/2013/08/05/
|
|
# configuring-apache-nginx-and-openssl-for-forward-secrecy
|
|
DEFAULT_SSL_CIPHER_LIST = 'EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM ' + \
|
|
'EECDH+ECDSA+SHA256 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EDH+aRSA ' + \
|
|
'EECDH RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS'
|
|
|
|
|
|
orig_util_HAS_SNI = util.HAS_SNI
|
|
orig_connection_ssl_wrap_socket = connection.ssl_wrap_socket
|
|
|
|
|
|
def inject_into_urllib3():
|
|
'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.'
|
|
|
|
connection.ssl_wrap_socket = ssl_wrap_socket
|
|
util.HAS_SNI = HAS_SNI
|
|
|
|
|
|
def extract_from_urllib3():
|
|
'Undo monkey-patching by :func:`inject_into_urllib3`.'
|
|
|
|
connection.ssl_wrap_socket = orig_connection_ssl_wrap_socket
|
|
util.HAS_SNI = orig_util_HAS_SNI
|
|
|
|
|
|
### Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
|
|
class SubjectAltName(BaseSubjectAltName):
|
|
'''ASN.1 implementation for subjectAltNames support'''
|
|
|
|
# There is no limit to how many SAN certificates a certificate may have,
|
|
# however this needs to have some limit so we'll set an arbitrarily high
|
|
# limit.
|
|
sizeSpec = univ.SequenceOf.sizeSpec + \
|
|
constraint.ValueSizeConstraint(1, 1024)
|
|
|
|
|
|
### Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
|
|
def get_subj_alt_name(peer_cert):
|
|
# Search through extensions
|
|
dns_name = []
|
|
if not SUBJ_ALT_NAME_SUPPORT:
|
|
return dns_name
|
|
|
|
general_names = SubjectAltName()
|
|
for i in range(peer_cert.get_extension_count()):
|
|
ext = peer_cert.get_extension(i)
|
|
ext_name = ext.get_short_name()
|
|
if ext_name != 'subjectAltName':
|
|
continue
|
|
|
|
# PyOpenSSL returns extension data in ASN.1 encoded form
|
|
ext_dat = ext.get_data()
|
|
decoded_dat = der_decoder.decode(ext_dat,
|
|
asn1Spec=general_names)
|
|
|
|
for name in decoded_dat:
|
|
if not isinstance(name, SubjectAltName):
|
|
continue
|
|
for entry in range(len(name)):
|
|
component = name.getComponentByPosition(entry)
|
|
if component.getName() != 'dNSName':
|
|
continue
|
|
dns_name.append(str(component.getComponent()))
|
|
|
|
return dns_name
|
|
|
|
|
|
class fileobject(_fileobject):
|
|
|
|
def read(self, size=-1):
|
|
# Use max, disallow tiny reads in a loop as they are very inefficient.
|
|
# We never leave read() with any leftover data from a new recv() call
|
|
# in our internal buffer.
|
|
rbufsize = max(self._rbufsize, self.default_bufsize)
|
|
# Our use of StringIO rather than lists of string objects returned by
|
|
# recv() minimizes memory usage and fragmentation that occurs when
|
|
# rbufsize is large compared to the typical return value of recv().
|
|
buf = self._rbuf
|
|
buf.seek(0, 2) # seek end
|
|
if size < 0:
|
|
# Read until EOF
|
|
self._rbuf = StringIO() # reset _rbuf. we consume it via buf.
|
|
while True:
|
|
try:
|
|
data = self._sock.recv(rbufsize)
|
|
except OpenSSL.SSL.WantReadError:
|
|
continue
|
|
if not data:
|
|
break
|
|
buf.write(data)
|
|
return buf.getvalue()
|
|
else:
|
|
# Read until size bytes or EOF seen, whichever comes first
|
|
buf_len = buf.tell()
|
|
if buf_len >= size:
|
|
# Already have size bytes in our buffer? Extract and return.
|
|
buf.seek(0)
|
|
rv = buf.read(size)
|
|
self._rbuf = StringIO()
|
|
self._rbuf.write(buf.read())
|
|
return rv
|
|
|
|
self._rbuf = StringIO() # reset _rbuf. we consume it via buf.
|
|
while True:
|
|
left = size - buf_len
|
|
# recv() will malloc the amount of memory given as its
|
|
# parameter even though it often returns much less data
|
|
# than that. The returned data string is short lived
|
|
# as we copy it into a StringIO and free it. This avoids
|
|
# fragmentation issues on many platforms.
|
|
try:
|
|
data = self._sock.recv(left)
|
|
except OpenSSL.SSL.WantReadError:
|
|
continue
|
|
if not data:
|
|
break
|
|
n = len(data)
|
|
if n == size and not buf_len:
|
|
# Shortcut. Avoid buffer data copies when:
|
|
# - We have no data in our buffer.
|
|
# AND
|
|
# - Our call to recv returned exactly the
|
|
# number of bytes we were asked to read.
|
|
return data
|
|
if n == left:
|
|
buf.write(data)
|
|
del data # explicit free
|
|
break
|
|
assert n <= left, "recv(%d) returned %d bytes" % (left, n)
|
|
buf.write(data)
|
|
buf_len += n
|
|
del data # explicit free
|
|
#assert buf_len == buf.tell()
|
|
return buf.getvalue()
|
|
|
|
def readline(self, size=-1):
|
|
buf = self._rbuf
|
|
buf.seek(0, 2) # seek end
|
|
if buf.tell() > 0:
|
|
# check if we already have it in our buffer
|
|
buf.seek(0)
|
|
bline = buf.readline(size)
|
|
if bline.endswith('\n') or len(bline) == size:
|
|
self._rbuf = StringIO()
|
|
self._rbuf.write(buf.read())
|
|
return bline
|
|
del bline
|
|
if size < 0:
|
|
# Read until \n or EOF, whichever comes first
|
|
if self._rbufsize <= 1:
|
|
# Speed up unbuffered case
|
|
buf.seek(0)
|
|
buffers = [buf.read()]
|
|
self._rbuf = StringIO() # reset _rbuf. we consume it via buf.
|
|
data = None
|
|
recv = self._sock.recv
|
|
while True:
|
|
try:
|
|
while data != "\n":
|
|
data = recv(1)
|
|
if not data:
|
|
break
|
|
buffers.append(data)
|
|
except OpenSSL.SSL.WantReadError:
|
|
continue
|
|
break
|
|
return "".join(buffers)
|
|
|
|
buf.seek(0, 2) # seek end
|
|
self._rbuf = StringIO() # reset _rbuf. we consume it via buf.
|
|
while True:
|
|
try:
|
|
data = self._sock.recv(self._rbufsize)
|
|
except OpenSSL.SSL.WantReadError:
|
|
continue
|
|
if not data:
|
|
break
|
|
nl = data.find('\n')
|
|
if nl >= 0:
|
|
nl += 1
|
|
buf.write(data[:nl])
|
|
self._rbuf.write(data[nl:])
|
|
del data
|
|
break
|
|
buf.write(data)
|
|
return buf.getvalue()
|
|
else:
|
|
# Read until size bytes or \n or EOF seen, whichever comes first
|
|
buf.seek(0, 2) # seek end
|
|
buf_len = buf.tell()
|
|
if buf_len >= size:
|
|
buf.seek(0)
|
|
rv = buf.read(size)
|
|
self._rbuf = StringIO()
|
|
self._rbuf.write(buf.read())
|
|
return rv
|
|
self._rbuf = StringIO() # reset _rbuf. we consume it via buf.
|
|
while True:
|
|
try:
|
|
data = self._sock.recv(self._rbufsize)
|
|
except OpenSSL.SSL.WantReadError:
|
|
continue
|
|
if not data:
|
|
break
|
|
left = size - buf_len
|
|
# did we just receive a newline?
|
|
nl = data.find('\n', 0, left)
|
|
if nl >= 0:
|
|
nl += 1
|
|
# save the excess data to _rbuf
|
|
self._rbuf.write(data[nl:])
|
|
if buf_len:
|
|
buf.write(data[:nl])
|
|
break
|
|
else:
|
|
# Shortcut. Avoid data copy through buf when returning
|
|
# a substring of our first recv().
|
|
return data[:nl]
|
|
n = len(data)
|
|
if n == size and not buf_len:
|
|
# Shortcut. Avoid data copy through buf when
|
|
# returning exactly all of our first recv().
|
|
return data
|
|
if n >= left:
|
|
buf.write(data[:left])
|
|
self._rbuf.write(data[left:])
|
|
break
|
|
buf.write(data)
|
|
buf_len += n
|
|
#assert buf_len == buf.tell()
|
|
return buf.getvalue()
|
|
|
|
|
|
class WrappedSocket(object):
|
|
'''API-compatibility wrapper for Python OpenSSL's Connection-class.'''
|
|
|
|
def __init__(self, connection, socket):
|
|
self.connection = connection
|
|
self.socket = socket
|
|
|
|
def fileno(self):
|
|
return self.socket.fileno()
|
|
|
|
def makefile(self, mode, bufsize=-1):
|
|
return fileobject(self.connection, mode, bufsize)
|
|
|
|
def settimeout(self, timeout):
|
|
return self.socket.settimeout(timeout)
|
|
|
|
def sendall(self, data):
|
|
return self.connection.sendall(data)
|
|
|
|
def close(self):
|
|
return self.connection.shutdown()
|
|
|
|
def getpeercert(self, binary_form=False):
|
|
x509 = self.connection.get_peer_certificate()
|
|
|
|
if not x509:
|
|
return x509
|
|
|
|
if binary_form:
|
|
return OpenSSL.crypto.dump_certificate(
|
|
OpenSSL.crypto.FILETYPE_ASN1,
|
|
x509)
|
|
|
|
return {
|
|
'subject': (
|
|
(('commonName', x509.get_subject().CN),),
|
|
),
|
|
'subjectAltName': [
|
|
('DNS', value)
|
|
for value in get_subj_alt_name(x509)
|
|
]
|
|
}
|
|
|
|
|
|
def _verify_callback(cnx, x509, err_no, err_depth, return_code):
|
|
return err_no == 0
|
|
|
|
|
|
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
|
|
ca_certs=None, server_hostname=None,
|
|
ssl_version=None):
|
|
ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version])
|
|
if certfile:
|
|
ctx.use_certificate_file(certfile)
|
|
if keyfile:
|
|
ctx.use_privatekey_file(keyfile)
|
|
if cert_reqs != ssl.CERT_NONE:
|
|
ctx.set_verify(_openssl_verify[cert_reqs], _verify_callback)
|
|
if ca_certs:
|
|
try:
|
|
ctx.load_verify_locations(ca_certs, None)
|
|
except OpenSSL.SSL.Error as e:
|
|
raise ssl.SSLError('bad ca_certs: %r' % ca_certs, e)
|
|
|
|
# Disable TLS compression to migitate CRIME attack (issue #309)
|
|
OP_NO_COMPRESSION = 0x20000
|
|
ctx.set_options(OP_NO_COMPRESSION)
|
|
|
|
# Set list of supported ciphersuites.
|
|
ctx.set_cipher_list(DEFAULT_SSL_CIPHER_LIST)
|
|
|
|
cnx = OpenSSL.SSL.Connection(ctx, sock)
|
|
cnx.set_tlsext_host_name(server_hostname)
|
|
cnx.set_connect_state()
|
|
while True:
|
|
try:
|
|
cnx.do_handshake()
|
|
except OpenSSL.SSL.WantReadError:
|
|
select.select([sock], [], [])
|
|
continue
|
|
except OpenSSL.SSL.Error as e:
|
|
raise ssl.SSLError('bad handshake', e)
|
|
break
|
|
|
|
return WrappedSocket(cnx, sock)
|