From 4b039b0313b6f0ea29f03466abf1542910c62af3 Mon Sep 17 00:00:00 2001
From: echel0n <sickrage.tv@gmail.com>
Date: Fri, 19 Dec 2014 21:57:11 -0800
Subject: [PATCH] Added new webproxy per-provider option to allow accessing
 blocked content from different countries that may have restrictions, this
 feature is beta.

Performed misc PEP8 code cleanups.

Fixed sickragetv/sickrage-issues#213

Fixed sickragetv/sickrage-issues#217
---
 lib/feedcache/cache.py               |  7 ++-
 lib/guessit/__init__.py              | 29 ++++++++-----
 sickbeard/helpers.py                 | 13 ++----
 sickbeard/providers/__init__.py      |  1 +
 sickbeard/providers/animezb.py       |  8 ++--
 sickbeard/providers/bitsoup.py       | 20 ++++-----
 sickbeard/providers/btn.py           |  5 ++-
 sickbeard/providers/ezrss.py         |  7 +--
 sickbeard/providers/fanzub.py        |  4 +-
 sickbeard/providers/freshontv.py     | 32 +++++++-------
 sickbeard/providers/generic.py       | 55 ++++++++++++++++++++---
 sickbeard/providers/hdbits.py        | 65 +++++++++++++---------------
 sickbeard/providers/hdtorrents.py    | 18 ++++----
 sickbeard/providers/iptorrents.py    | 10 ++---
 sickbeard/providers/kat.py           | 30 +++++--------
 sickbeard/providers/newznab.py       | 17 ++------
 sickbeard/providers/nextgen.py       | 15 ++++---
 sickbeard/providers/nyaatorrents.py  |  4 +-
 sickbeard/providers/omgwtfnzbs.py    |  8 +++-
 sickbeard/providers/rsstorrent.py    | 15 +++----
 sickbeard/providers/scc.py           | 19 ++++----
 sickbeard/providers/speedcd.py       | 13 +++---
 sickbeard/providers/t411.py          | 12 ++---
 sickbeard/providers/thepiratebay.py  | 65 +++-------------------------
 sickbeard/providers/tokyotoshokan.py |  3 +-
 sickbeard/providers/torrentbytes.py  | 13 +++---
 sickbeard/providers/torrentday.py    | 11 ++---
 sickbeard/providers/torrentleech.py  | 19 ++++----
 sickbeard/providers/tvtorrents.py    |  3 +-
 sickbeard/providers/womble.py        |  3 +-
 sickbeard/rssfeeds.py                |  4 +-
 sickbeard/tvcache.py                 |  6 +--
 sickbeard/webapi.py                  |  4 +-
 33 files changed, 267 insertions(+), 271 deletions(-)

diff --git a/lib/feedcache/cache.py b/lib/feedcache/cache.py
index 6ec0a39a..6a3ad53e 100644
--- a/lib/feedcache/cache.py
+++ b/lib/feedcache/cache.py
@@ -98,7 +98,7 @@ class Cache:
                     del self.storage[url]
         return
 
-    def fetch(self, url, force_update=False, offline=False, request_headers=None):
+    def fetch(self, url, force_update=False, offline=False, request_headers=None, referrer=None):
         """Return the feed at url.
 
         url - The URL of the feed.
@@ -112,6 +112,10 @@ class Cache:
                                  cache and never access the remote
                                  URL.
 
+        request_headers=None - Add addition request headers to request
+
+        referrer=None - Added a referrer to request
+
         If there is data for that feed in the cache already, check
         the expiration date before accessing the server.  If the
         cached data has not expired, return it without accessing the
@@ -175,6 +179,7 @@ class Cache:
                                          agent=self.user_agent,
                                          modified=modified,
                                          etag=etag,
+                                         referrer=referrer,
                                          request_headers=request_headers)
 
         status = parsed_result.get('status', None)
diff --git a/lib/guessit/__init__.py b/lib/guessit/__init__.py
index 86d51212..30478c4a 100644
--- a/lib/guessit/__init__.py
+++ b/lib/guessit/__init__.py
@@ -30,47 +30,56 @@ __all__ = ['Guess', 'Language',
 # it will then always be available
 # with code from http://lucumr.pocoo.org/2011/1/22/forwards-compatible-python/
 import sys
+
 if sys.version_info[0] >= 3:
     PY3 = True
     unicode_text_type = str
     native_text_type = str
     base_text_type = str
+
     def u(x):
         return str(x)
+
     def s(x):
         return x
+
     class UnicodeMixin(object):
         __str__ = lambda x: x.__unicode__()
+
     import binascii
+
     def to_hex(x):
         return binascii.hexlify(x).decode('utf-8')
 
 else:
     PY3 = False
-    __all__ = [ str(s) for s in __all__ ] # fix imports for python2
+    __all__ = [str(s) for s in __all__]  # fix imports for python2
     unicode_text_type = unicode
     native_text_type = str
     base_text_type = basestring
+
     def u(x):
         if isinstance(x, str):
             return x.decode('utf-8')
         return unicode(x)
+
     def s(x):
         if isinstance(x, unicode):
             return x.encode('utf-8')
         if isinstance(x, list):
-            return [ s(y) for y in x ]
+            return [s(y) for y in x]
         if isinstance(x, tuple):
             return tuple(s(y) for y in x)
         if isinstance(x, dict):
             return dict((s(key), s(value)) for key, value in x.items())
         return x
+
     class UnicodeMixin(object):
         __str__ = lambda x: unicode(x).encode('utf-8')
+
     def to_hex(x):
         return x.encode('hex')
 
-
 from guessit.guess import Guess, merge_all
 from guessit.language import Language
 from guessit.matcher import IterativeMatcher
@@ -80,7 +89,6 @@ import logging
 log = logging.getLogger(__name__)
 
 
-
 class NullHandler(logging.Handler):
     def emit(self, record):
         pass
@@ -112,7 +120,6 @@ def _guess_filename(filename, filetype):
         mtree = IterativeMatcher(filename, filetype=filetype,
                                  opts=['skip_first_year'])
 
-
     m = mtree.matched()
 
     if 'language' not in m and 'subtitleLanguage' not in m:
@@ -123,7 +130,6 @@ def _guess_filename(filename, filetype):
                               opts=['nolanguage', 'nocountry'])
     m2 = mtree2.matched()
 
-
     if m.get('title') is None:
         return m
 
@@ -156,9 +162,9 @@ def _guess_filename(filename, filetype):
         # if filetype is subtitle and the language appears last, just before
         # the extension, then it is likely a subtitle language
         parts = clean_string(title.root.value).split()
-        if (m['type'] in ['moviesubtitle', 'episodesubtitle'] and
-            parts.index(lang.value) == len(parts) - 2):
-            return m
+        if (m['type'] in ['moviesubtitle', 'episodesubtitle']):
+            if lang.value in parts and (parts.index(lang.value) == len(parts) - 2):
+                return m
 
         # if the language was in the middle of the other potential title,
         # keep the other title (eg: The Italian Job), except if it is at the
@@ -177,7 +183,6 @@ def _guess_filename(filename, filetype):
 
         return warning('Not sure of the title because of the language position')
 
-
     return m
 
 
@@ -206,6 +211,7 @@ def guess_file_info(filename, filetype, info=None):
 
         elif infotype == 'hash_mpc':
             from guessit.hash_mpc import hash_file
+
             try:
                 result.append(Guess({'hash_mpc': hash_file(filename)},
                                     confidence=1.0))
@@ -214,6 +220,7 @@ def guess_file_info(filename, filetype, info=None):
 
         elif infotype == 'hash_ed2k':
             from guessit.hash_ed2k import hash_file
+
             try:
                 result.append(Guess({'hash_ed2k': hash_file(filename)},
                                     confidence=1.0))
@@ -222,6 +229,7 @@ def guess_file_info(filename, filetype, info=None):
 
         elif infotype.startswith('hash_'):
             import hashlib
+
             hashname = infotype[5:]
             try:
                 hasher = getattr(hashlib, hashname)()
@@ -259,7 +267,6 @@ def guess_file_info(filename, filetype, info=None):
     if 'series' in result and 'country' in result:
         result['series'] += ' (%s)' % result['country'].alpha2.upper()
 
-
     return result
 
 
diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py
index ee2015d4..fce33426 100644
--- a/sickbeard/helpers.py
+++ b/sickbeard/helpers.py
@@ -1154,7 +1154,7 @@ def _getTempDir():
 
     return os.path.join(tempfile.gettempdir(), "sickrage-%s" % (uid))
 
-def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=None, json=False):
+def getURL(url, post_data=None, params=None, headers={}, timeout=30, session=None, json=False):
     """
     Returns a byte-string retrieved from the url provider.
     """
@@ -1164,10 +1164,8 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N
     session = CacheControl(sess=session, cache=caches.FileCache(os.path.join(cache_dir, 'sessions')))
 
     # request session headers
-    req_headers = {'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate'}
-    if headers:
-        req_headers.update(headers)
-    session.headers.update(req_headers)
+    session.headers.update({'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate'})
+    session.headers.update(headers)
 
     # request session ssl verify
     session.verify = False
@@ -1176,11 +1174,6 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N
     session.params = params
 
     try:
-        # Remove double-slashes from url
-        parsed = list(urlparse.urlparse(url))
-        parsed[2] = re.sub("/{2,}", "/", parsed[2])  # replace two or more / with one
-        url = urlparse.urlunparse(parsed)
-
         # request session proxies
         if sickbeard.PROXY_SETTING:
             logger.log("Using proxy for url: " + url, logger.DEBUG)
diff --git a/sickbeard/providers/__init__.py b/sickbeard/providers/__init__.py
index 72f320a3..50af6be7 100755
--- a/sickbeard/providers/__init__.py
+++ b/sickbeard/providers/__init__.py
@@ -43,6 +43,7 @@ __all__ = ['ezrss',
 
 import sickbeard
 import generic
+
 from sickbeard import logger
 from os import sys
 from random import shuffle
diff --git a/sickbeard/providers/animezb.py b/sickbeard/providers/animezb.py
index 72c435a4..075fedcd 100644
--- a/sickbeard/providers/animezb.py
+++ b/sickbeard/providers/animezb.py
@@ -27,15 +27,15 @@ from sickbeard import classes, show_name_helpers, helpers
 from sickbeard import exceptions, logger
 from sickbeard.common import *
 from sickbeard import tvcache
-from lib.dateutil.parser import parse as parseDate
-
 
 class Animezb(generic.NZBProvider):
 
     def __init__(self):
-
         generic.NZBProvider.__init__(self, "Animezb")
 
+        self.urls = {'base_url': 'https://animezb.com/'}
+        self.url = self.urls['base_url']
+
         self.supportsBacklog = False
         self.supportsAbsoluteNumbering = True
         self.anime_only = True
@@ -44,8 +44,6 @@ class Animezb(generic.NZBProvider):
 
         self.cache = AnimezbCache(self)
 
-        self.url = 'https://animezb.com/'
-
     def isEnabled(self):
         return self.enabled
 
diff --git a/sickbeard/providers/bitsoup.py b/sickbeard/providers/bitsoup.py
index 187bc965..f1dd6635 100644
--- a/sickbeard/providers/bitsoup.py
+++ b/sickbeard/providers/bitsoup.py
@@ -38,19 +38,19 @@ from unidecode import unidecode
 
 
 class BitSoupProvider(generic.TorrentProvider):
-    urls = {'base_url': 'https://www.bitsoup.me',
-            'login': 'https://www.bitsoup.me/takelogin.php',
-            'detail': 'https://www.bitsoup.me/details.php?id=%s',
-            'search': 'https://www.bitsoup.me/browse.php?search=%s%s',
-            'download': 'https://bitsoup.me/%s',
-    }
-
     def __init__(self):
-
         generic.TorrentProvider.__init__(self, "BitSoup")
 
-        self.supportsBacklog = True
+        self.urls = {'base_url': 'https://www.bitsoup.me',
+                'login': 'https://www.bitsoup.me/takelogin.php',
+                'detail': 'https://www.bitsoup.me/details.php?id=%s',
+                'search': 'https://www.bitsoup.me/browse.php?search=%s%s',
+                'download': 'https://bitsoup.me/%s',
+        }
 
+        self.url = self.urls['base_url']
+
+        self.supportsBacklog = True
         self.enabled = False
         self.username = None
         self.password = None
@@ -60,8 +60,6 @@ class BitSoupProvider(generic.TorrentProvider):
 
         self.cache = BitSoupCache(self)
 
-        self.url = self.urls['base_url']
-
         self.categories = "&c42=1&c45=1&c49=1&c7=1"
 
     def isEnabled(self):
diff --git a/sickbeard/providers/btn.py b/sickbeard/providers/btn.py
index 912f3050..23689cc2 100644
--- a/sickbeard/providers/btn.py
+++ b/sickbeard/providers/btn.py
@@ -47,7 +47,10 @@ class BTNProvider(generic.TorrentProvider):
 
         self.cache = BTNCache(self)
 
-        self.url = "http://api.btnapps.net"
+        self.urls = {'base_url': "http://api.btnapps.net"}
+
+
+        self.url = self.urls['base_url']
 
     def isEnabled(self):
         return self.enabled
diff --git a/sickbeard/providers/ezrss.py b/sickbeard/providers/ezrss.py
index 38a2d56c..d466d682 100644
--- a/sickbeard/providers/ezrss.py
+++ b/sickbeard/providers/ezrss.py
@@ -36,17 +36,18 @@ from sickbeard import helpers
 class EZRSSProvider(generic.TorrentProvider):
     def __init__(self):
 
+        self.urls = {'base_url': 'https://www.ezrss.it/'}
+
+        self.url = self.urls['base_url']
+
         generic.TorrentProvider.__init__(self, "EZRSS")
 
         self.supportsBacklog = True
-
         self.enabled = False
         self.ratio = None
 
         self.cache = EZRSSCache(self)
 
-        self.url = 'https://www.ezrss.it/'
-
     def isEnabled(self):
         return self.enabled
 
diff --git a/sickbeard/providers/fanzub.py b/sickbeard/providers/fanzub.py
index b1fa3e1d..19d709cf 100644
--- a/sickbeard/providers/fanzub.py
+++ b/sickbeard/providers/fanzub.py
@@ -44,7 +44,9 @@ class Fanzub(generic.NZBProvider):
 
         self.cache = FanzubCache(self)
 
-        self.url = 'https://fanzub.com/'
+        self.urls = {'base_url': 'https://fanzub.com/'}
+
+        self.url = self.urls['base_url']
 
     def isEnabled(self):
         return self.enabled
diff --git a/sickbeard/providers/freshontv.py b/sickbeard/providers/freshontv.py
index 6c82c067..a60b7989 100755
--- a/sickbeard/providers/freshontv.py
+++ b/sickbeard/providers/freshontv.py
@@ -39,12 +39,6 @@ from sickbeard.helpers import sanitizeSceneName
 
 
 class FreshOnTVProvider(generic.TorrentProvider):
-    urls = {'base_url': 'http://freshon.tv/',
-            'login': 'http://freshon.tv/login.php?action=makelogin',
-            'detail': 'http://freshon.tv/details.php?id=%s',
-            'search': 'http://freshon.tv/browse.php?incldead=%s&words=0&cat=0&search=%s',
-            'download': 'http://freshon.tv/download.php?id=%s&type=torrent',
-    }
 
     def __init__(self):
 
@@ -64,7 +58,15 @@ class FreshOnTVProvider(generic.TorrentProvider):
 
         self.cache = FreshOnTVCache(self)
 
+        self.urls = {'base_url': 'http://freshon.tv/',
+                'login': 'http://freshon.tv/login.php?action=makelogin',
+                'detail': 'http://freshon.tv/details.php?id=%s',
+                'search': 'http://freshon.tv/browse.php?incldead=%s&words=0&cat=0&search=%s',
+                'download': 'http://freshon.tv/download.php?id=%s&type=torrent',
+                }
+
         self.url = self.urls['base_url']
+
         self.cookies = None
 
     def isEnabled(self):
@@ -90,7 +92,7 @@ class FreshOnTVProvider(generic.TorrentProvider):
             return True
 
         if self._uid and self._hash:
-           requests.utils.add_dict_to_cookiejar(self.session.cookies, self.cookies)
+            requests.utils.add_dict_to_cookiejar(self.session.cookies, self.cookies)
         else:
             login_params = {'username': self.username,
                             'password': self.password,
@@ -107,18 +109,18 @@ class FreshOnTVProvider(generic.TorrentProvider):
                 return False
 
             if re.search('Username does not exist in the userbase or the account is not confirmed yet.', response.text):
-               logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR)
-               return False
+                logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR)
+                return False
 
             try:
                 if requests.utils.dict_from_cookiejar(self.session.cookies)['uid'] and requests.utils.dict_from_cookiejar(self.session.cookies)['pass']:
-                        self._uid = requests.utils.dict_from_cookiejar(self.session.cookies)['uid']
-                        self._hash = requests.utils.dict_from_cookiejar(self.session.cookies)['pass']
+                    self._uid = requests.utils.dict_from_cookiejar(self.session.cookies)['uid']
+                    self._hash = requests.utils.dict_from_cookiejar(self.session.cookies)['pass']
 
-                        self.cookies = {'uid': self._uid,
-                                        'pass': self._hash
-                        }
-                        return True
+                    self.cookies = {'uid': self._uid,
+                                    'pass': self._hash
+                    }
+                    return True
             except:
                 pass
 
diff --git a/sickbeard/providers/generic.py b/sickbeard/providers/generic.py
index f6af5a01..c71bec1e 100644
--- a/sickbeard/providers/generic.py
+++ b/sickbeard/providers/generic.py
@@ -23,6 +23,7 @@ import datetime
 import os
 import re
 import itertools
+import urllib
 import sickbeard
 import requests
 
@@ -33,7 +34,6 @@ from sickbeard import encodingKludge as ek
 from sickbeard.exceptions import ex
 from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
 from sickbeard.common import Quality
-from sickbeard import clients
 
 from hachoir_parser import createParser
 from base64 import b16encode, b32decode
@@ -46,6 +46,9 @@ class GenericProvider:
         # these need to be set in the subclass
         self.providerType = None
         self.name = name
+
+        self.proxy = ProviderProxy()
+        self.urls = {}
         self.url = ''
 
         self.show = None
@@ -63,11 +66,7 @@ class GenericProvider:
 
         self.session = requests.session()
 
-        self.headers = {
-            # Using USER_AGENT instead of Mozilla to keep same user agent along authentication and download phases,
-            #otherwise session might be broken and download fail, asking again for authentication
-            #'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36'}
-            'User-Agent': USER_AGENT}
+        self.headers = {'User-Agent': USER_AGENT}
 
     def getID(self):
         return GenericProvider.makeID(self.name)
@@ -125,7 +124,8 @@ class GenericProvider:
         if not self._doLogin():
             return
 
-        return helpers.getURL(url, post_data=post_data, params=params, headers=self.headers, timeout=timeout,
+        self.headers.update({'Referer': self.proxy.getProxyURL()})
+        return helpers.getURL(self.proxy._buildURL(url), post_data=post_data, params=params, headers=self.headers, timeout=timeout,
                               session=self.session, json=json)
 
     def downloadResult(self, result):
@@ -471,3 +471,44 @@ class TorrentProvider(GenericProvider):
         GenericProvider.__init__(self, name)
 
         self.providerType = GenericProvider.TORRENT
+
+class ProviderProxy:
+    def __init__(self):
+        self.Type = 'GlypeProxy'
+        self.param = 'browse.php?u='
+        self.option = '&b=32&f=norefer'
+        self.enabled = False
+        self.url = None
+
+        self.urls = {
+            'getprivate.eu (NL)': 'http://getprivate.eu/',
+            'hideme.nl (NL)': 'http://hideme.nl/',
+            'proxite.eu (DE)': 'http://proxite.eu/',
+            'interproxy.net (EU)': 'http://interproxy.net/',
+        }
+
+    def isEnabled(self):
+        """ Return True if we Choose to call TPB via Proxy """
+        return self.enabled
+
+    def getProxyURL(self):
+        """ Return the Proxy URL Choosen via Provider Setting """
+        return str(self.url)
+
+    def _buildURL(self, url):
+        """ Return the Proxyfied URL of the page """
+        if self.isEnabled():
+            url = self.getProxyURL() + self.param + urllib.quote_plus(url) + self.option
+            logger.log(u"Proxified URL: " + url, logger.DEBUG)
+
+        return url
+
+    def _buildRE(self, regx):
+        """ Return the Proxyfied RE string """
+        if self.isEnabled():
+            regx = re.sub('//1', self.option, regx).replace('&', '&amp;')
+            logger.log(u"Proxified REGEX: " + regx, logger.DEBUG)
+        else:
+            regx = re.sub('//1', '', regx)
+
+        return regx
\ No newline at end of file
diff --git a/sickbeard/providers/hdbits.py b/sickbeard/providers/hdbits.py
index 60431a3b..55b64bca 100644
--- a/sickbeard/providers/hdbits.py
+++ b/sickbeard/providers/hdbits.py
@@ -1,35 +1,28 @@
 # This file is part of SickRage.
-#                                                                                                                                        
+#
 # SickRage is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by                                                                   
-# the Free Software Foundation, either version 3 of the License, or                                                                      
-# (at your option) any later version.                                                                                                    
-#                                                                                                                                        
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
 # SickRage is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of                                                                         
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                                                                          
-#  GNU General Public License for more details.                                                                                          
-#                                                                                                                                        
-# You should have received a copy of the GNU General Public License                                                                      
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
 # along with SickRage.  If not, see <http://www.gnu.org/licenses/>.
 
-import re
-import time
 import datetime
 import urllib
-import urlparse
-import sys
-import generic
-import sickbeard
 
-from lib import requests
-from lib.requests import exceptions
+import sickbeard
+import generic
+
 from sickbeard import classes
-from sickbeard import logger, tvcache, exceptions
-from sickbeard import helpers
-from sickbeard import clients
-from sickbeard.common import cpu_presets
-from sickbeard.exceptions import ex, AuthException
+from sickbeard import logger, tvcache
+from sickbeard.exceptions import AuthException
+
 try:
     import json
 except ImportError:
@@ -50,10 +43,13 @@ class HDBitsProvider(generic.TorrentProvider):
 
         self.cache = HDBitsCache(self)
 
-        self.url = 'https://hdbits.org'
-        self.search_url = 'https://hdbits.org/api/torrents'
-        self.rss_url = 'https://hdbits.org/api/torrents'
-        self.download_url = 'https://hdbits.org/download.php?'
+        self.urls = {'base_url': 'https://hdbits.org',
+                     'search': 'https://hdbits.org/api/torrents',
+                     'rss': 'https://hdbits.org/api/torrents',
+                     'download': 'https://hdbits.org/download.php?'
+        }
+
+        self.url = self.urls['base_url']
 
     def isEnabled(self):
         return self.enabled
@@ -91,7 +87,7 @@ class HDBitsProvider(generic.TorrentProvider):
             title = u'' + title
             title = title.replace(' ', '.')
 
-        url = self.download_url + urllib.urlencode({'id': item['id'], 'passkey': self.passkey})
+        url = self.urls['download'] + urllib.urlencode({'id': item['id'], 'passkey': self.passkey})
 
         return (title, url)
 
@@ -100,9 +96,10 @@ class HDBitsProvider(generic.TorrentProvider):
 
         self._checkAuth()
 
-        logger.log(u"Search url: " + self.search_url + " search_params: " + search_params, logger.DEBUG)
+        logger.log(u"Search url: " + self.urls['search'] + " search_params: " + search_params,
+                   logger.DEBUG)
 
-        parsedJSON = self.getURL(self.search_url, post_data=search_params, json=True)
+        parsedJSON = self.getURL(self.urls['search'], post_data=search_params, json=True)
         if not parsedJSON:
             return []
 
@@ -201,15 +198,14 @@ class HDBitsCache(tvcache.TVCache):
 
         tvcache.TVCache.__init__(self, provider)
 
-        # only poll HDBits every 15 minutes max                                                                                          
+        # only poll HDBits every 15 minutes max
         self.minTime = 15
 
     def _getRSSData(self):
         results = []
 
         try:
-            parsedJSON = self.provider.getURL(self.provider.rss_url, post_data=self.provider._make_post_data_JSON(),
-                                              json=True)
+            parsedJSON = self.provider.getURL(self.provider.urls['rss'], post_data=self.provider._make_post_data_JSON(), json=True)
 
             if self.provider._checkAuthFromData(parsedJSON):
                 results = parsedJSON['data']
@@ -218,4 +214,5 @@ class HDBitsCache(tvcache.TVCache):
 
         return {'entries': results}
 
-provider = HDBitsProvider()                                                                                                              
+
+provider = HDBitsProvider()
diff --git a/sickbeard/providers/hdtorrents.py b/sickbeard/providers/hdtorrents.py
index 405981d9..de2972d1 100644
--- a/sickbeard/providers/hdtorrents.py
+++ b/sickbeard/providers/hdtorrents.py
@@ -40,14 +40,6 @@ from sickbeard.helpers import sanitizeSceneName
 
 
 class HDTorrentsProvider(generic.TorrentProvider):
-    urls = {'base_url': 'https://hdts.ru/index.php',
-            'login': 'https://hdts.ru/login.php',
-            'detail': 'https://www.hdts.ru/details.php?id=%s',
-            'search': 'https://hdts.ru/torrents.php?search=%s&active=1&options=0%s',
-            'download': 'https://www.sceneaccess.eu/%s',
-            'home': 'https://www.hdts.ru/%s'
-    }
-
     def __init__(self):
 
         generic.TorrentProvider.__init__(self, "HDTorrents")
@@ -63,10 +55,18 @@ class HDTorrentsProvider(generic.TorrentProvider):
         self.minseed = None
         self.minleech = None
 
-        self.cache = HDTorrentsCache(self)
+        self.urls = {'base_url': 'https://hdts.ru/index.php',
+                     'login': 'https://hdts.ru/login.php',
+                     'detail': 'https://www.hdts.ru/details.php?id=%s',
+                     'search': 'https://hdts.ru/torrents.php?search=%s&active=1&options=0%s',
+                     'download': 'https://www.sceneaccess.eu/%s',
+                     'home': 'https://www.hdts.ru/%s'
+        }
 
         self.url = self.urls['base_url']
 
+        self.cache = HDTorrentsCache(self)
+
         self.categories = "&category[]=59&category[]=60&category[]=30&category[]=38"
 
         self.cookies = None
diff --git a/sickbeard/providers/iptorrents.py b/sickbeard/providers/iptorrents.py
index 09f4a45c..5ee800ca 100644
--- a/sickbeard/providers/iptorrents.py
+++ b/sickbeard/providers/iptorrents.py
@@ -40,11 +40,6 @@ from sickbeard.show_name_helpers import allPossibleShowNames
 
 
 class IPTorrentsProvider(generic.TorrentProvider):
-    urls = {'base_url': 'https://www.iptorrents.com',
-            'login': 'https://www.iptorrents.com/torrents/',
-            'search': 'https://www.iptorrents.com/torrents/?%s%s&q=%s&qf=ti',
-    }
-
     def __init__(self):
 
         generic.TorrentProvider.__init__(self, "IPTorrents")
@@ -59,6 +54,11 @@ class IPTorrentsProvider(generic.TorrentProvider):
 
         self.cache = IPTorrentsCache(self)
 
+        self.urls = {'base_url': 'https://www.iptorrents.com',
+                'login': 'https://www.iptorrents.com/torrents/',
+                'search': 'https://www.iptorrents.com/torrents/?%s%s&q=%s&qf=ti',
+        }
+
         self.url = self.urls['base_url']
 
         self.categorie = 'l73=1&l78=1&l66=1&l65=1&l79=1&l5=1&l4=1'
diff --git a/sickbeard/providers/kat.py b/sickbeard/providers/kat.py
index cf21d5b6..6fe91f6e 100644
--- a/sickbeard/providers/kat.py
+++ b/sickbeard/providers/kat.py
@@ -19,16 +19,15 @@
 
 from __future__ import with_statement
 
-import sys
 import os
 import traceback
 import urllib
 import re
 import datetime
-import urlparse
 
 import sickbeard
 import generic
+
 from sickbeard.common import Quality
 from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
 from sickbeard import logger
@@ -37,12 +36,7 @@ from sickbeard import helpers
 from sickbeard import db
 from sickbeard import classes
 from sickbeard.show_name_helpers import allPossibleShowNames, sanitizeSceneName
-from sickbeard.exceptions import ex
-from sickbeard import encodingKludge as ek
-from sickbeard import clients
 from sickbeard.bs4_parser import BS4Parser
-from lib import requests
-from lib.requests import exceptions
 from lib.unidecode import unidecode
 
 
@@ -61,8 +55,9 @@ class KATProvider(generic.TorrentProvider):
 
         self.cache = KATCache(self)
 
-        self.urls = ['http://kickass.so/', 'http://katproxy.com/', 'http://www.kickass.to/']
-        self.url = self.urls[0]        
+        self.urls = {'base_url': 'http://kickass.so/'}
+
+        self.url = self.urls['base_url']
 
     def isEnabled(self):
         return self.enabled
@@ -229,20 +224,15 @@ class KATProvider(generic.TorrentProvider):
                 if isinstance(search_string, unicode):
                     search_string = unidecode(search_string)
 
-                entries = []
-                for url in self.urls:
-                    if mode != 'RSS':
-                        searchURL = url + 'usearch/%s/?field=seeders&sorder=desc&rss=1' % urllib.quote(search_string)
-                    else:
-                        searchURL = url + 'tv/?field=time_add&sorder=desc&rss=1'
+                if mode != 'RSS':
+                    searchURL = self.url + 'usearch/%s/?field=seeders&sorder=desc&rss=1' % urllib.quote_plus(search_string)
+                else:
+                    searchURL = self.url + 'tv/?field=time_add&sorder=desc&rss=1'
 
-                    logger.log(u"Search string: " + searchURL, logger.DEBUG)
-
-                    entries = self.cache.getRSSFeed(searchURL, items=['entries', 'feed'])['entries']
-                    if entries:
-                        break
+                logger.log(u"Search string: " + searchURL, logger.DEBUG)
 
                 try:
+                    entries = self.cache.getRSSFeed(searchURL, items=['entries', 'feed'])['entries']
                     for item in entries or []:
                         try:
                             link = item['link']
diff --git a/sickbeard/providers/newznab.py b/sickbeard/providers/newznab.py
index 8ce32b26..d0199584 100755
--- a/sickbeard/providers/newznab.py
+++ b/sickbeard/providers/newznab.py
@@ -50,7 +50,9 @@ class NewznabProvider(generic.NZBProvider):
 
         self.cache = NewznabCache(self)
 
-        self.url = url
+        self.urls = {'base_url': url}
+
+        self.url = self.urls['base_url']
 
         self.key = key
 
@@ -91,18 +93,7 @@ class NewznabProvider(generic.NZBProvider):
         return self.enabled
 
     def _getURL(self, url, post_data=None, params=None, timeout=30, json=False):
-        """
-        By default this is just a simple urlopen call but this method should be overridden
-        for providers with special URL requirements (like cookies)
-        Not really changed much from the superclass, can be used in future.
-        """
-
-        # check for auth
-        if not self._doLogin():
-            return
-
-        return helpers.getURL(url, post_data=post_data, params=params, headers=self.headers, timeout=timeout,
-                              session=self.session, json=json)
+        return self.getURL(url, post_data=post_data, params=params, timeout=timeout, json=json)
     
     def get_newznab_categories(self):
         """
diff --git a/sickbeard/providers/nextgen.py b/sickbeard/providers/nextgen.py
index 06d61bac..a8db7dc0 100644
--- a/sickbeard/providers/nextgen.py
+++ b/sickbeard/providers/nextgen.py
@@ -42,13 +42,6 @@ from sickbeard.helpers import sanitizeSceneName
 
 
 class NextGenProvider(generic.TorrentProvider):
-    urls = {'base_url': 'https://nxtgn.org/',
-            'search': 'https://nxtgn.org/browse.php?search=%s&cat=0&incldead=0&modes=%s',
-            'login_page': 'https://nxtgn.org/login.php',
-            'detail': 'https://nxtgn.org/details.php?id=%s',
-            'download': 'https://nxtgn.org/download.php?id=%s',
-            'takelogin': 'https://nxtgn.org/takelogin.php?csrf=',
-    }
 
     def __init__(self):
 
@@ -63,6 +56,14 @@ class NextGenProvider(generic.TorrentProvider):
 
         self.cache = NextGenCache(self)
 
+        self.urls = {'base_url': 'https://nxtgn.org/',
+                'search': 'https://nxtgn.org/browse.php?search=%s&cat=0&incldead=0&modes=%s',
+                'login_page': 'https://nxtgn.org/login.php',
+                'detail': 'https://nxtgn.org/details.php?id=%s',
+                'download': 'https://nxtgn.org/download.php?id=%s',
+                'takelogin': 'https://nxtgn.org/takelogin.php?csrf=',
+                }
+
         self.url = self.urls['base_url']
 
         self.categories = '&c7=1&c24=1&c17=1&c22=1&c42=1&c46=1&c26=1&c28=1&c43=1&c4=1&c31=1&c45=1&c33=1'
diff --git a/sickbeard/providers/nyaatorrents.py b/sickbeard/providers/nyaatorrents.py
index 1b10393b..cf53dce3 100644
--- a/sickbeard/providers/nyaatorrents.py
+++ b/sickbeard/providers/nyaatorrents.py
@@ -42,7 +42,9 @@ class NyaaProvider(generic.TorrentProvider):
 
         self.cache = NyaaCache(self)
 
-        self.url = 'http://www.nyaa.se/'
+        self.urls = {'base_url': 'http://www.nyaa.se/'}
+
+        self.url = self.urls['base_url']
 
     def isEnabled(self):
         return self.enabled
diff --git a/sickbeard/providers/omgwtfnzbs.py b/sickbeard/providers/omgwtfnzbs.py
index 179c498b..45d93120 100644
--- a/sickbeard/providers/omgwtfnzbs.py
+++ b/sickbeard/providers/omgwtfnzbs.py
@@ -17,8 +17,9 @@
 # along with SickRage. If not, see <http://www.gnu.org/licenses/>.
 
 import urllib
-import generic
+
 import sickbeard
+import generic
 
 from sickbeard import tvcache
 from sickbeard import helpers
@@ -46,7 +47,10 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
         self.username = None
         self.api_key = None
         self.cache = OmgwtfnzbsCache(self)
-        self.url = 'https://omgwtfnzbs.org/'
+
+        self.urls = {'base_url': 'https://omgwtfnzbs.org/'}
+        self.url = self.urls['base_url']
+
         self.supportsBacklog = True
 
     def isEnabled(self):
diff --git a/sickbeard/providers/rsstorrent.py b/sickbeard/providers/rsstorrent.py
index 5e56396c..d55f4459 100644
--- a/sickbeard/providers/rsstorrent.py
+++ b/sickbeard/providers/rsstorrent.py
@@ -18,7 +18,6 @@
 
 import os
 import re
-import urlparse
 
 import sickbeard
 import generic
@@ -27,11 +26,9 @@ from sickbeard import helpers
 from sickbeard import encodingKludge as ek
 from sickbeard import logger
 from sickbeard import tvcache
-from sickbeard import clients
 from sickbeard.exceptions import ex
 
 from lib import requests
-from lib.requests import exceptions
 from lib.bencode import bdecode
 
 
@@ -40,8 +37,11 @@ class TorrentRssProvider(generic.TorrentProvider):
                  enable_backlog=False):
         generic.TorrentProvider.__init__(self, name)
         self.cache = TorrentRssCache(self)
-        self.url = re.sub('\/$', '', url)
-        self.url = url
+
+        self.urls = {'base_url': re.sub('\/$', '', url)}
+
+        self.url = self.urls['base_url']
+
         self.enabled = True
         self.ratio = None
         self.supportsBacklog = False
@@ -162,8 +162,7 @@ class TorrentRssCache(tvcache.TVCache):
     def _getRSSData(self):
         logger.log(u"TorrentRssCache cache update URL: " + self.provider.url, logger.DEBUG)
 
-        request_headers = None
         if self.provider.cookies:
-            request_headers = {'Cookie': self.provider.cookies}
+            self.provider.headers.update({'Cookie': self.provider.cookies})
 
-        return self.getRSSFeed(self.provider.url, request_headers=request_headers, items=['entries', 'feed'])
\ No newline at end of file
+        return self.getRSSFeed(self.provider.url, items=['entries', 'feed'])
\ No newline at end of file
diff --git a/sickbeard/providers/scc.py b/sickbeard/providers/scc.py
index a713ca56..45c1092c 100644
--- a/sickbeard/providers/scc.py
+++ b/sickbeard/providers/scc.py
@@ -40,15 +40,6 @@ from sickbeard.helpers import sanitizeSceneName
 
 
 class SCCProvider(generic.TorrentProvider):
-    urls = {'base_url': 'https://sceneaccess.eu',
-            'login': 'https://sceneaccess.eu/login',
-            'detail': 'https://www.sceneaccess.eu/details?id=%s',
-            'search': 'https://sceneaccess.eu/browse?search=%s&method=1&%s',
-            'nonscene': 'https://sceneaccess.eu/nonscene?search=%s&method=1&c44=44&c45=44',
-            'foreign': 'https://sceneaccess.eu/foreign?search=%s&method=1&c34=34&c33=33',
-            'archive': 'https://sceneaccess.eu/archive?search=%s&method=1&c26=26',
-            'download': 'https://www.sceneaccess.eu/%s',
-    }
 
     def __init__(self):
 
@@ -65,6 +56,16 @@ class SCCProvider(generic.TorrentProvider):
 
         self.cache = SCCCache(self)
 
+        self.urls = {'base_url': 'https://sceneaccess.eu',
+                'login': 'https://sceneaccess.eu/login',
+                'detail': 'https://www.sceneaccess.eu/details?id=%s',
+                'search': 'https://sceneaccess.eu/browse?search=%s&method=1&%s',
+                'nonscene': 'https://sceneaccess.eu/nonscene?search=%s&method=1&c44=44&c45=44',
+                'foreign': 'https://sceneaccess.eu/foreign?search=%s&method=1&c34=34&c33=33',
+                'archive': 'https://sceneaccess.eu/archive?search=%s&method=1&c26=26',
+                'download': 'https://www.sceneaccess.eu/%s',
+                }
+
         self.url = self.urls['base_url']
 
         self.categories = "c27=27&c17=17&c11=11"
diff --git a/sickbeard/providers/speedcd.py b/sickbeard/providers/speedcd.py
index f01973bd..cbe5726d 100644
--- a/sickbeard/providers/speedcd.py
+++ b/sickbeard/providers/speedcd.py
@@ -39,12 +39,6 @@ from sickbeard.helpers import sanitizeSceneName
 
 
 class SpeedCDProvider(generic.TorrentProvider):
-    urls = {'base_url': 'http://speed.cd/',
-            'login': 'http://speed.cd/take_login.php',
-            'detail': 'http://speed.cd/t/%s',
-            'search': 'http://speed.cd/V3/API/API.php',
-            'download': 'http://speed.cd/download.php?torrent=%s',
-    }
 
     def __init__(self):
 
@@ -62,6 +56,13 @@ class SpeedCDProvider(generic.TorrentProvider):
 
         self.cache = SpeedCDCache(self)
 
+        self.urls = {'base_url': 'http://speed.cd/',
+                'login': 'http://speed.cd/take_login.php',
+                'detail': 'http://speed.cd/t/%s',
+                'search': 'http://speed.cd/V3/API/API.php',
+                'download': 'http://speed.cd/download.php?torrent=%s',
+                }
+
         self.url = self.urls['base_url']
 
         self.categories = {'Season': {'c14': 1}, 'Episode': {'c2': 1, 'c49': 1}, 'RSS': {'c14': 1, 'c2': 1, 'c49': 1}}
diff --git a/sickbeard/providers/t411.py b/sickbeard/providers/t411.py
index 2b8ec250..a298432b 100644
--- a/sickbeard/providers/t411.py
+++ b/sickbeard/providers/t411.py
@@ -40,11 +40,6 @@ from sickbeard.exceptions import ex
 
 
 class T411Provider(generic.TorrentProvider):
-    urls = {'base_url': 'http://www.t411.me/',
-            'search': 'http://www.t411.me/torrents/search/?name=%s&cat=210&subcat=%s&search=%s&submit=Recherche',
-            'login_page': 'http://www.t411.me/users/login/',
-            'download': 'http://www.t411.me/torrents/download/?id=%s',
-    }
 
     def __init__(self):
         generic.TorrentProvider.__init__(self, "T411")
@@ -56,6 +51,13 @@ class T411Provider(generic.TorrentProvider):
         self.ratio = None
 
         self.cache = T411Cache(self)
+
+        self.urls = {'base_url': 'http://www.t411.me/',
+                'search': 'http://www.t411.me/torrents/search/?name=%s&cat=210&subcat=%s&search=%s&submit=Recherche',
+                'login_page': 'http://www.t411.me/users/login/',
+                'download': 'http://www.t411.me/torrents/download/?id=%s',
+                }
+
         self.url = self.urls['base_url']
 
         self.subcategories = [637, 455, 433]
diff --git a/sickbeard/providers/thepiratebay.py b/sickbeard/providers/thepiratebay.py
index b982c37b..1d5df98b 100644
--- a/sickbeard/providers/thepiratebay.py
+++ b/sickbeard/providers/thepiratebay.py
@@ -59,9 +59,9 @@ class ThePirateBayProvider(generic.TorrentProvider):
 
         self.cache = ThePirateBayCache(self)
 
-        self.proxy = ThePirateBayWebproxy()
+        self.urls = {'base_url': 'https://oldpiratebay.org/'}
 
-        self.url = 'https://oldpiratebay.org/'
+        self.url = self.urls['base_url']
 
         self.searchurl = self.url + 'search.php?q=%s&Torrent_sort=seeders.desc' # order by seed
 
@@ -114,11 +114,7 @@ class ThePirateBayProvider(generic.TorrentProvider):
 
         fileName = None
 
-        fileURL = self.proxy._buildURL(self.url + 'ajax_details_filelist.php?id=' + str(torrent_id))
-
-        if self.proxy and self.proxy.isEnabled():
-            self.headers.update({'referer': self.proxy.getProxyURL()})
-
+        fileURL = self.url + 'ajax_details_filelist.php?id=' + str(torrent_id)
         data = self.getURL(fileURL)
         if not data:
             return None
@@ -225,18 +221,15 @@ class ThePirateBayProvider(generic.TorrentProvider):
         results = []
         items = {'Season': [], 'Episode': [], 'RSS': []}
 
-        if self.proxy and self.proxy.isEnabled():
-            self.headers.update({'referer': self.proxy.getProxyURL()})
-
         for mode in search_params.keys():
             for search_string in search_params[mode]:
                 if isinstance(search_string, unicode):
                     search_string = unidecode(search_string)
 
                 if mode != 'RSS':
-                    searchURL = self.proxy._buildURL(self.searchurl % (urllib.quote(search_string)))
+                    searchURL = self.searchurl % (urllib.quote(search_string))
                 else:
-                    searchURL = self.proxy._buildURL(self.url + 'tv/latest/')
+                    searchURL = self.url + 'tv/latest/'
 
                 logger.log(u"Search string: " + searchURL, logger.DEBUG)
 
@@ -245,9 +238,8 @@ class ThePirateBayProvider(generic.TorrentProvider):
                     continue
 
                 re_title_url = self.proxy._buildRE(self.re_title_url)
-
-                #Extracting torrent information from data returned by searchURL                   
                 match = re.compile(re_title_url, re.DOTALL).finditer(urllib.unquote(data))
+
                 for torrent in match:
 
                     title = torrent.group('title').replace('_',
@@ -344,49 +336,4 @@ class ThePirateBayCache(tvcache.TVCache):
         search_params = {'RSS': ['rss']}
         return {'entries': self.provider._doSearch(search_params)}
 
-class ThePirateBayWebproxy:
-    def __init__(self):
-        self.Type = 'GlypeProxy'
-        self.param = 'browse.php?u='
-        self.option = '&b=32'
-        self.enabled = False
-        self.url = None
-
-        self.urls = {
-            'Getprivate.eu (NL)': 'http://getprivate.eu/',
-            '15bb51.info (US)': 'http://15bb51.info/',
-            'Hideme.nl (NL)': 'http://hideme.nl/',
-            'Proxite.eu (DE)': 'http://proxite.eu/',
-            'Webproxy.cz (CZ)': 'http://webproxy.cz/',
-            '2me2u (CZ)': 'http://2me2u.me/',
-            'Interproxy.net (EU)': 'http://interproxy.net/',
-            'Unblockersurf.info (DK)': 'http://unblockersurf.info/',
-            'Hiload.org (NL)': 'http://hiload.org/',
-        }
-
-    def isEnabled(self):
-        """ Return True if we Choose to call TPB via Proxy """
-        return self.enabled
-
-    def getProxyURL(self):
-        """ Return the Proxy URL Choosen via Provider Setting """
-        return str(self.url)
-
-    def _buildURL(self, url):
-        """ Return the Proxyfied URL of the page """
-        if self.isEnabled():
-            url = self.getProxyURL() + self.param + url + self.option
-
-        return url
-
-    def _buildRE(self, regx):
-        """ Return the Proxyfied RE string """
-        if self.isEnabled():
-            regx = re.sub('//1', self.option, regx).replace('&', '&amp;')
-        else:
-            regx = re.sub('//1', '', regx)
-
-        return regx
-
-
 provider = ThePirateBayProvider()
diff --git a/sickbeard/providers/tokyotoshokan.py b/sickbeard/providers/tokyotoshokan.py
index f8dde424..609916b3 100644
--- a/sickbeard/providers/tokyotoshokan.py
+++ b/sickbeard/providers/tokyotoshokan.py
@@ -44,7 +44,8 @@ class TokyoToshokanProvider(generic.TorrentProvider):
 
         self.cache = TokyoToshokanCache(self)
 
-        self.url = 'http://tokyotosho.info/'
+        self.urls = {'base_url': 'http://tokyotosho.info/'}
+        self.url = self.urls['base_url']
 
     def isEnabled(self):
         return self.enabled
diff --git a/sickbeard/providers/torrentbytes.py b/sickbeard/providers/torrentbytes.py
index 2ef871bf..71d69425 100644
--- a/sickbeard/providers/torrentbytes.py
+++ b/sickbeard/providers/torrentbytes.py
@@ -39,12 +39,6 @@ from sickbeard.helpers import sanitizeSceneName
 
 
 class TorrentBytesProvider(generic.TorrentProvider):
-    urls = {'base_url': 'https://www.torrentbytes.net',
-            'login': 'https://www.torrentbytes.net/takelogin.php',
-            'detail': 'https://www.torrentbytes.net/details.php?id=%s',
-            'search': 'https://www.torrentbytes.net/browse.php?search=%s%s',
-            'download': 'https://www.torrentbytes.net/download.php?id=%s&name=%s',
-    }
 
     def __init__(self):
 
@@ -61,6 +55,13 @@ class TorrentBytesProvider(generic.TorrentProvider):
 
         self.cache = TorrentBytesCache(self)
 
+        self.urls = {'base_url': 'https://www.torrentbytes.net',
+                'login': 'https://www.torrentbytes.net/takelogin.php',
+                'detail': 'https://www.torrentbytes.net/details.php?id=%s',
+                'search': 'https://www.torrentbytes.net/browse.php?search=%s%s',
+                'download': 'https://www.torrentbytes.net/download.php?id=%s&name=%s',
+                }
+
         self.url = self.urls['base_url']
 
         self.categories = "&c41=1&c33=1&c38=1&c32=1&c37=1"
diff --git a/sickbeard/providers/torrentday.py b/sickbeard/providers/torrentday.py
index ce9c5cf0..9b6738d3 100644
--- a/sickbeard/providers/torrentday.py
+++ b/sickbeard/providers/torrentday.py
@@ -35,11 +35,6 @@ from sickbeard.helpers import sanitizeSceneName
 
 
 class TorrentDayProvider(generic.TorrentProvider):
-    urls = {'base_url': 'http://www.td.af',
-            'login': 'http://www.td.af/torrents/',
-            'search': 'http://www.td.af/V3/API/API.php',
-            'download': 'http://www.td.af/download.php/%s/%s'
-    }
 
     def __init__(self):
 
@@ -59,6 +54,12 @@ class TorrentDayProvider(generic.TorrentProvider):
 
         self.cache = TorrentDayCache(self)
 
+        self.urls = {'base_url': 'http://www.td.af',
+                'login': 'http://www.td.af/torrents/',
+                'search': 'http://www.td.af/V3/API/API.php',
+                'download': 'http://www.td.af/download.php/%s/%s'
+        }
+
         self.url = self.urls['base_url']
 
         self.cookies = None
diff --git a/sickbeard/providers/torrentleech.py b/sickbeard/providers/torrentleech.py
index 55be7100..5f4a428d 100644
--- a/sickbeard/providers/torrentleech.py
+++ b/sickbeard/providers/torrentleech.py
@@ -20,8 +20,10 @@ import re
 import traceback
 import datetime
 import urlparse
+
 import sickbeard
 import generic
+
 from sickbeard.common import Quality, cpu_presets
 from sickbeard import logger
 from sickbeard import tvcache
@@ -40,13 +42,6 @@ from sickbeard.helpers import sanitizeSceneName
 
 
 class TorrentLeechProvider(generic.TorrentProvider):
-    urls = {'base_url': 'https://torrentleech.org/',
-            'login': 'https://torrentleech.org/user/account/login/',
-            'detail': 'https://torrentleech.org/torrent/%s',
-            'search': 'https://torrentleech.org/torrents/browse/index/query/%s/categories/%s',
-            'download': 'https://torrentleech.org%s',
-            'index': 'https://torrentleech.org/torrents/browse/index/categories/%s',
-    }
 
     def __init__(self):
 
@@ -63,6 +58,14 @@ class TorrentLeechProvider(generic.TorrentProvider):
 
         self.cache = TorrentLeechCache(self)
 
+        self.urls = {'base_url': 'https://torrentleech.org/',
+                'login': 'https://torrentleech.org/user/account/login/',
+                'detail': 'https://torrentleech.org/torrent/%s',
+                'search': 'https://torrentleech.org/torrents/browse/index/query/%s/categories/%s',
+                'download': 'https://torrentleech.org%s',
+                'index': 'https://torrentleech.org/torrents/browse/index/categories/%s',
+                }
+
         self.url = self.urls['base_url']
 
         self.categories = "2,26,27,32"
@@ -84,7 +87,7 @@ class TorrentLeechProvider(generic.TorrentProvider):
                         'password': self.password,
                         'remember_me': 'on',
                         'login': 'submit',
-        }
+                        }
 
         self.session = requests.Session()
 
diff --git a/sickbeard/providers/tvtorrents.py b/sickbeard/providers/tvtorrents.py
index ed07c43f..d8a9dbf9 100644
--- a/sickbeard/providers/tvtorrents.py
+++ b/sickbeard/providers/tvtorrents.py
@@ -45,7 +45,8 @@ class TvTorrentsProvider(generic.TorrentProvider):
 
         self.cache = TvTorrentsCache(self)
 
-        self.url = 'https://www.tvtorrents.com/'
+        self.urls = {'base_url': 'https://www.tvtorrents.com/'}
+        self.url = self.urls['base_url']
 
     def isEnabled(self):
         return self.enabled
diff --git a/sickbeard/providers/womble.py b/sickbeard/providers/womble.py
index d7200e6f..bf1d94d6 100644
--- a/sickbeard/providers/womble.py
+++ b/sickbeard/providers/womble.py
@@ -30,7 +30,8 @@ class WombleProvider(generic.NZBProvider):
         generic.NZBProvider.__init__(self, "Womble's Index")
         self.enabled = False
         self.cache = WombleCache(self)
-        self.url = 'https://newshost.co.za/'
+        self.urls = {'base_url': 'https://newshost.co.za/'}
+        self.url = self.urls['base_url']
 
     def isEnabled(self):
         return self.enabled
diff --git a/sickbeard/rssfeeds.py b/sickbeard/rssfeeds.py
index a8848627..f87650c1 100644
--- a/sickbeard/rssfeeds.py
+++ b/sickbeard/rssfeeds.py
@@ -34,7 +34,7 @@ class RSSFeeds:
         finally:
             self.rssDB.close()
 
-    def getFeed(self, url, post_data=None, request_headers=None, items=[]):
+    def getFeed(self, url, post_data=None, request_headers=None, referrer=None, items=[]):
         parsed = list(urlparse.urlparse(url))
         parsed[2] = re.sub("/{2,}", "/", parsed[2])  # replace two or more / with one
 
@@ -45,7 +45,7 @@ class RSSFeeds:
 
         try:
             fc = Cache(self.rssDB)
-            resp = fc.fetch(url, False, False, request_headers)
+            resp = fc.fetch(url, False, False, request_headers=request_headers, referrer=referrer)
 
             for item in items:
                 try:
diff --git a/sickbeard/tvcache.py b/sickbeard/tvcache.py
index e2ba8918..8ec8b92a 100644
--- a/sickbeard/tvcache.py
+++ b/sickbeard/tvcache.py
@@ -79,7 +79,6 @@ class CacheDBConnection(db.DBConnection):
 
 class TVCache():
     def __init__(self, provider):
-
         self.provider = provider
         self.providerID = self.provider.getID()
         self.providerDB = None
@@ -139,8 +138,9 @@ class TVCache():
             logger.log(u"Error while searching " + self.provider.name + ", skipping: " + ex(e), logger.ERROR)
             logger.log(traceback.format_exc(), logger.DEBUG)
 
-    def getRSSFeed(self, url, post_data=None, request_headers=None, items=[]):
-        return RSSFeeds(self.providerID).getFeed(url, post_data, request_headers, items)
+    def getRSSFeed(self, url, post_data=None, items=[]):
+        referrer = self.provider.proxy.getProxyURL()
+        return RSSFeeds(self.providerID).getFeed(self.provider.proxy._buildURL(url), post_data, self.provider.headers, referrer, items)
 
     def _translateTitle(self, title):
         return u'' + title.replace(' ', '.')
diff --git a/sickbeard/webapi.py b/sickbeard/webapi.py
index 5485b007..65088748 100644
--- a/sickbeard/webapi.py
+++ b/sickbeard/webapi.py
@@ -77,6 +77,9 @@ class ApiHandler(RequestHandler):
     def __init__(self, *args, **kwargs):
         super(ApiHandler, self).__init__(*args, **kwargs)
 
+    def set_default_headers(self):
+        self.set_header('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0')
+
     def get(self, *args, **kwargs):
         kwargs = self.request.arguments
         for arg, value in kwargs.items():
@@ -1635,7 +1638,6 @@ class CMD_SickBeardPing(ApiCall):
 
     def run(self):
         """ check to see if sickrage is running """
-        self.set_header('Cache-Control', "max-age=0,no-cache,no-store")
         if sickbeard.started:
             return _responds(RESULT_SUCCESS, {"pid": sickbeard.PID}, "Pong")
         else: