mirror of
https://github.com/moparisthebest/SickRage
synced 2024-11-14 05:15:12 -05:00
0d9fbc1ad7
This version of SickBeard uses both TVDB and TVRage to search and gather it's series data from allowing you to now have access to and download shows that you couldn't before because of being locked into only what TheTVDB had to offer. Also this edition is based off the code we used in our XEM editon so it does come with scene numbering support as well as all the other features our XEM edition has to offer. Please before using this with your existing database (sickbeard.db) please make a backup copy of it and delete any other database files such as cache.db and failed.db if present, we HIGHLY recommend starting out with no database files at all to make this a fresh start but the choice is at your own risk! Enjoy!
286 lines
10 KiB
Python
286 lines
10 KiB
Python
__all__ = [
|
|
'HTML5TreeBuilder',
|
|
]
|
|
|
|
import warnings
|
|
from bs4.builder import (
|
|
PERMISSIVE,
|
|
HTML,
|
|
HTML_5,
|
|
HTMLTreeBuilder,
|
|
)
|
|
from bs4.element import NamespacedAttribute
|
|
import html5lib
|
|
from html5lib.constants import namespaces
|
|
from bs4.element import (
|
|
Comment,
|
|
Doctype,
|
|
NavigableString,
|
|
Tag,
|
|
)
|
|
|
|
class HTML5TreeBuilder(HTMLTreeBuilder):
|
|
"""Use html5lib to build a tree."""
|
|
|
|
features = ['html5lib', PERMISSIVE, HTML_5, HTML]
|
|
|
|
def prepare_markup(self, markup, user_specified_encoding):
|
|
# Store the user-specified encoding for use later on.
|
|
self.user_specified_encoding = user_specified_encoding
|
|
yield (markup, None, None, False)
|
|
|
|
# These methods are defined by Beautiful Soup.
|
|
def feed(self, markup):
|
|
if self.soup.parse_only is not None:
|
|
warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
|
|
parser = html5lib.HTMLParser(tree=self.create_treebuilder)
|
|
doc = parser.parse(markup, encoding=self.user_specified_encoding)
|
|
|
|
# Set the character encoding detected by the tokenizer.
|
|
if isinstance(markup, unicode):
|
|
# We need to special-case this because html5lib sets
|
|
# charEncoding to UTF-8 if it gets Unicode input.
|
|
doc.original_encoding = None
|
|
else:
|
|
doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
|
|
|
|
def create_treebuilder(self, namespaceHTMLElements):
|
|
self.underlying_builder = TreeBuilderForHtml5lib(
|
|
self.soup, namespaceHTMLElements)
|
|
return self.underlying_builder
|
|
|
|
def test_fragment_to_document(self, fragment):
|
|
"""See `TreeBuilder`."""
|
|
return u'<html><head></head><body>%s</body></html>' % fragment
|
|
|
|
|
|
class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
|
|
|
|
def __init__(self, soup, namespaceHTMLElements):
|
|
self.soup = soup
|
|
super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
|
|
|
|
def documentClass(self):
|
|
self.soup.reset()
|
|
return Element(self.soup, self.soup, None)
|
|
|
|
def insertDoctype(self, token):
|
|
name = token["name"]
|
|
publicId = token["publicId"]
|
|
systemId = token["systemId"]
|
|
|
|
doctype = Doctype.for_name_and_ids(name, publicId, systemId)
|
|
self.soup.object_was_parsed(doctype)
|
|
|
|
def elementClass(self, name, namespace):
|
|
tag = self.soup.new_tag(name, namespace)
|
|
return Element(tag, self.soup, namespace)
|
|
|
|
def commentClass(self, data):
|
|
return TextNode(Comment(data), self.soup)
|
|
|
|
def fragmentClass(self):
|
|
self.soup = BeautifulSoup("")
|
|
self.soup.name = "[document_fragment]"
|
|
return Element(self.soup, self.soup, None)
|
|
|
|
def appendChild(self, node):
|
|
# XXX This code is not covered by the BS4 tests.
|
|
self.soup.append(node.element)
|
|
|
|
def getDocument(self):
|
|
return self.soup
|
|
|
|
def getFragment(self):
|
|
return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element
|
|
|
|
class AttrList(object):
|
|
def __init__(self, element):
|
|
self.element = element
|
|
self.attrs = dict(self.element.attrs)
|
|
def __iter__(self):
|
|
return list(self.attrs.items()).__iter__()
|
|
def __setitem__(self, name, value):
|
|
"set attr", name, value
|
|
self.element[name] = value
|
|
def items(self):
|
|
return list(self.attrs.items())
|
|
def keys(self):
|
|
return list(self.attrs.keys())
|
|
def __len__(self):
|
|
return len(self.attrs)
|
|
def __getitem__(self, name):
|
|
return self.attrs[name]
|
|
def __contains__(self, name):
|
|
return name in list(self.attrs.keys())
|
|
|
|
|
|
class Element(html5lib.treebuilders._base.Node):
|
|
def __init__(self, element, soup, namespace):
|
|
html5lib.treebuilders._base.Node.__init__(self, element.name)
|
|
self.element = element
|
|
self.soup = soup
|
|
self.namespace = namespace
|
|
|
|
def appendChild(self, node):
|
|
string_child = child = None
|
|
if isinstance(node, basestring):
|
|
# Some other piece of code decided to pass in a string
|
|
# instead of creating a TextElement object to contain the
|
|
# string.
|
|
string_child = child = node
|
|
elif isinstance(node, Tag):
|
|
# Some other piece of code decided to pass in a Tag
|
|
# instead of creating an Element object to contain the
|
|
# Tag.
|
|
child = node
|
|
elif node.element.__class__ == NavigableString:
|
|
string_child = child = node.element
|
|
else:
|
|
child = node.element
|
|
|
|
if not isinstance(child, basestring) and child.parent is not None:
|
|
node.element.extract()
|
|
|
|
if (string_child and self.element.contents
|
|
and self.element.contents[-1].__class__ == NavigableString):
|
|
# We are appending a string onto another string.
|
|
# TODO This has O(n^2) performance, for input like
|
|
# "a</a>a</a>a</a>..."
|
|
old_element = self.element.contents[-1]
|
|
new_element = self.soup.new_string(old_element + string_child)
|
|
old_element.replace_with(new_element)
|
|
self.soup._most_recent_element = new_element
|
|
else:
|
|
if isinstance(node, basestring):
|
|
# Create a brand new NavigableString from this string.
|
|
child = self.soup.new_string(node)
|
|
|
|
# Tell Beautiful Soup to act as if it parsed this element
|
|
# immediately after the parent's last descendant. (Or
|
|
# immediately after the parent, if it has no children.)
|
|
if self.element.contents:
|
|
most_recent_element = self.element._last_descendant(False)
|
|
else:
|
|
most_recent_element = self.element
|
|
|
|
self.soup.object_was_parsed(
|
|
child, parent=self.element,
|
|
most_recent_element=most_recent_element)
|
|
|
|
def getAttributes(self):
|
|
return AttrList(self.element)
|
|
|
|
def setAttributes(self, attributes):
|
|
if attributes is not None and len(attributes) > 0:
|
|
|
|
converted_attributes = []
|
|
for name, value in list(attributes.items()):
|
|
if isinstance(name, tuple):
|
|
new_name = NamespacedAttribute(*name)
|
|
del attributes[name]
|
|
attributes[new_name] = value
|
|
|
|
self.soup.builder._replace_cdata_list_attribute_values(
|
|
self.name, attributes)
|
|
for name, value in attributes.items():
|
|
self.element[name] = value
|
|
|
|
# The attributes may contain variables that need substitution.
|
|
# Call set_up_substitutions manually.
|
|
#
|
|
# The Tag constructor called this method when the Tag was created,
|
|
# but we just set/changed the attributes, so call it again.
|
|
self.soup.builder.set_up_substitutions(self.element)
|
|
attributes = property(getAttributes, setAttributes)
|
|
|
|
def insertText(self, data, insertBefore=None):
|
|
if insertBefore:
|
|
text = TextNode(self.soup.new_string(data), self.soup)
|
|
self.insertBefore(data, insertBefore)
|
|
else:
|
|
self.appendChild(data)
|
|
|
|
def insertBefore(self, node, refNode):
|
|
index = self.element.index(refNode.element)
|
|
if (node.element.__class__ == NavigableString and self.element.contents
|
|
and self.element.contents[index-1].__class__ == NavigableString):
|
|
# (See comments in appendChild)
|
|
old_node = self.element.contents[index-1]
|
|
new_str = self.soup.new_string(old_node + node.element)
|
|
old_node.replace_with(new_str)
|
|
else:
|
|
self.element.insert(index, node.element)
|
|
node.parent = self
|
|
|
|
def removeChild(self, node):
|
|
node.element.extract()
|
|
|
|
def reparentChildren(self, new_parent):
|
|
"""Move all of this tag's children into another tag."""
|
|
element = self.element
|
|
new_parent_element = new_parent.element
|
|
# Determine what this tag's next_element will be once all the children
|
|
# are removed.
|
|
final_next_element = element.next_sibling
|
|
|
|
new_parents_last_descendant = new_parent_element._last_descendant(False, False)
|
|
if len(new_parent_element.contents) > 0:
|
|
# The new parent already contains children. We will be
|
|
# appending this tag's children to the end.
|
|
new_parents_last_child = new_parent_element.contents[-1]
|
|
new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
|
|
else:
|
|
# The new parent contains no children.
|
|
new_parents_last_child = None
|
|
new_parents_last_descendant_next_element = new_parent_element.next_element
|
|
|
|
to_append = element.contents
|
|
append_after = new_parent.element.contents
|
|
if len(to_append) > 0:
|
|
# Set the first child's previous_element and previous_sibling
|
|
# to elements within the new parent
|
|
first_child = to_append[0]
|
|
first_child.previous_element = new_parents_last_descendant
|
|
first_child.previous_sibling = new_parents_last_child
|
|
|
|
# Fix the last child's next_element and next_sibling
|
|
last_child = to_append[-1]
|
|
last_child.next_element = new_parents_last_descendant_next_element
|
|
last_child.next_sibling = None
|
|
|
|
for child in to_append:
|
|
child.parent = new_parent_element
|
|
new_parent_element.contents.append(child)
|
|
|
|
# Now that this element has no children, change its .next_element.
|
|
element.contents = []
|
|
element.next_element = final_next_element
|
|
|
|
def cloneNode(self):
|
|
tag = self.soup.new_tag(self.element.name, self.namespace)
|
|
node = Element(tag, self.soup, self.namespace)
|
|
for key,value in self.attributes:
|
|
node.attributes[key] = value
|
|
return node
|
|
|
|
def hasContent(self):
|
|
return self.element.contents
|
|
|
|
def getNameTuple(self):
|
|
if self.namespace == None:
|
|
return namespaces["html"], self.name
|
|
else:
|
|
return self.namespace, self.name
|
|
|
|
nameTuple = property(getNameTuple)
|
|
|
|
class TextNode(Element):
|
|
def __init__(self, element, soup):
|
|
html5lib.treebuilders._base.Node.__init__(self, None)
|
|
self.element = element
|
|
self.soup = soup
|
|
|
|
def cloneNode(self):
|
|
raise NotImplementedError
|