mirror of
https://github.com/moparisthebest/SickRage
synced 2024-11-11 11:55:03 -05:00
2dcd26e69c
Fixed invalid indexer id issues for TVRage shows. Fixed issues for getting posters and backdrops for TVRage shows. We now convert XML straight to a dict object for Indexer APIs, improved overall performance api's Fixed issues with TVRage shows and displaying genre's properly.
1945 lines
79 KiB
Python
1945 lines
79 KiB
Python
"""
|
|
parser.http.movieParser module (imdb package).
|
|
|
|
This module provides the classes (and the instances), used to parse the
|
|
IMDb pages on the akas.imdb.com server about a movie.
|
|
E.g., for Brian De Palma's "The Untouchables", the referred
|
|
pages would be:
|
|
combined details: http://akas.imdb.com/title/tt0094226/combined
|
|
plot summary: http://akas.imdb.com/title/tt0094226/plotsummary
|
|
...and so on...
|
|
|
|
Copyright 2004-2013 Davide Alberani <da@erlug.linux.it>
|
|
2008 H. Turgut Uyar <uyar@tekir.org>
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 2 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program; if not, write to the Free Software
|
|
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
"""
|
|
|
|
import re
|
|
import urllib
|
|
|
|
from imdb import imdbURL_base
|
|
from imdb.Person import Person
|
|
from imdb.Movie import Movie
|
|
from imdb.Company import Company
|
|
from imdb.utils import analyze_title, split_company_name_notes, _Container
|
|
from utils import build_person, DOMParserBase, Attribute, Extractor, \
|
|
analyze_imdbid
|
|
|
|
|
|
# Dictionary used to convert some section's names.
|
|
_SECT_CONV = {
|
|
'directed': 'director',
|
|
'directed by': 'director',
|
|
'directors': 'director',
|
|
'editors': 'editor',
|
|
'writing credits': 'writer',
|
|
'writers': 'writer',
|
|
'produced': 'producer',
|
|
'cinematography': 'cinematographer',
|
|
'film editing': 'editor',
|
|
'casting': 'casting director',
|
|
'costume design': 'costume designer',
|
|
'makeup department': 'make up',
|
|
'production management': 'production manager',
|
|
'second unit director or assistant director': 'assistant director',
|
|
'costume and wardrobe department': 'costume department',
|
|
'sound department': 'sound crew',
|
|
'stunts': 'stunt performer',
|
|
'other crew': 'miscellaneous crew',
|
|
'also known as': 'akas',
|
|
'country': 'countries',
|
|
'runtime': 'runtimes',
|
|
'language': 'languages',
|
|
'certification': 'certificates',
|
|
'genre': 'genres',
|
|
'created': 'creator',
|
|
'creators': 'creator',
|
|
'color': 'color info',
|
|
'plot': 'plot outline',
|
|
'seasons': 'number of seasons',
|
|
'art directors': 'art direction',
|
|
'assistant directors': 'assistant director',
|
|
'set decorators': 'set decoration',
|
|
'visual effects department': 'visual effects',
|
|
'production managers': 'production manager',
|
|
'miscellaneous': 'miscellaneous crew',
|
|
'make up department': 'make up',
|
|
'plot summary': 'plot outline',
|
|
'cinematographers': 'cinematographer',
|
|
'camera department': 'camera and electrical department',
|
|
'costume designers': 'costume designer',
|
|
'production designers': 'production design',
|
|
'production managers': 'production manager',
|
|
'music original': 'original music',
|
|
'casting directors': 'casting director',
|
|
'other companies': 'miscellaneous companies',
|
|
'producers': 'producer',
|
|
'special effects by': 'special effects department',
|
|
'special effects': 'special effects companies'
|
|
}
|
|
|
|
|
|
def _manageRoles(mo):
|
|
"""Perform some transformation on the html, so that roleIDs can
|
|
be easily retrieved."""
|
|
firstHalf = mo.group(1)
|
|
secondHalf = mo.group(2)
|
|
newRoles = []
|
|
roles = secondHalf.split(' / ')
|
|
for role in roles:
|
|
role = role.strip()
|
|
if not role:
|
|
continue
|
|
roleID = analyze_imdbid(role)
|
|
if roleID is None:
|
|
roleID = u'/'
|
|
else:
|
|
roleID += u'/'
|
|
newRoles.append(u'<div class="_imdbpyrole" roleid="%s">%s</div>' % \
|
|
(roleID, role.strip()))
|
|
return firstHalf + u' / '.join(newRoles) + mo.group(3)
|
|
|
|
|
|
_reRolesMovie = re.compile(r'(<td class="char">)(.*?)(</td>)',
|
|
re.I | re.M | re.S)
|
|
|
|
def _replaceBR(mo):
|
|
"""Replaces <br> tags with '::' (useful for some akas)"""
|
|
txt = mo.group(0)
|
|
return txt.replace('<br>', '::')
|
|
|
|
_reAkas = re.compile(r'<h5>also known as:</h5>.*?</div>', re.I | re.M | re.S)
|
|
|
|
def makeSplitter(lstrip=None, sep='|', comments=True,
|
|
origNotesSep=' (', newNotesSep='::(', strip=None):
|
|
"""Return a splitter function suitable for a given set of data."""
|
|
def splitter(x):
|
|
if not x: return x
|
|
x = x.strip()
|
|
if not x: return x
|
|
if lstrip is not None:
|
|
x = x.lstrip(lstrip).lstrip()
|
|
lx = x.split(sep)
|
|
lx[:] = filter(None, [j.strip() for j in lx])
|
|
if comments:
|
|
lx[:] = [j.replace(origNotesSep, newNotesSep, 1) for j in lx]
|
|
if strip:
|
|
lx[:] = [j.strip(strip) for j in lx]
|
|
return lx
|
|
return splitter
|
|
|
|
|
|
def _toInt(val, replace=()):
|
|
"""Return the value, converted to integer, or None; if present, 'replace'
|
|
must be a list of tuples of values to replace."""
|
|
for before, after in replace:
|
|
val = val.replace(before, after)
|
|
try:
|
|
return int(val)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
class DOMHTMLMovieParser(DOMParserBase):
|
|
"""Parser for the "combined details" (and if instance.mdparse is
|
|
True also for the "main details") page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
mparser = DOMHTMLMovieParser()
|
|
result = mparser.parse(combined_details_html_string)
|
|
"""
|
|
_containsObjects = True
|
|
|
|
extractors = [Extractor(label='title',
|
|
path="//h1",
|
|
attrs=Attribute(key='title',
|
|
path=".//text()",
|
|
postprocess=analyze_title)),
|
|
|
|
Extractor(label='glossarysections',
|
|
group="//a[@class='glossary']",
|
|
group_key="./@name",
|
|
group_key_normalize=lambda x: x.replace('_', ' '),
|
|
path="../../../..//tr",
|
|
attrs=Attribute(key=None,
|
|
multi=True,
|
|
path={'person': ".//text()",
|
|
'link': "./td[1]/a[@href]/@href"},
|
|
postprocess=lambda x: \
|
|
build_person(x.get('person') or u'',
|
|
personID=analyze_imdbid(x.get('link')))
|
|
)),
|
|
|
|
Extractor(label='cast',
|
|
path="//table[@class='cast']//tr",
|
|
attrs=Attribute(key="cast",
|
|
multi=True,
|
|
path={'person': ".//text()",
|
|
'link': "td[2]/a/@href",
|
|
'roleID': \
|
|
"td[4]/div[@class='_imdbpyrole']/@roleid"},
|
|
postprocess=lambda x: \
|
|
build_person(x.get('person') or u'',
|
|
personID=analyze_imdbid(x.get('link')),
|
|
roleID=(x.get('roleID') or u'').split('/'))
|
|
)),
|
|
|
|
Extractor(label='genres',
|
|
path="//div[@class='info']//a[starts-with(@href," \
|
|
" '/Sections/Genres')]",
|
|
attrs=Attribute(key="genres",
|
|
multi=True,
|
|
path="./text()")),
|
|
|
|
Extractor(label='h5sections',
|
|
path="//div[@class='info']/h5/..",
|
|
attrs=[
|
|
Attribute(key="plot summary",
|
|
path="./h5[starts-with(text(), " \
|
|
"'Plot:')]/../div/text()",
|
|
postprocess=lambda x: \
|
|
x.strip().rstrip('|').rstrip()),
|
|
Attribute(key="aspect ratio",
|
|
path="./h5[starts-with(text()," \
|
|
" 'Aspect')]/../div/text()",
|
|
postprocess=lambda x: x.strip()),
|
|
Attribute(key="mpaa",
|
|
path="./h5/a[starts-with(text()," \
|
|
" 'MPAA')]/../../div/text()",
|
|
postprocess=lambda x: x.strip()),
|
|
Attribute(key="countries",
|
|
path="./h5[starts-with(text(), " \
|
|
"'Countr')]/../div[@class='info-content']//text()",
|
|
postprocess=makeSplitter('|')),
|
|
Attribute(key="language",
|
|
path="./h5[starts-with(text(), " \
|
|
"'Language')]/..//text()",
|
|
postprocess=makeSplitter('Language:')),
|
|
Attribute(key='color info',
|
|
path="./h5[starts-with(text(), " \
|
|
"'Color')]/..//text()",
|
|
postprocess=makeSplitter('Color:')),
|
|
Attribute(key='sound mix',
|
|
path="./h5[starts-with(text(), " \
|
|
"'Sound Mix')]/..//text()",
|
|
postprocess=makeSplitter('Sound Mix:')),
|
|
# Collects akas not encosed in <i> tags.
|
|
Attribute(key='other akas',
|
|
path="./h5[starts-with(text(), " \
|
|
"'Also Known As')]/../div//text()",
|
|
postprocess=makeSplitter(sep='::',
|
|
origNotesSep='" - ',
|
|
newNotesSep='::',
|
|
strip='"')),
|
|
Attribute(key='runtimes',
|
|
path="./h5[starts-with(text(), " \
|
|
"'Runtime')]/../div/text()",
|
|
postprocess=makeSplitter()),
|
|
Attribute(key='certificates',
|
|
path="./h5[starts-with(text(), " \
|
|
"'Certificat')]/..//text()",
|
|
postprocess=makeSplitter('Certification:')),
|
|
Attribute(key='number of seasons',
|
|
path="./h5[starts-with(text(), " \
|
|
"'Seasons')]/..//text()",
|
|
postprocess=lambda x: x.count('|') + 1),
|
|
Attribute(key='original air date',
|
|
path="./h5[starts-with(text(), " \
|
|
"'Original Air Date')]/../div/text()"),
|
|
Attribute(key='tv series link',
|
|
path="./h5[starts-with(text(), " \
|
|
"'TV Series')]/..//a/@href"),
|
|
Attribute(key='tv series title',
|
|
path="./h5[starts-with(text(), " \
|
|
"'TV Series')]/..//a/text()")
|
|
]),
|
|
|
|
Extractor(label='language codes',
|
|
path="//h5[starts-with(text(), 'Language')]/..//a[starts-with(@href, '/language/')]",
|
|
attrs=Attribute(key='language codes', multi=True,
|
|
path="./@href",
|
|
postprocess=lambda x: x.split('/')[2].strip()
|
|
)),
|
|
|
|
Extractor(label='country codes',
|
|
path="//h5[starts-with(text(), 'Country')]/..//a[starts-with(@href, '/country/')]",
|
|
attrs=Attribute(key='country codes', multi=True,
|
|
path="./@href",
|
|
postprocess=lambda x: x.split('/')[2].strip()
|
|
)),
|
|
|
|
Extractor(label='creator',
|
|
path="//h5[starts-with(text(), 'Creator')]/..//a",
|
|
attrs=Attribute(key='creator', multi=True,
|
|
path={'name': "./text()",
|
|
'link': "./@href"},
|
|
postprocess=lambda x: \
|
|
build_person(x.get('name') or u'',
|
|
personID=analyze_imdbid(x.get('link')))
|
|
)),
|
|
|
|
Extractor(label='thin writer',
|
|
path="//h5[starts-with(text(), 'Writer')]/..//a",
|
|
attrs=Attribute(key='thin writer', multi=True,
|
|
path={'name': "./text()",
|
|
'link': "./@href"},
|
|
postprocess=lambda x: \
|
|
build_person(x.get('name') or u'',
|
|
personID=analyze_imdbid(x.get('link')))
|
|
)),
|
|
|
|
Extractor(label='thin director',
|
|
path="//h5[starts-with(text(), 'Director')]/..//a",
|
|
attrs=Attribute(key='thin director', multi=True,
|
|
path={'name': "./text()",
|
|
'link': "@href"},
|
|
postprocess=lambda x: \
|
|
build_person(x.get('name') or u'',
|
|
personID=analyze_imdbid(x.get('link')))
|
|
)),
|
|
|
|
Extractor(label='top 250/bottom 100',
|
|
path="//div[@class='starbar-special']/" \
|
|
"a[starts-with(@href, '/chart/')]",
|
|
attrs=Attribute(key='top/bottom rank',
|
|
path="./text()")),
|
|
|
|
Extractor(label='series years',
|
|
path="//div[@id='tn15title']//span" \
|
|
"[starts-with(text(), 'TV series')]",
|
|
attrs=Attribute(key='series years',
|
|
path="./text()",
|
|
postprocess=lambda x: \
|
|
x.replace('TV series','').strip())),
|
|
|
|
Extractor(label='number of episodes',
|
|
path="//a[@title='Full Episode List']",
|
|
attrs=Attribute(key='number of episodes',
|
|
path="./text()",
|
|
postprocess=lambda x: \
|
|
_toInt(x, [(' Episodes', '')]))),
|
|
|
|
Extractor(label='akas',
|
|
path="//i[@class='transl']",
|
|
attrs=Attribute(key='akas', multi=True, path='text()',
|
|
postprocess=lambda x:
|
|
x.replace(' ', ' ').rstrip('-').replace('" - ',
|
|
'"::', 1).strip('"').replace(' ', ' '))),
|
|
|
|
Extractor(label='production notes/status',
|
|
path="//h5[starts-with(text(), 'Status:')]/..//div[@class='info-content']",
|
|
attrs=Attribute(key='production status',
|
|
path=".//text()",
|
|
postprocess=lambda x: x.strip().split('|')[0].strip().lower())),
|
|
|
|
Extractor(label='production notes/status updated',
|
|
path="//h5[starts-with(text(), 'Status Updated:')]/..//div[@class='info-content']",
|
|
attrs=Attribute(key='production status updated',
|
|
path=".//text()",
|
|
postprocess=lambda x: x.strip())),
|
|
|
|
Extractor(label='production notes/comments',
|
|
path="//h5[starts-with(text(), 'Comments:')]/..//div[@class='info-content']",
|
|
attrs=Attribute(key='production comments',
|
|
path=".//text()",
|
|
postprocess=lambda x: x.strip())),
|
|
|
|
Extractor(label='production notes/note',
|
|
path="//h5[starts-with(text(), 'Note:')]/..//div[@class='info-content']",
|
|
attrs=Attribute(key='production note',
|
|
path=".//text()",
|
|
postprocess=lambda x: x.strip())),
|
|
|
|
Extractor(label='blackcatheader',
|
|
group="//b[@class='blackcatheader']",
|
|
group_key="./text()",
|
|
group_key_normalize=lambda x: x.lower(),
|
|
path="../ul/li",
|
|
attrs=Attribute(key=None,
|
|
multi=True,
|
|
path={'name': "./a//text()",
|
|
'comp-link': "./a/@href",
|
|
'notes': "./text()"},
|
|
postprocess=lambda x: \
|
|
Company(name=x.get('name') or u'',
|
|
companyID=analyze_imdbid(x.get('comp-link')),
|
|
notes=(x.get('notes') or u'').strip())
|
|
)),
|
|
|
|
Extractor(label='rating',
|
|
path="//div[@class='starbar-meta']/b",
|
|
attrs=Attribute(key='rating',
|
|
path=".//text()")),
|
|
|
|
Extractor(label='votes',
|
|
path="//div[@class='starbar-meta']/a[@href]",
|
|
attrs=Attribute(key='votes',
|
|
path=".//text()")),
|
|
|
|
Extractor(label='cover url',
|
|
path="//a[@name='poster']",
|
|
attrs=Attribute(key='cover url',
|
|
path="./img/@src"))
|
|
]
|
|
|
|
preprocessors = [
|
|
(re.compile(r'(<b class="blackcatheader">.+?</b>)', re.I),
|
|
r'</div><div>\1'),
|
|
('<small>Full cast and crew for<br>', ''),
|
|
('<td> </td>', '<td>...</td>'),
|
|
('<span class="tv-extra">TV mini-series</span>',
|
|
'<span class="tv-extra">(mini)</span>'),
|
|
(_reRolesMovie, _manageRoles),
|
|
(_reAkas, _replaceBR)]
|
|
|
|
def preprocess_dom(self, dom):
|
|
# Handle series information.
|
|
xpath = self.xpath(dom, "//b[text()='Series Crew']")
|
|
if xpath:
|
|
b = xpath[-1] # In doubt, take the last one.
|
|
for a in self.xpath(b, "./following::h5/a[@class='glossary']"):
|
|
name = a.get('name')
|
|
if name:
|
|
a.set('name', 'series %s' % name)
|
|
# Remove links to IMDbPro.
|
|
for proLink in self.xpath(dom, "//span[@class='pro-link']"):
|
|
proLink.drop_tree()
|
|
# Remove some 'more' links (keep others, like the one around
|
|
# the number of votes).
|
|
for tn15more in self.xpath(dom,
|
|
"//a[@class='tn15more'][starts-with(@href, '/title/')]"):
|
|
tn15more.drop_tree()
|
|
return dom
|
|
|
|
re_space = re.compile(r'\s+')
|
|
re_airdate = re.compile(r'(.*)\s*\(season (\d+), episode (\d+)\)', re.I)
|
|
def postprocess_data(self, data):
|
|
# Convert section names.
|
|
for sect in data.keys():
|
|
if sect in _SECT_CONV:
|
|
data[_SECT_CONV[sect]] = data[sect]
|
|
del data[sect]
|
|
sect = _SECT_CONV[sect]
|
|
# Filter out fake values.
|
|
for key in data:
|
|
value = data[key]
|
|
if isinstance(value, list) and value:
|
|
if isinstance(value[0], Person):
|
|
data[key] = filter(lambda x: x.personID is not None, value)
|
|
if isinstance(value[0], _Container):
|
|
for obj in data[key]:
|
|
obj.accessSystem = self._as
|
|
obj.modFunct = self._modFunct
|
|
if 'akas' in data or 'other akas' in data:
|
|
akas = data.get('akas') or []
|
|
other_akas = data.get('other akas') or []
|
|
akas += other_akas
|
|
nakas = []
|
|
for aka in akas:
|
|
aka = aka.strip()
|
|
if aka.endswith('" -'):
|
|
aka = aka[:-3].rstrip()
|
|
nakas.append(aka)
|
|
if 'akas' in data:
|
|
del data['akas']
|
|
if 'other akas' in data:
|
|
del data['other akas']
|
|
if nakas:
|
|
data['akas'] = nakas
|
|
if 'runtimes' in data:
|
|
data['runtimes'] = [x.replace(' min', u'')
|
|
for x in data['runtimes']]
|
|
if 'original air date' in data:
|
|
oid = self.re_space.sub(' ', data['original air date']).strip()
|
|
data['original air date'] = oid
|
|
aid = self.re_airdate.findall(oid)
|
|
if aid and len(aid[0]) == 3:
|
|
date, season, episode = aid[0]
|
|
date = date.strip()
|
|
try: season = int(season)
|
|
except: pass
|
|
try: episode = int(episode)
|
|
except: pass
|
|
if date and date != '????':
|
|
data['original air date'] = date
|
|
else:
|
|
del data['original air date']
|
|
# Handle also "episode 0".
|
|
if season or type(season) is type(0):
|
|
data['season'] = season
|
|
if episode or type(season) is type(0):
|
|
data['episode'] = episode
|
|
for k in ('writer', 'director'):
|
|
t_k = 'thin %s' % k
|
|
if t_k not in data:
|
|
continue
|
|
if k not in data:
|
|
data[k] = data[t_k]
|
|
del data[t_k]
|
|
if 'top/bottom rank' in data:
|
|
tbVal = data['top/bottom rank'].lower()
|
|
if tbVal.startswith('top'):
|
|
tbKey = 'top 250 rank'
|
|
tbVal = _toInt(tbVal, [('top 250: #', '')])
|
|
else:
|
|
tbKey = 'bottom 100 rank'
|
|
tbVal = _toInt(tbVal, [('bottom 100: #', '')])
|
|
if tbVal:
|
|
data[tbKey] = tbVal
|
|
del data['top/bottom rank']
|
|
if 'year' in data and data['year'] == '????':
|
|
del data['year']
|
|
if 'tv series link' in data:
|
|
if 'tv series title' in data:
|
|
data['episode of'] = Movie(title=data['tv series title'],
|
|
movieID=analyze_imdbid(
|
|
data['tv series link']),
|
|
accessSystem=self._as,
|
|
modFunct=self._modFunct)
|
|
del data['tv series title']
|
|
del data['tv series link']
|
|
if 'rating' in data:
|
|
try:
|
|
data['rating'] = float(data['rating'].replace('/10', ''))
|
|
except (TypeError, ValueError):
|
|
pass
|
|
if 'votes' in data:
|
|
try:
|
|
votes = data['votes'].replace(',', '').replace('votes', '')
|
|
data['votes'] = int(votes)
|
|
except (TypeError, ValueError):
|
|
pass
|
|
return data
|
|
|
|
|
|
def _process_plotsummary(x):
|
|
"""Process a plot (contributed by Rdian06)."""
|
|
xauthor = x.get('author')
|
|
xplot = x.get('plot', u'').strip()
|
|
if xauthor:
|
|
xplot += u'::%s' % xauthor
|
|
return xplot
|
|
|
|
class DOMHTMLPlotParser(DOMParserBase):
|
|
"""Parser for the "plot summary" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a 'plot' key, containing a list
|
|
of string with the structure: 'summary::summary_author <author@email>'.
|
|
|
|
Example:
|
|
pparser = HTMLPlotParser()
|
|
result = pparser.parse(plot_summary_html_string)
|
|
"""
|
|
_defGetRefs = True
|
|
|
|
# Notice that recently IMDb started to put the email of the
|
|
# author only in the link, that we're not collecting, here.
|
|
extractors = [Extractor(label='plot',
|
|
path="//ul[@class='zebraList']//p",
|
|
attrs=Attribute(key='plot',
|
|
multi=True,
|
|
path={'plot': './text()[1]',
|
|
'author': './span/em/a/text()'},
|
|
postprocess=_process_plotsummary))]
|
|
|
|
|
|
def _process_award(x):
|
|
award = {}
|
|
_award = x.get('award')
|
|
if _award is not None:
|
|
_award = _award.strip()
|
|
award['award'] = _award
|
|
if not award['award']:
|
|
return {}
|
|
award['year'] = x.get('year').strip()
|
|
if award['year'] and award['year'].isdigit():
|
|
award['year'] = int(award['year'])
|
|
award['result'] = x.get('result').strip()
|
|
category = x.get('category').strip()
|
|
if category:
|
|
award['category'] = category
|
|
received_with = x.get('with')
|
|
if received_with is not None:
|
|
award['with'] = received_with.strip()
|
|
notes = x.get('notes')
|
|
if notes is not None:
|
|
notes = notes.strip()
|
|
if notes:
|
|
award['notes'] = notes
|
|
award['anchor'] = x.get('anchor')
|
|
return award
|
|
|
|
|
|
|
|
class DOMHTMLAwardsParser(DOMParserBase):
|
|
"""Parser for the "awards" page of a given person or movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
awparser = HTMLAwardsParser()
|
|
result = awparser.parse(awards_html_string)
|
|
"""
|
|
subject = 'title'
|
|
_containsObjects = True
|
|
|
|
extractors = [
|
|
Extractor(label='awards',
|
|
group="//table//big",
|
|
group_key="./a",
|
|
path="./ancestor::tr[1]/following-sibling::tr/" \
|
|
"td[last()][not(@colspan)]",
|
|
attrs=Attribute(key=None,
|
|
multi=True,
|
|
path={
|
|
'year': "../td[1]/a/text()",
|
|
'result': "../td[2]/b/text()",
|
|
'award': "../td[3]/text()",
|
|
'category': "./text()[1]",
|
|
# FIXME: takes only the first co-recipient
|
|
'with': "./small[starts-with(text()," \
|
|
" 'Shared with:')]/following-sibling::a[1]/text()",
|
|
'notes': "./small[last()]//text()",
|
|
'anchor': ".//text()"
|
|
},
|
|
postprocess=_process_award
|
|
)),
|
|
Extractor(label='recipients',
|
|
group="//table//big",
|
|
group_key="./a",
|
|
path="./ancestor::tr[1]/following-sibling::tr/" \
|
|
"td[last()]/small[1]/preceding-sibling::a",
|
|
attrs=Attribute(key=None,
|
|
multi=True,
|
|
path={
|
|
'name': "./text()",
|
|
'link': "./@href",
|
|
'anchor': "..//text()"
|
|
}
|
|
))
|
|
]
|
|
|
|
preprocessors = [
|
|
(re.compile('(<tr><td[^>]*>.*?</td></tr>\n\n</table>)', re.I),
|
|
r'\1</table>'),
|
|
(re.compile('(<tr><td[^>]*>\n\n<big>.*?</big></td></tr>)', re.I),
|
|
r'</table><table class="_imdbpy">\1'),
|
|
(re.compile('(<table[^>]*>\n\n)</table>(<table)', re.I), r'\1\2'),
|
|
(re.compile('(<small>.*?)<br>(.*?</small)', re.I), r'\1 \2'),
|
|
(re.compile('(</tr>\n\n)(<td)', re.I), r'\1<tr>\2')
|
|
]
|
|
|
|
def preprocess_dom(self, dom):
|
|
"""Repeat td elements according to their rowspan attributes
|
|
in subsequent tr elements.
|
|
"""
|
|
cols = self.xpath(dom, "//td[@rowspan]")
|
|
for col in cols:
|
|
span = int(col.get('rowspan'))
|
|
del col.attrib['rowspan']
|
|
position = len(self.xpath(col, "./preceding-sibling::td"))
|
|
row = col.getparent()
|
|
for tr in self.xpath(row, "./following-sibling::tr")[:span-1]:
|
|
# if not cloned, child will be moved to new parent
|
|
clone = self.clone(col)
|
|
# XXX: beware that here we don't use an "adapted" function,
|
|
# because both BeautifulSoup and lxml uses the same
|
|
# "insert" method.
|
|
tr.insert(position, clone)
|
|
return dom
|
|
|
|
def postprocess_data(self, data):
|
|
if len(data) == 0:
|
|
return {}
|
|
nd = []
|
|
for key in data.keys():
|
|
dom = self.get_dom(key)
|
|
assigner = self.xpath(dom, "//a/text()")[0]
|
|
for entry in data[key]:
|
|
if not entry.has_key('name'):
|
|
if not entry:
|
|
continue
|
|
# this is an award, not a recipient
|
|
entry['assigner'] = assigner.strip()
|
|
# find the recipients
|
|
matches = [p for p in data[key]
|
|
if p.has_key('name') and (entry['anchor'] ==
|
|
p['anchor'])]
|
|
if self.subject == 'title':
|
|
recipients = [Person(name=recipient['name'],
|
|
personID=analyze_imdbid(recipient['link']))
|
|
for recipient in matches]
|
|
entry['to'] = recipients
|
|
elif self.subject == 'name':
|
|
recipients = [Movie(title=recipient['name'],
|
|
movieID=analyze_imdbid(recipient['link']))
|
|
for recipient in matches]
|
|
entry['for'] = recipients
|
|
nd.append(entry)
|
|
del entry['anchor']
|
|
return {'awards': nd}
|
|
|
|
|
|
class DOMHTMLTaglinesParser(DOMParserBase):
|
|
"""Parser for the "taglines" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
tparser = DOMHTMLTaglinesParser()
|
|
result = tparser.parse(taglines_html_string)
|
|
"""
|
|
extractors = [Extractor(label='taglines',
|
|
path='//*[contains(concat(" ", normalize-space(@class), " "), " soda ")]',
|
|
attrs=Attribute(key='taglines',
|
|
multi=True,
|
|
path="./text()"))]
|
|
|
|
def postprocess_data(self, data):
|
|
if 'taglines' in data:
|
|
data['taglines'] = [tagline.strip() for tagline in data['taglines']]
|
|
return data
|
|
|
|
|
|
class DOMHTMLKeywordsParser(DOMParserBase):
|
|
"""Parser for the "keywords" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
kwparser = DOMHTMLKeywordsParser()
|
|
result = kwparser.parse(keywords_html_string)
|
|
"""
|
|
extractors = [Extractor(label='keywords',
|
|
path="//a[starts-with(@href, '/keyword/')]",
|
|
attrs=Attribute(key='keywords',
|
|
path="./text()", multi=True,
|
|
postprocess=lambda x: \
|
|
x.lower().replace(' ', '-')))]
|
|
|
|
|
|
class DOMHTMLAlternateVersionsParser(DOMParserBase):
|
|
"""Parser for the "alternate versions" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
avparser = HTMLAlternateVersionsParser()
|
|
result = avparser.parse(alternateversions_html_string)
|
|
"""
|
|
_defGetRefs = True
|
|
extractors = [Extractor(label='alternate versions',
|
|
path="//ul[@class='trivia']/li",
|
|
attrs=Attribute(key='alternate versions',
|
|
multi=True,
|
|
path=".//text()",
|
|
postprocess=lambda x: x.strip()))]
|
|
|
|
|
|
class DOMHTMLTriviaParser(DOMParserBase):
|
|
"""Parser for the "trivia" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
avparser = HTMLAlternateVersionsParser()
|
|
result = avparser.parse(alternateversions_html_string)
|
|
"""
|
|
_defGetRefs = True
|
|
extractors = [Extractor(label='alternate versions',
|
|
path="//div[@class='sodatext']",
|
|
attrs=Attribute(key='trivia',
|
|
multi=True,
|
|
path=".//text()",
|
|
postprocess=lambda x: x.strip()))]
|
|
|
|
def preprocess_dom(self, dom):
|
|
# Remove "link this quote" links.
|
|
for qLink in self.xpath(dom, "//span[@class='linksoda']"):
|
|
qLink.drop_tree()
|
|
return dom
|
|
|
|
|
|
|
|
class DOMHTMLSoundtrackParser(DOMHTMLAlternateVersionsParser):
|
|
kind = 'soundtrack'
|
|
|
|
preprocessors = [
|
|
('<br>', '\n')
|
|
]
|
|
|
|
def postprocess_data(self, data):
|
|
if 'alternate versions' in data:
|
|
nd = []
|
|
for x in data['alternate versions']:
|
|
ds = x.split('\n')
|
|
title = ds[0]
|
|
if title[0] == '"' and title[-1] == '"':
|
|
title = title[1:-1]
|
|
nds = []
|
|
newData = {}
|
|
for l in ds[1:]:
|
|
if ' with ' in l or ' by ' in l or ' from ' in l \
|
|
or ' of ' in l or l.startswith('From '):
|
|
nds.append(l)
|
|
else:
|
|
if nds:
|
|
nds[-1] += l
|
|
else:
|
|
nds.append(l)
|
|
newData[title] = {}
|
|
for l in nds:
|
|
skip = False
|
|
for sep in ('From ',):
|
|
if l.startswith(sep):
|
|
fdix = len(sep)
|
|
kind = l[:fdix].rstrip().lower()
|
|
info = l[fdix:].lstrip()
|
|
newData[title][kind] = info
|
|
skip = True
|
|
if not skip:
|
|
for sep in ' with ', ' by ', ' from ', ' of ':
|
|
fdix = l.find(sep)
|
|
if fdix != -1:
|
|
fdix = fdix+len(sep)
|
|
kind = l[:fdix].rstrip().lower()
|
|
info = l[fdix:].lstrip()
|
|
newData[title][kind] = info
|
|
break
|
|
nd.append(newData)
|
|
data['soundtrack'] = nd
|
|
return data
|
|
|
|
|
|
class DOMHTMLCrazyCreditsParser(DOMParserBase):
|
|
"""Parser for the "crazy credits" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
ccparser = DOMHTMLCrazyCreditsParser()
|
|
result = ccparser.parse(crazycredits_html_string)
|
|
"""
|
|
_defGetRefs = True
|
|
|
|
extractors = [Extractor(label='crazy credits', path="//ul/li/tt",
|
|
attrs=Attribute(key='crazy credits', multi=True,
|
|
path=".//text()",
|
|
postprocess=lambda x: \
|
|
x.replace('\n', ' ').replace(' ', ' ')))]
|
|
|
|
|
|
def _process_goof(x):
|
|
if x['spoiler_category']:
|
|
return x['spoiler_category'].strip() + ': SPOILER: ' + x['text'].strip()
|
|
else:
|
|
return x['category'].strip() + ': ' + x['text'].strip()
|
|
|
|
|
|
class DOMHTMLGoofsParser(DOMParserBase):
|
|
"""Parser for the "goofs" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
gparser = DOMHTMLGoofsParser()
|
|
result = gparser.parse(goofs_html_string)
|
|
"""
|
|
_defGetRefs = True
|
|
|
|
extractors = [Extractor(label='goofs', path="//div[@class='soda odd']",
|
|
attrs=Attribute(key='goofs', multi=True,
|
|
path={
|
|
'text':"./text()",
|
|
'category':'./preceding-sibling::h4[1]/text()',
|
|
'spoiler_category': './h4/text()'
|
|
},
|
|
postprocess=_process_goof))]
|
|
|
|
|
|
class DOMHTMLQuotesParser(DOMParserBase):
|
|
"""Parser for the "memorable quotes" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
qparser = DOMHTMLQuotesParser()
|
|
result = qparser.parse(quotes_html_string)
|
|
"""
|
|
_defGetRefs = True
|
|
|
|
extractors = [
|
|
Extractor(label='quotes_odd',
|
|
path="//div[@class='quote soda odd']",
|
|
attrs=Attribute(key='quotes_odd',
|
|
multi=True,
|
|
path=".//text()",
|
|
postprocess=lambda x: x.strip().replace(' \n',
|
|
'::').replace('::\n', '::').replace('\n', ' '))),
|
|
Extractor(label='quotes_even',
|
|
path="//div[@class='quote soda even']",
|
|
attrs=Attribute(key='quotes_even',
|
|
multi=True,
|
|
path=".//text()",
|
|
postprocess=lambda x: x.strip().replace(' \n',
|
|
'::').replace('::\n', '::').replace('\n', ' ')))
|
|
]
|
|
|
|
preprocessors = [
|
|
(re.compile('<a href="#" class="hidesoda hidden">Hide options</a><br>', re.I), '')
|
|
]
|
|
|
|
def preprocess_dom(self, dom):
|
|
# Remove "link this quote" links.
|
|
for qLink in self.xpath(dom, "//span[@class='linksoda']"):
|
|
qLink.drop_tree()
|
|
for qLink in self.xpath(dom, "//div[@class='sharesoda_pre']"):
|
|
qLink.drop_tree()
|
|
return dom
|
|
|
|
def postprocess_data(self, data):
|
|
quotes = data.get('quotes_odd', []) + data.get('quotes_even', [])
|
|
if not quotes:
|
|
return {}
|
|
quotes = [q.split('::') for q in quotes]
|
|
return {'quotes': quotes}
|
|
|
|
|
|
class DOMHTMLReleaseinfoParser(DOMParserBase):
|
|
"""Parser for the "release dates" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
rdparser = DOMHTMLReleaseinfoParser()
|
|
result = rdparser.parse(releaseinfo_html_string)
|
|
"""
|
|
extractors = [Extractor(label='release dates',
|
|
path="//table[@id='release_dates']//tr",
|
|
attrs=Attribute(key='release dates', multi=True,
|
|
path={'country': ".//td[1]//text()",
|
|
'date': ".//td[2]//text()",
|
|
'notes': ".//td[3]//text()"})),
|
|
Extractor(label='akas',
|
|
path="//table[@id='akas']//tr",
|
|
attrs=Attribute(key='akas', multi=True,
|
|
path={'title': "./td[1]/text()",
|
|
'countries': "./td[2]/text()"}))]
|
|
|
|
preprocessors = [
|
|
(re.compile('(<h5><a name="?akas"?.*</table>)', re.I | re.M | re.S),
|
|
r'<div class="_imdbpy_akas">\1</div>')]
|
|
|
|
def postprocess_data(self, data):
|
|
if not ('release dates' in data or 'akas' in data): return data
|
|
releases = data.get('release dates') or []
|
|
rl = []
|
|
for i in releases:
|
|
country = i.get('country')
|
|
date = i.get('date')
|
|
if not (country and date): continue
|
|
country = country.strip()
|
|
date = date.strip()
|
|
if not (country and date): continue
|
|
notes = i['notes']
|
|
info = u'%s::%s' % (country, date)
|
|
if notes:
|
|
info += notes
|
|
rl.append(info)
|
|
if releases:
|
|
del data['release dates']
|
|
if rl:
|
|
data['release dates'] = rl
|
|
akas = data.get('akas') or []
|
|
nakas = []
|
|
for aka in akas:
|
|
title = (aka.get('title') or '').strip()
|
|
if not title:
|
|
continue
|
|
countries = (aka.get('countries') or '').split(',')
|
|
if not countries:
|
|
nakas.append(title)
|
|
else:
|
|
for country in countries:
|
|
nakas.append('%s::%s' % (title, country.strip()))
|
|
if akas:
|
|
del data['akas']
|
|
if nakas:
|
|
data['akas from release info'] = nakas
|
|
return data
|
|
|
|
|
|
class DOMHTMLRatingsParser(DOMParserBase):
|
|
"""Parser for the "user ratings" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
rparser = DOMHTMLRatingsParser()
|
|
result = rparser.parse(userratings_html_string)
|
|
"""
|
|
re_means = re.compile('mean\s*=\s*([0-9]\.[0-9])\.\s*median\s*=\s*([0-9])',
|
|
re.I)
|
|
extractors = [
|
|
Extractor(label='number of votes',
|
|
path="//td[b='Percentage']/../../tr",
|
|
attrs=[Attribute(key='votes',
|
|
multi=True,
|
|
path={
|
|
'votes': "td[1]//text()",
|
|
'ordinal': "td[3]//text()"
|
|
})]),
|
|
Extractor(label='mean and median',
|
|
path="//p[starts-with(text(), 'Arithmetic mean')]",
|
|
attrs=Attribute(key='mean and median',
|
|
path="text()")),
|
|
Extractor(label='rating',
|
|
path="//a[starts-with(@href, '/search/title?user_rating=')]",
|
|
attrs=Attribute(key='rating',
|
|
path="text()")),
|
|
Extractor(label='demographic voters',
|
|
path="//td[b='Average']/../../tr",
|
|
attrs=Attribute(key='demographic voters',
|
|
multi=True,
|
|
path={
|
|
'voters': "td[1]//text()",
|
|
'votes': "td[2]//text()",
|
|
'average': "td[3]//text()"
|
|
})),
|
|
Extractor(label='top 250',
|
|
path="//a[text()='top 250']",
|
|
attrs=Attribute(key='top 250',
|
|
path="./preceding-sibling::text()[1]"))
|
|
]
|
|
|
|
def postprocess_data(self, data):
|
|
nd = {}
|
|
votes = data.get('votes', [])
|
|
if votes:
|
|
nd['number of votes'] = {}
|
|
for i in xrange(1, 11):
|
|
_ordinal = int(votes[i]['ordinal'])
|
|
_strvts = votes[i]['votes'] or '0'
|
|
nd['number of votes'][_ordinal] = \
|
|
int(_strvts.replace(',', ''))
|
|
mean = data.get('mean and median', '')
|
|
if mean:
|
|
means = self.re_means.findall(mean)
|
|
if means and len(means[0]) == 2:
|
|
am, med = means[0]
|
|
try: am = float(am)
|
|
except (ValueError, OverflowError): pass
|
|
if type(am) is type(1.0):
|
|
nd['arithmetic mean'] = am
|
|
try: med = int(med)
|
|
except (ValueError, OverflowError): pass
|
|
if type(med) is type(0):
|
|
nd['median'] = med
|
|
if 'rating' in data:
|
|
nd['rating'] = float(data['rating'])
|
|
dem_voters = data.get('demographic voters')
|
|
if dem_voters:
|
|
nd['demographic'] = {}
|
|
for i in xrange(1, len(dem_voters)):
|
|
if (dem_voters[i]['votes'] is not None) \
|
|
and (dem_voters[i]['votes'].strip()):
|
|
nd['demographic'][dem_voters[i]['voters'].strip().lower()] \
|
|
= (int(dem_voters[i]['votes'].replace(',', '')),
|
|
float(dem_voters[i]['average']))
|
|
if 'imdb users' in nd.get('demographic', {}):
|
|
nd['votes'] = nd['demographic']['imdb users'][0]
|
|
nd['demographic']['all votes'] = nd['demographic']['imdb users']
|
|
del nd['demographic']['imdb users']
|
|
top250 = data.get('top 250')
|
|
if top250:
|
|
sd = top250[9:]
|
|
i = sd.find(' ')
|
|
if i != -1:
|
|
sd = sd[:i]
|
|
try: sd = int(sd)
|
|
except (ValueError, OverflowError): pass
|
|
if type(sd) is type(0):
|
|
nd['top 250 rank'] = sd
|
|
return nd
|
|
|
|
|
|
class DOMHTMLEpisodesRatings(DOMParserBase):
|
|
"""Parser for the "episode ratings ... by date" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
erparser = DOMHTMLEpisodesRatings()
|
|
result = erparser.parse(eprating_html_string)
|
|
"""
|
|
_containsObjects = True
|
|
|
|
extractors = [Extractor(label='title', path="//title",
|
|
attrs=Attribute(key='title', path="./text()")),
|
|
Extractor(label='ep ratings',
|
|
path="//th/../..//tr",
|
|
attrs=Attribute(key='episodes', multi=True,
|
|
path={'nr': ".//td[1]/text()",
|
|
'ep title': ".//td[2]//text()",
|
|
'movieID': ".//td[2]/a/@href",
|
|
'rating': ".//td[3]/text()",
|
|
'votes': ".//td[4]/text()"}))]
|
|
|
|
def postprocess_data(self, data):
|
|
if 'title' not in data or 'episodes' not in data: return {}
|
|
nd = []
|
|
title = data['title']
|
|
for i in data['episodes']:
|
|
ept = i['ep title']
|
|
movieID = analyze_imdbid(i['movieID'])
|
|
votes = i['votes']
|
|
rating = i['rating']
|
|
if not (ept and movieID and votes and rating): continue
|
|
try:
|
|
votes = int(votes.replace(',', '').replace('.', ''))
|
|
except:
|
|
pass
|
|
try:
|
|
rating = float(rating)
|
|
except:
|
|
pass
|
|
ept = ept.strip()
|
|
ept = u'%s {%s' % (title, ept)
|
|
nr = i['nr']
|
|
if nr:
|
|
ept += u' (#%s)' % nr.strip()
|
|
ept += '}'
|
|
if movieID is not None:
|
|
movieID = str(movieID)
|
|
m = Movie(title=ept, movieID=movieID, accessSystem=self._as,
|
|
modFunct=self._modFunct)
|
|
epofdict = m.get('episode of')
|
|
if epofdict is not None:
|
|
m['episode of'] = Movie(data=epofdict, accessSystem=self._as,
|
|
modFunct=self._modFunct)
|
|
nd.append({'episode': m, 'votes': votes, 'rating': rating})
|
|
return {'episodes rating': nd}
|
|
|
|
|
|
def _normalize_href(href):
|
|
if (href is not None) and (not href.lower().startswith('http://')):
|
|
if href.startswith('/'): href = href[1:]
|
|
# TODO: imdbURL_base may be set by the user!
|
|
href = '%s%s' % (imdbURL_base, href)
|
|
return href
|
|
|
|
class DOMHTMLCriticReviewsParser(DOMParserBase):
|
|
"""Parser for the "critic reviews" pages of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
osparser = DOMHTMLCriticReviewsParser()
|
|
result = osparser.parse(officialsites_html_string)
|
|
"""
|
|
kind = 'critic reviews'
|
|
|
|
extractors = [
|
|
Extractor(label='metascore',
|
|
path="//div[@class='metascore_wrap']/div/span",
|
|
attrs=Attribute(key='metascore',
|
|
path=".//text()")),
|
|
Extractor(label='metacritic url',
|
|
path="//div[@class='article']/div[@class='see-more']/a",
|
|
attrs=Attribute(key='metacritic url',
|
|
path="./@href")) ]
|
|
|
|
class DOMHTMLOfficialsitesParser(DOMParserBase):
|
|
"""Parser for the "official sites", "external reviews", "newsgroup
|
|
reviews", "miscellaneous links", "sound clips", "video clips" and
|
|
"photographs" pages of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
osparser = DOMHTMLOfficialsitesParser()
|
|
result = osparser.parse(officialsites_html_string)
|
|
"""
|
|
kind = 'official sites'
|
|
|
|
extractors = [
|
|
Extractor(label='site',
|
|
path="//ol/li/a",
|
|
attrs=Attribute(key='self.kind',
|
|
multi=True,
|
|
path={
|
|
'link': "./@href",
|
|
'info': "./text()"
|
|
},
|
|
postprocess=lambda x: (x.get('info').strip(),
|
|
urllib.unquote(_normalize_href(x.get('link'))))))
|
|
]
|
|
|
|
|
|
class DOMHTMLConnectionParser(DOMParserBase):
|
|
"""Parser for the "connections" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
connparser = DOMHTMLConnectionParser()
|
|
result = connparser.parse(connections_html_string)
|
|
"""
|
|
_containsObjects = True
|
|
|
|
extractors = [Extractor(label='connection',
|
|
group="//div[@class='_imdbpy']",
|
|
group_key="./h5/text()",
|
|
group_key_normalize=lambda x: x.lower(),
|
|
path="./a",
|
|
attrs=Attribute(key=None,
|
|
path={'title': "./text()",
|
|
'movieID': "./@href"},
|
|
multi=True))]
|
|
|
|
preprocessors = [
|
|
('<h5>', '</div><div class="_imdbpy"><h5>'),
|
|
# To get the movie's year.
|
|
('</a> (', ' ('),
|
|
('\n<br/>', '</a>'),
|
|
('<br/> - ', '::')
|
|
]
|
|
|
|
def postprocess_data(self, data):
|
|
for key in data.keys():
|
|
nl = []
|
|
for v in data[key]:
|
|
title = v['title']
|
|
ts = title.split('::', 1)
|
|
title = ts[0].strip()
|
|
notes = u''
|
|
if len(ts) == 2:
|
|
notes = ts[1].strip()
|
|
m = Movie(title=title,
|
|
movieID=analyze_imdbid(v['movieID']),
|
|
accessSystem=self._as, notes=notes,
|
|
modFunct=self._modFunct)
|
|
nl.append(m)
|
|
data[key] = nl
|
|
if not data: return {}
|
|
return {'connections': data}
|
|
|
|
|
|
class DOMHTMLLocationsParser(DOMParserBase):
|
|
"""Parser for the "locations" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
lparser = DOMHTMLLocationsParser()
|
|
result = lparser.parse(locations_html_string)
|
|
"""
|
|
extractors = [Extractor(label='locations', path="//dt",
|
|
attrs=Attribute(key='locations', multi=True,
|
|
path={'place': ".//text()",
|
|
'note': "./following-sibling::dd[1]" \
|
|
"//text()"},
|
|
postprocess=lambda x: (u'%s::%s' % (
|
|
x['place'].strip(),
|
|
(x['note'] or u'').strip())).strip(':')))]
|
|
|
|
|
|
class DOMHTMLTechParser(DOMParserBase):
|
|
"""Parser for the "technical", "business", "literature",
|
|
"publicity" (for people) and "contacts (for people) pages of
|
|
a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
tparser = HTMLTechParser()
|
|
result = tparser.parse(technical_html_string)
|
|
"""
|
|
kind = 'tech'
|
|
|
|
extractors = [Extractor(label='tech',
|
|
group="//h5",
|
|
group_key="./text()",
|
|
group_key_normalize=lambda x: x.lower(),
|
|
path="./following-sibling::div[1]",
|
|
attrs=Attribute(key=None,
|
|
path=".//text()",
|
|
postprocess=lambda x: [t.strip()
|
|
for t in x.split('\n') if t.strip()]))]
|
|
|
|
preprocessors = [
|
|
(re.compile('(<h5>.*?</h5>)', re.I), r'</div>\1<div class="_imdbpy">'),
|
|
(re.compile('((<br/>|</p>|</table>))\n?<br/>(?!<a)', re.I),
|
|
r'\1</div>'),
|
|
# the ones below are for the publicity parser
|
|
(re.compile('<p>(.*?)</p>', re.I), r'\1<br/>'),
|
|
(re.compile('(</td><td valign="top">)', re.I), r'\1::'),
|
|
(re.compile('(</tr><tr>)', re.I), r'\n\1'),
|
|
# this is for splitting individual entries
|
|
(re.compile('<br/>', re.I), r'\n'),
|
|
]
|
|
|
|
def postprocess_data(self, data):
|
|
for key in data:
|
|
data[key] = filter(None, data[key])
|
|
if self.kind in ('literature', 'business', 'contacts') and data:
|
|
if 'screenplay/teleplay' in data:
|
|
data['screenplay-teleplay'] = data['screenplay/teleplay']
|
|
del data['screenplay/teleplay']
|
|
data = {self.kind: data}
|
|
else:
|
|
if self.kind == 'publicity':
|
|
if 'biography (print)' in data:
|
|
data['biography-print'] = data['biography (print)']
|
|
del data['biography (print)']
|
|
# Tech info.
|
|
for key in data.keys():
|
|
if key.startswith('film negative format'):
|
|
data['film negative format'] = data[key]
|
|
del data[key]
|
|
elif key.startswith('film length'):
|
|
data['film length'] = data[key]
|
|
del data[key]
|
|
return data
|
|
|
|
|
|
class DOMHTMLRecParser(DOMParserBase):
|
|
"""Parser for the "recommendations" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
rparser = HTMLRecParser()
|
|
result = rparser.parse(recommendations_html_string)
|
|
"""
|
|
_containsObjects = True
|
|
|
|
extractors = [Extractor(label='recommendations',
|
|
path="//td[@valign='middle'][1]",
|
|
attrs=Attribute(key='../../tr/td[1]//text()',
|
|
multi=True,
|
|
path={'title': ".//text()",
|
|
'movieID': ".//a/@href"}))]
|
|
def postprocess_data(self, data):
|
|
for key in data.keys():
|
|
n_key = key
|
|
n_keyl = n_key.lower()
|
|
if n_keyl == 'suggested by the database':
|
|
n_key = 'database'
|
|
elif n_keyl == 'imdb users recommend':
|
|
n_key = 'users'
|
|
data[n_key] = [Movie(title=x['title'],
|
|
movieID=analyze_imdbid(x['movieID']),
|
|
accessSystem=self._as, modFunct=self._modFunct)
|
|
for x in data[key]]
|
|
del data[key]
|
|
if data: return {'recommendations': data}
|
|
return data
|
|
|
|
|
|
class DOMHTMLNewsParser(DOMParserBase):
|
|
"""Parser for the "news" page of a given movie or person.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
nwparser = DOMHTMLNewsParser()
|
|
result = nwparser.parse(news_html_string)
|
|
"""
|
|
_defGetRefs = True
|
|
|
|
extractors = [
|
|
Extractor(label='news',
|
|
path="//h2",
|
|
attrs=Attribute(key='news',
|
|
multi=True,
|
|
path={
|
|
'title': "./text()",
|
|
'fromdate': "../following-sibling::p[1]/small//text()",
|
|
# FIXME: sometimes (see The Matrix (1999)) <p> is found
|
|
# inside news text.
|
|
'body': "../following-sibling::p[2]//text()",
|
|
'link': "../..//a[text()='Permalink']/@href",
|
|
'fulllink': "../..//a[starts-with(text(), " \
|
|
"'See full article at')]/@href"
|
|
},
|
|
postprocess=lambda x: {
|
|
'title': x.get('title').strip(),
|
|
'date': x.get('fromdate').split('|')[0].strip(),
|
|
'from': x.get('fromdate').split('|')[1].replace('From ',
|
|
'').strip(),
|
|
'body': (x.get('body') or u'').strip(),
|
|
'link': _normalize_href(x.get('link')),
|
|
'full article link': _normalize_href(x.get('fulllink'))
|
|
}))
|
|
]
|
|
|
|
preprocessors = [
|
|
(re.compile('(<a name=[^>]+><h2>)', re.I), r'<div class="_imdbpy">\1'),
|
|
(re.compile('(<hr/>)', re.I), r'</div>\1'),
|
|
(re.compile('<p></p>', re.I), r'')
|
|
]
|
|
|
|
def postprocess_data(self, data):
|
|
if not data.has_key('news'):
|
|
return {}
|
|
for news in data['news']:
|
|
if news.has_key('full article link'):
|
|
if news['full article link'] is None:
|
|
del news['full article link']
|
|
return data
|
|
|
|
|
|
def _parse_review(x):
|
|
result = {}
|
|
title = x.get('title').strip()
|
|
if title[-1] == ':': title = title[:-1]
|
|
result['title'] = title
|
|
result['link'] = _normalize_href(x.get('link'))
|
|
kind = x.get('kind').strip()
|
|
if kind[-1] == ':': kind = kind[:-1]
|
|
result['review kind'] = kind
|
|
text = x.get('review').replace('\n\n', '||').replace('\n', ' ').split('||')
|
|
review = '\n'.join(text)
|
|
if x.get('author') is not None:
|
|
author = x.get('author').strip()
|
|
review = review.split(author)[0].strip()
|
|
result['review author'] = author[2:]
|
|
if x.get('item') is not None:
|
|
item = x.get('item').strip()
|
|
review = review[len(item):].strip()
|
|
review = "%s: %s" % (item, review)
|
|
result['review'] = review
|
|
return result
|
|
|
|
|
|
class DOMHTMLSeasonEpisodesParser(DOMParserBase):
|
|
"""Parser for the "episode list" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
sparser = DOMHTMLSeasonEpisodesParser()
|
|
result = sparser.parse(episodes_html_string)
|
|
"""
|
|
extractors = [
|
|
Extractor(label='series link',
|
|
path="//div[@class='parent']",
|
|
attrs=[Attribute(key='series link',
|
|
path=".//a/@href")]
|
|
),
|
|
|
|
Extractor(label='series title',
|
|
path="//head/meta[@property='og:title']",
|
|
attrs=[Attribute(key='series title',
|
|
path="./@content")]
|
|
),
|
|
|
|
Extractor(label='seasons list',
|
|
path="//select[@id='bySeason']//option",
|
|
attrs=[Attribute(key='_seasons',
|
|
multi=True,
|
|
path="./@value")]),
|
|
|
|
Extractor(label='selected season',
|
|
path="//select[@id='bySeason']//option[@selected]",
|
|
attrs=[Attribute(key='_current_season',
|
|
path='./@value')]),
|
|
|
|
Extractor(label='episodes',
|
|
path=".",
|
|
group="//div[@class='info']",
|
|
group_key=".//meta/@content",
|
|
group_key_normalize=lambda x: 'episode %s' % x,
|
|
attrs=[Attribute(key=None,
|
|
multi=True,
|
|
path={
|
|
"link": ".//strong//a[@href][1]/@href",
|
|
"original air date": ".//div[@class='airdate']/text()",
|
|
"title": ".//strong//text()",
|
|
"plot": ".//div[@class='item_description']//text()"
|
|
}
|
|
)]
|
|
)
|
|
]
|
|
|
|
def postprocess_data(self, data):
|
|
series_id = analyze_imdbid(data.get('series link'))
|
|
series_title = data.get('series title', '').strip()
|
|
selected_season = data.get('_current_season',
|
|
'unknown season').strip()
|
|
if not (series_id and series_title):
|
|
return {}
|
|
series = Movie(title=series_title, movieID=str(series_id),
|
|
accessSystem=self._as, modFunct=self._modFunct)
|
|
if series.get('kind') == 'movie':
|
|
series['kind'] = u'tv series'
|
|
try: selected_season = int(selected_season)
|
|
except: pass
|
|
nd = {selected_season: {}}
|
|
if 'episode -1' in data:
|
|
counter = 1
|
|
for episode in data['episode -1']:
|
|
while 'episode %d' % counter in data:
|
|
counter += 1
|
|
k = 'episode %d' % counter
|
|
data[k] = [episode]
|
|
del data['episode -1']
|
|
for episode_nr, episode in data.iteritems():
|
|
if not (episode and episode[0] and
|
|
episode_nr.startswith('episode ')):
|
|
continue
|
|
episode = episode[0]
|
|
episode_nr = episode_nr[8:].rstrip()
|
|
try: episode_nr = int(episode_nr)
|
|
except: pass
|
|
episode_id = analyze_imdbid(episode.get('link' ''))
|
|
episode_air_date = episode.get('original air date',
|
|
'').strip()
|
|
episode_title = episode.get('title', '').strip()
|
|
episode_plot = episode.get('plot', '')
|
|
if not (episode_nr and episode_id and episode_title):
|
|
continue
|
|
ep_obj = Movie(movieID=episode_id, title=episode_title,
|
|
accessSystem=self._as, modFunct=self._modFunct)
|
|
ep_obj['kind'] = u'episode'
|
|
ep_obj['episode of'] = series
|
|
ep_obj['season'] = selected_season
|
|
ep_obj['episode'] = episode_nr
|
|
if episode_air_date:
|
|
ep_obj['original air date'] = episode_air_date
|
|
if episode_air_date[-4:].isdigit():
|
|
ep_obj['year'] = episode_air_date[-4:]
|
|
if episode_plot:
|
|
ep_obj['plot'] = episode_plot
|
|
nd[selected_season][episode_nr] = ep_obj
|
|
_seasons = data.get('_seasons') or []
|
|
for idx, season in enumerate(_seasons):
|
|
try: _seasons[idx] = int(season)
|
|
except: pass
|
|
return {'episodes': nd, '_seasons': _seasons,
|
|
'_current_season': selected_season}
|
|
|
|
|
|
def _build_episode(x):
|
|
"""Create a Movie object for a given series' episode."""
|
|
episode_id = analyze_imdbid(x.get('link'))
|
|
episode_title = x.get('title')
|
|
e = Movie(movieID=episode_id, title=episode_title)
|
|
e['kind'] = u'episode'
|
|
oad = x.get('oad')
|
|
if oad:
|
|
e['original air date'] = oad.strip()
|
|
year = x.get('year')
|
|
if year is not None:
|
|
year = year[5:]
|
|
if year == 'unknown': year = u'????'
|
|
if year and year.isdigit():
|
|
year = int(year)
|
|
e['year'] = year
|
|
else:
|
|
if oad and oad[-4:].isdigit():
|
|
e['year'] = int(oad[-4:])
|
|
epinfo = x.get('episode')
|
|
if epinfo is not None:
|
|
season, episode = epinfo.split(':')[0].split(',')
|
|
e['season'] = int(season[7:])
|
|
e['episode'] = int(episode[8:])
|
|
else:
|
|
e['season'] = 'unknown'
|
|
e['episode'] = 'unknown'
|
|
plot = x.get('plot')
|
|
if plot:
|
|
e['plot'] = plot.strip()
|
|
return e
|
|
|
|
|
|
class DOMHTMLEpisodesParser(DOMParserBase):
|
|
"""Parser for the "episode list" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
eparser = DOMHTMLEpisodesParser()
|
|
result = eparser.parse(episodes_html_string)
|
|
"""
|
|
# XXX: no more used for the list of episodes parser,
|
|
# but only for the episodes cast parser (see below).
|
|
_containsObjects = True
|
|
|
|
kind = 'episodes list'
|
|
_episodes_path = "..//h4"
|
|
_oad_path = "./following-sibling::span/strong[1]/text()"
|
|
|
|
def _init(self):
|
|
self.extractors = [
|
|
Extractor(label='series',
|
|
path="//html",
|
|
attrs=[Attribute(key='series title',
|
|
path=".//title/text()"),
|
|
Attribute(key='series movieID',
|
|
path=".//h1/a[@class='main']/@href",
|
|
postprocess=analyze_imdbid)
|
|
]),
|
|
Extractor(label='episodes',
|
|
group="//div[@class='_imdbpy']/h3",
|
|
group_key="./a/@name",
|
|
path=self._episodes_path,
|
|
attrs=Attribute(key=None,
|
|
multi=True,
|
|
path={
|
|
'link': "./a/@href",
|
|
'title': "./a/text()",
|
|
'year': "./preceding-sibling::a[1]/@name",
|
|
'episode': "./text()[1]",
|
|
'oad': self._oad_path,
|
|
'plot': "./following-sibling::text()[1]"
|
|
},
|
|
postprocess=_build_episode))]
|
|
if self.kind == 'episodes cast':
|
|
self.extractors += [
|
|
Extractor(label='cast',
|
|
group="//h4",
|
|
group_key="./text()[1]",
|
|
group_key_normalize=lambda x: x.strip(),
|
|
path="./following-sibling::table[1]//td[@class='nm']",
|
|
attrs=Attribute(key=None,
|
|
multi=True,
|
|
path={'person': "..//text()",
|
|
'link': "./a/@href",
|
|
'roleID': \
|
|
"../td[4]/div[@class='_imdbpyrole']/@roleid"},
|
|
postprocess=lambda x: \
|
|
build_person(x.get('person') or u'',
|
|
personID=analyze_imdbid(x.get('link')),
|
|
roleID=(x.get('roleID') or u'').split('/'),
|
|
accessSystem=self._as,
|
|
modFunct=self._modFunct)))
|
|
]
|
|
|
|
preprocessors = [
|
|
(re.compile('(<hr/>\n)(<h3>)', re.I),
|
|
r'</div>\1<div class="_imdbpy">\2'),
|
|
(re.compile('(</p>\n\n)</div>', re.I), r'\1'),
|
|
(re.compile('<h3>(.*?)</h3>', re.I), r'<h4>\1</h4>'),
|
|
(_reRolesMovie, _manageRoles),
|
|
(re.compile('(<br/> <br/>\n)(<hr/>)', re.I), r'\1</div>\2')
|
|
]
|
|
|
|
def postprocess_data(self, data):
|
|
# A bit extreme?
|
|
if not 'series title' in data: return {}
|
|
if not 'series movieID' in data: return {}
|
|
stitle = data['series title'].replace('- Episode list', '')
|
|
stitle = stitle.replace('- Episodes list', '')
|
|
stitle = stitle.replace('- Episode cast', '')
|
|
stitle = stitle.replace('- Episodes cast', '')
|
|
stitle = stitle.strip()
|
|
if not stitle: return {}
|
|
seriesID = data['series movieID']
|
|
if seriesID is None: return {}
|
|
series = Movie(title=stitle, movieID=str(seriesID),
|
|
accessSystem=self._as, modFunct=self._modFunct)
|
|
nd = {}
|
|
for key in data.keys():
|
|
if key.startswith('filter-season-') or key.startswith('season-'):
|
|
season_key = key.replace('filter-season-', '').replace('season-', '')
|
|
try: season_key = int(season_key)
|
|
except: pass
|
|
nd[season_key] = {}
|
|
ep_counter = 1
|
|
for episode in data[key]:
|
|
if not episode: continue
|
|
episode_key = episode.get('episode')
|
|
if episode_key is None: continue
|
|
if not isinstance(episode_key, int):
|
|
episode_key = ep_counter
|
|
ep_counter += 1
|
|
cast_key = 'Season %s, Episode %s:' % (season_key,
|
|
episode_key)
|
|
if data.has_key(cast_key):
|
|
cast = data[cast_key]
|
|
for i in xrange(len(cast)):
|
|
cast[i].billingPos = i + 1
|
|
episode['cast'] = cast
|
|
episode['episode of'] = series
|
|
nd[season_key][episode_key] = episode
|
|
if len(nd) == 0:
|
|
return {}
|
|
return {'episodes': nd}
|
|
|
|
|
|
class DOMHTMLEpisodesCastParser(DOMHTMLEpisodesParser):
|
|
"""Parser for the "episodes cast" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
eparser = DOMHTMLEpisodesParser()
|
|
result = eparser.parse(episodes_html_string)
|
|
"""
|
|
kind = 'episodes cast'
|
|
_episodes_path = "..//h4"
|
|
_oad_path = "./following-sibling::b[1]/text()"
|
|
|
|
|
|
class DOMHTMLFaqsParser(DOMParserBase):
|
|
"""Parser for the "FAQ" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
fparser = DOMHTMLFaqsParser()
|
|
result = fparser.parse(faqs_html_string)
|
|
"""
|
|
_defGetRefs = True
|
|
|
|
# XXX: bsoup and lxml don't match (looks like a minor issue, anyway).
|
|
|
|
extractors = [
|
|
Extractor(label='faqs',
|
|
path="//div[@class='section']",
|
|
attrs=Attribute(key='faqs',
|
|
multi=True,
|
|
path={
|
|
'question': "./h3/a/span/text()",
|
|
'answer': "../following-sibling::div[1]//text()"
|
|
},
|
|
postprocess=lambda x: u'%s::%s' % (x.get('question').strip(),
|
|
'\n\n'.join(x.get('answer').replace(
|
|
'\n\n', '\n').strip().split('||')))))
|
|
]
|
|
|
|
preprocessors = [
|
|
(re.compile('<br/><br/>', re.I), r'||'),
|
|
(re.compile('<h4>(.*?)</h4>\n', re.I), r'||\1--'),
|
|
(re.compile('<span class="spoiler"><span>(.*?)</span></span>', re.I),
|
|
r'[spoiler]\1[/spoiler]')
|
|
]
|
|
|
|
|
|
class DOMHTMLAiringParser(DOMParserBase):
|
|
"""Parser for the "airing" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
aparser = DOMHTMLAiringParser()
|
|
result = aparser.parse(airing_html_string)
|
|
"""
|
|
_containsObjects = True
|
|
|
|
extractors = [
|
|
Extractor(label='series title',
|
|
path="//title",
|
|
attrs=Attribute(key='series title', path="./text()",
|
|
postprocess=lambda x: \
|
|
x.replace(' - TV schedule', u''))),
|
|
Extractor(label='series id',
|
|
path="//h1/a[@href]",
|
|
attrs=Attribute(key='series id', path="./@href")),
|
|
|
|
Extractor(label='tv airings',
|
|
path="//tr[@class]",
|
|
attrs=Attribute(key='airing',
|
|
multi=True,
|
|
path={
|
|
'date': "./td[1]//text()",
|
|
'time': "./td[2]//text()",
|
|
'channel': "./td[3]//text()",
|
|
'link': "./td[4]/a[1]/@href",
|
|
'title': "./td[4]//text()",
|
|
'season': "./td[5]//text()",
|
|
},
|
|
postprocess=lambda x: {
|
|
'date': x.get('date'),
|
|
'time': x.get('time'),
|
|
'channel': x.get('channel').strip(),
|
|
'link': x.get('link'),
|
|
'title': x.get('title'),
|
|
'season': (x.get('season') or '').strip()
|
|
}
|
|
))
|
|
]
|
|
|
|
def postprocess_data(self, data):
|
|
if len(data) == 0:
|
|
return {}
|
|
seriesTitle = data['series title']
|
|
seriesID = analyze_imdbid(data['series id'])
|
|
if data.has_key('airing'):
|
|
for airing in data['airing']:
|
|
title = airing.get('title', '').strip()
|
|
if not title:
|
|
epsTitle = seriesTitle
|
|
if seriesID is None:
|
|
continue
|
|
epsID = seriesID
|
|
else:
|
|
epsTitle = '%s {%s}' % (data['series title'],
|
|
airing['title'])
|
|
epsID = analyze_imdbid(airing['link'])
|
|
e = Movie(title=epsTitle, movieID=epsID)
|
|
airing['episode'] = e
|
|
del airing['link']
|
|
del airing['title']
|
|
if not airing['season']:
|
|
del airing['season']
|
|
if 'series title' in data:
|
|
del data['series title']
|
|
if 'series id' in data:
|
|
del data['series id']
|
|
if 'airing' in data:
|
|
data['airing'] = filter(None, data['airing'])
|
|
if 'airing' not in data or not data['airing']:
|
|
return {}
|
|
return data
|
|
|
|
|
|
class DOMHTMLSynopsisParser(DOMParserBase):
|
|
"""Parser for the "synopsis" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
sparser = HTMLSynopsisParser()
|
|
result = sparser.parse(synopsis_html_string)
|
|
"""
|
|
extractors = [
|
|
Extractor(label='synopsis',
|
|
path="//div[@class='display'][not(@style)]",
|
|
attrs=Attribute(key='synopsis',
|
|
path=".//text()",
|
|
postprocess=lambda x: '\n\n'.join(x.strip().split('||'))))
|
|
]
|
|
|
|
preprocessors = [
|
|
(re.compile('<br/><br/>', re.I), r'||')
|
|
]
|
|
|
|
|
|
class DOMHTMLParentsGuideParser(DOMParserBase):
|
|
"""Parser for the "parents guide" page of a given movie.
|
|
The page should be provided as a string, as taken from
|
|
the akas.imdb.com server. The final result will be a
|
|
dictionary, with a key for every relevant section.
|
|
|
|
Example:
|
|
pgparser = HTMLParentsGuideParser()
|
|
result = pgparser.parse(parentsguide_html_string)
|
|
"""
|
|
extractors = [
|
|
Extractor(label='parents guide',
|
|
group="//div[@class='section']",
|
|
group_key="./h3/a/span/text()",
|
|
group_key_normalize=lambda x: x.lower(),
|
|
path="../following-sibling::div[1]/p",
|
|
attrs=Attribute(key=None,
|
|
path=".//text()",
|
|
postprocess=lambda x: [t.strip().replace('\n', ' ')
|
|
for t in x.split('||') if t.strip()]))
|
|
]
|
|
|
|
preprocessors = [
|
|
(re.compile('<br/><br/>', re.I), r'||')
|
|
]
|
|
|
|
def postprocess_data(self, data):
|
|
data2 = {}
|
|
for key in data:
|
|
if data[key]:
|
|
data2[key] = data[key]
|
|
if not data2:
|
|
return {}
|
|
return {'parents guide': data2}
|
|
|
|
|
|
_OBJECTS = {
|
|
'movie_parser': ((DOMHTMLMovieParser,), None),
|
|
'plot_parser': ((DOMHTMLPlotParser,), None),
|
|
'movie_awards_parser': ((DOMHTMLAwardsParser,), None),
|
|
'taglines_parser': ((DOMHTMLTaglinesParser,), None),
|
|
'keywords_parser': ((DOMHTMLKeywordsParser,), None),
|
|
'crazycredits_parser': ((DOMHTMLCrazyCreditsParser,), None),
|
|
'goofs_parser': ((DOMHTMLGoofsParser,), None),
|
|
'alternateversions_parser': ((DOMHTMLAlternateVersionsParser,), None),
|
|
'trivia_parser': ((DOMHTMLTriviaParser,), None),
|
|
'soundtrack_parser': ((DOMHTMLSoundtrackParser,), {'kind': 'soundtrack'}),
|
|
'quotes_parser': ((DOMHTMLQuotesParser,), None),
|
|
'releasedates_parser': ((DOMHTMLReleaseinfoParser,), None),
|
|
'ratings_parser': ((DOMHTMLRatingsParser,), None),
|
|
'officialsites_parser': ((DOMHTMLOfficialsitesParser,), None),
|
|
'criticrev_parser': ((DOMHTMLCriticReviewsParser,),
|
|
{'kind': 'critic reviews'}),
|
|
'externalrev_parser': ((DOMHTMLOfficialsitesParser,),
|
|
{'kind': 'external reviews'}),
|
|
'newsgrouprev_parser': ((DOMHTMLOfficialsitesParser,),
|
|
{'kind': 'newsgroup reviews'}),
|
|
'misclinks_parser': ((DOMHTMLOfficialsitesParser,),
|
|
{'kind': 'misc links'}),
|
|
'soundclips_parser': ((DOMHTMLOfficialsitesParser,),
|
|
{'kind': 'sound clips'}),
|
|
'videoclips_parser': ((DOMHTMLOfficialsitesParser,),
|
|
{'kind': 'video clips'}),
|
|
'photosites_parser': ((DOMHTMLOfficialsitesParser,),
|
|
{'kind': 'photo sites'}),
|
|
'connections_parser': ((DOMHTMLConnectionParser,), None),
|
|
'tech_parser': ((DOMHTMLTechParser,), None),
|
|
'business_parser': ((DOMHTMLTechParser,),
|
|
{'kind': 'business', '_defGetRefs': 1}),
|
|
'literature_parser': ((DOMHTMLTechParser,), {'kind': 'literature'}),
|
|
'locations_parser': ((DOMHTMLLocationsParser,), None),
|
|
'rec_parser': ((DOMHTMLRecParser,), None),
|
|
'news_parser': ((DOMHTMLNewsParser,), None),
|
|
'episodes_parser': ((DOMHTMLEpisodesParser,), None),
|
|
'season_episodes_parser': ((DOMHTMLSeasonEpisodesParser,), None),
|
|
'episodes_cast_parser': ((DOMHTMLEpisodesCastParser,), None),
|
|
'eprating_parser': ((DOMHTMLEpisodesRatings,), None),
|
|
'movie_faqs_parser': ((DOMHTMLFaqsParser,), None),
|
|
'airing_parser': ((DOMHTMLAiringParser,), None),
|
|
'synopsis_parser': ((DOMHTMLSynopsisParser,), None),
|
|
'parentsguide_parser': ((DOMHTMLParentsGuideParser,), None)
|
|
}
|
|
|