From 44cd76bae127319302172ae246fb7dcdcc552ffd Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Tue, 24 Jul 2012 17:44:54 +0000 Subject: [PATCH] + updated to TheMovieDB API v3 * lots of changes because now imdbid/tmdbid exist equally, but may not be available depending on the circumstances, so so there is lots of workarounds in MediaBindingBean to account for both ids * updated artwork scripts to use current TMDb class --- .../filebot/cli/CmdlineOperations.java | 9 +- .../filebot/format/MediaBindingBean.java | 35 +- .../filebot/media/ReleaseInfo.java | 4 +- .../ui/rename/BindingDialog.properties | 2 +- .../sourceforge/filebot/web/IMDbClient.java | 16 +- source/net/sourceforge/filebot/web/Movie.java | 18 +- .../sourceforge/filebot/web/MovieFormat.java | 11 +- .../sourceforge/filebot/web/MoviePart.java | 8 +- .../filebot/web/OpenSubtitlesXmlRpc.java | 6 +- .../filebot/web/SublightSubtitleClient.java | 3 +- .../sourceforge/filebot/web/TMDbClient.java | 551 ++++++++---------- .../web/SublightSubtitleClientTest.java | 8 +- .../filebot/web/TMDbClientTest.java | 44 +- website/scripts/artwork.tmdb.groovy | 53 +- website/scripts/lib/xbmc.groovy | 16 +- 15 files changed, 358 insertions(+), 426 deletions(-) diff --git a/source/net/sourceforge/filebot/cli/CmdlineOperations.java b/source/net/sourceforge/filebot/cli/CmdlineOperations.java index 8d7a83ab..a97066cc 100644 --- a/source/net/sourceforge/filebot/cli/CmdlineOperations.java +++ b/source/net/sourceforge/filebot/cli/CmdlineOperations.java @@ -85,8 +85,7 @@ import net.sourceforge.tuned.FileUtilities.ParentFilter; public class CmdlineOperations implements CmdlineInterface { @Override - public List rename(Collection files, RenameAction action, String conflict, String output, String formatExpression, String db, String query, String sortOrder, String filterExpression, String lang, boolean strict) - throws Exception { + public List rename(Collection files, RenameAction action, String conflict, String output, String formatExpression, String db, String query, String sortOrder, String filterExpression, String lang, boolean strict) throws Exception { ExpressionFormat format = (formatExpression != null) ? new ExpressionFormat(formatExpression) : null; ExpressionFilter filter = (filterExpression != null) ? new ExpressionFilter(filterExpression) : null; File outputDir = (output != null && output.length() > 0) ? new File(output).getAbsoluteFile() : null; @@ -143,8 +142,7 @@ public class CmdlineOperations implements CmdlineInterface { } - public List renameSeries(Collection files, RenameAction renameAction, ConflictAction conflictAction, File outputDir, ExpressionFormat format, EpisodeListProvider db, String query, SortOrder sortOrder, - ExpressionFilter filter, Locale locale, boolean strict) throws Exception { + public List renameSeries(Collection files, RenameAction renameAction, ConflictAction conflictAction, File outputDir, ExpressionFormat format, EpisodeListProvider db, String query, SortOrder sortOrder, ExpressionFilter filter, Locale locale, boolean strict) throws Exception { CLILogger.config(format("Rename episodes using [%s]", db.getName())); List mediaFiles = filter(files, VIDEO_FILES, SUBTITLE_FILES); @@ -290,8 +288,7 @@ public class CmdlineOperations implements CmdlineInterface { } - public List renameMovie(Collection files, RenameAction renameAction, ConflictAction conflictAction, File outputDir, ExpressionFormat format, MovieIdentificationService service, String query, Locale locale, boolean strict) - throws Exception { + public List renameMovie(Collection files, RenameAction renameAction, ConflictAction conflictAction, File outputDir, ExpressionFormat format, MovieIdentificationService service, String query, Locale locale, boolean strict) throws Exception { CLILogger.config(format("Rename movies using [%s]", service.getName())); // ignore sample files diff --git a/source/net/sourceforge/filebot/format/MediaBindingBean.java b/source/net/sourceforge/filebot/format/MediaBindingBean.java index c5120b93..1a46de94 100644 --- a/source/net/sourceforge/filebot/format/MediaBindingBean.java +++ b/source/net/sourceforge/filebot/format/MediaBindingBean.java @@ -146,13 +146,19 @@ public class MediaBindingBean { @Define("imdbid") - public String getImdbId() { - int imdb = getMovie().getImdbId(); + public String getImdbId() throws Exception { + int imdbid = getMovie().getImdbId(); - if (imdb <= 0) - return null; + if (imdbid <= 0) { + if (getMovie().getTmdbId() <= 0) { + return null; + } + + // lookup IMDbID for TMDbID + imdbid = WebServices.TMDb.getMovieInfo(getMovie(), null).getImdbId(); + } - return String.format("%07d", imdb); + return String.format("%07d", imdbid); } @@ -381,6 +387,12 @@ public class MediaBindingBean { } + @Define("collection") + public Object getCollection() { + return getMetaInfo().getProperty("collection"); + } + + @Define("info") public synchronized AssociativeScriptObject getMetaInfo() { if (metaInfo == null) { @@ -388,7 +400,7 @@ public class MediaBindingBean { if (infoObject instanceof Episode) metaInfo = WebServices.TheTVDB.getSeriesInfoByName(((Episode) infoObject).getSeriesName(), Locale.ENGLISH); if (infoObject instanceof Movie) - metaInfo = WebServices.TMDb.getMovieInfo((Movie) infoObject, Locale.ENGLISH); + metaInfo = WebServices.TMDb.getMovieInfo(getMovie(), Locale.ENGLISH); } catch (Exception e) { throw new RuntimeException("Failed to retrieve metadata: " + infoObject, e); } @@ -403,10 +415,13 @@ public class MediaBindingBean { Object data = null; try { - if (infoObject instanceof Episode) - data = WebServices.IMDb.getImdbApiMovieInfo(new Movie(getEpisode().getSeriesName(), getEpisode().getSeriesStartDate().getYear(), -1)); - if (infoObject instanceof Movie) - data = WebServices.IMDb.getImdbApiMovieInfo(getMovie()); + if (infoObject instanceof Episode) { + data = WebServices.IMDb.getImdbApiMovieInfo(new Movie(getEpisode().getSeriesName(), getEpisode().getSeriesStartDate().getYear(), -1, -1)); + } + if (infoObject instanceof Movie) { + Movie m = getMovie(); + data = WebServices.IMDb.getImdbApiMovieInfo(m.getImdbId() > 0 ? m : new Movie(null, -1, WebServices.TMDb.getMovieInfo(getMovie(), Locale.ENGLISH).getId(), -1)); + } } catch (Exception e) { throw new RuntimeException("Failed to retrieve metadata: " + infoObject, e); } diff --git a/source/net/sourceforge/filebot/media/ReleaseInfo.java b/source/net/sourceforge/filebot/media/ReleaseInfo.java index 311ab79c..81f08f46 100644 --- a/source/net/sourceforge/filebot/media/ReleaseInfo.java +++ b/source/net/sourceforge/filebot/media/ReleaseInfo.java @@ -226,7 +226,6 @@ public class ReleaseInfo { return new FileFolderNameFilter(compile(getBundle(getClass().getName()).getString("pattern.file.ignore"))); } - // fetch release group names online and try to update the data every other day protected final CachedResource releaseGroupResource = new PatternResource(getBundle(getClass().getName()).getString("url.release-groups")); protected final CachedResource queryBlacklistResource = new PatternResource(getBundle(getClass().getName()).getString("url.query-blacklist")); @@ -264,7 +263,7 @@ public class ReleaseInfo { int imdbid = scanner.nextInt(); String name = scanner.next(); int year = scanner.nextInt(); - movies.add(new Movie(name, year, imdbid)); + movies.add(new Movie(name, year, imdbid, -1)); } return movies.toArray(new Movie[0]); @@ -335,7 +334,6 @@ public class ReleaseInfo { return patterns; } - private final Map, Map> languageMapCache = synchronizedMap(new WeakHashMap, Map>(2)); diff --git a/source/net/sourceforge/filebot/ui/rename/BindingDialog.properties b/source/net/sourceforge/filebot/ui/rename/BindingDialog.properties index 60af046f..073de65b 100644 --- a/source/net/sourceforge/filebot/ui/rename/BindingDialog.properties +++ b/source/net/sourceforge/filebot/ui/rename/BindingDialog.properties @@ -2,4 +2,4 @@ parameter.exclude: ^StreamKind|Count$ # preview expressions (keys are tagged so they can be sorted alphabetically) -expressions: n,y,s,e,t,airdate,startdate,absolute,special,imdbid,episode,sxe,s00e00,movie,vc,ac,cf,vf,af,resolution,hpi,ws,sdhd,source,group,crc32,fn,ext,file,pi,pn,lang,actors,director,genres,certification,rating,dim,info.runtime,info.status,imdb.rating,imdb.votes,media.title,media.durationString,media.overallBitRateString,video.codecID,video.frameRate,video.displayAspectRatioString,video.height,video.scanType,audio.format,audio.bitRateString,audio.language,text.codecInfo,text.language +expressions: n,y,s,e,t,airdate,startdate,absolute,special,imdbid,episode,sxe,s00e00,movie,vc,ac,cf,vf,af,resolution,hpi,ws,sdhd,source,group,crc32,fn,ext,file,pi,pn,lang,actors,director,collection,genres,certification,rating,dim,info.runtime,info.status,imdb.rating,imdb.votes,media.title,media.durationString,media.overallBitRateString,video.codecID,video.frameRate,video.displayAspectRatioString,video.height,video.scanType,audio.format,audio.bitRateString,audio.language,text.codecInfo,text.language diff --git a/source/net/sourceforge/filebot/web/IMDbClient.java b/source/net/sourceforge/filebot/web/IMDbClient.java index d6956055..c9f28850 100644 --- a/source/net/sourceforge/filebot/web/IMDbClient.java +++ b/source/net/sourceforge/filebot/web/IMDbClient.java @@ -29,7 +29,6 @@ import java.util.regex.Pattern; import javax.swing.Icon; import net.sourceforge.filebot.ResourceManager; -import net.sourceforge.filebot.web.TMDbClient.Artwork; import net.sourceforge.filebot.web.TMDbClient.MovieInfo; import net.sourceforge.filebot.web.TMDbClient.MovieInfo.MovieProperty; import net.sourceforge.filebot.web.TMDbClient.Person; @@ -85,7 +84,7 @@ public class IMDbClient implements MovieIdentificationService { String year = node.getNextSibling().getTextContent().replaceAll("[\\p{Punct}\\p{Space}]+", ""); // remove non-number characters String href = getAttribute("href", node); - results.add(new Movie(name, Integer.parseInt(year), getImdbId(href))); + results.add(new Movie(name, Integer.parseInt(year), getImdbId(href), -1)); } catch (Exception e) { // ignore illegal movies (TV Shows, Videos, Video Games, etc) } @@ -139,7 +138,7 @@ public class IMDbClient implements MovieIdentificationService { } } - return new Movie(name, Pattern.matches("\\d{4}", year) ? Integer.parseInt(year) : -1, getImdbId(url)); + return new Movie(name, Pattern.matches("\\d{4}", year) ? Integer.parseInt(year) : -1, getImdbId(url), -1); } catch (Exception e) { // ignore, we probably got redirected to an error page return null; @@ -211,11 +210,11 @@ public class IMDbClient implements MovieIdentificationService { } Map fields = new EnumMap(MovieProperty.class); - fields.put(MovieProperty.name, data.get("title")); + fields.put(MovieProperty.title, data.get("title")); fields.put(MovieProperty.certification, data.get("rated")); fields.put(MovieProperty.tagline, data.get("plot")); - fields.put(MovieProperty.rating, data.get("imdbRating")); - fields.put(MovieProperty.votes, data.get("imdbVotes").replaceAll("\\D", "")); + fields.put(MovieProperty.vote_average, data.get("imdbRating")); + fields.put(MovieProperty.vote_count, data.get("imdbVotes").replaceAll("\\D", "")); fields.put(MovieProperty.imdb_id, data.get("imdbID")); List genres = new ArrayList(); @@ -225,10 +224,9 @@ public class IMDbClient implements MovieIdentificationService { List actors = new ArrayList(); for (String it : data.get("actors").split(",")) { - actors.add(new Person(it.trim(), null, "Actor", null, null)); + actors.add(new Person(it.trim(), null, null)); } - List image = singletonList(new Artwork("poster", data.get("poster"), null, null, null)); - return new MovieInfo(fields, genres, actors, image); + return new MovieInfo(fields, genres, new ArrayList(0), actors); } } diff --git a/source/net/sourceforge/filebot/web/Movie.java b/source/net/sourceforge/filebot/web/Movie.java index c64a39ab..0b912a37 100644 --- a/source/net/sourceforge/filebot/web/Movie.java +++ b/source/net/sourceforge/filebot/web/Movie.java @@ -2,10 +2,14 @@ package net.sourceforge.filebot.web; +import java.util.Arrays; + + public class Movie extends SearchResult { protected int year; protected int imdbId; + protected int tmdbId; protected Movie() { @@ -14,14 +18,15 @@ public class Movie extends SearchResult { public Movie(Movie obj) { - this(obj.name, obj.year, obj.imdbId); + this(obj.name, obj.year, obj.imdbId, obj.tmdbId); } - public Movie(String name, int year, int imdbId) { + public Movie(String name, int year, int imdbId, int tmdbId) { super(name); this.year = year; this.imdbId = imdbId; + this.tmdbId = tmdbId; } @@ -35,12 +40,19 @@ public class Movie extends SearchResult { } + public int getTmdbId() { + return tmdbId; + } + + @Override public boolean equals(Object object) { if (object instanceof Movie) { Movie other = (Movie) object; if (imdbId > 0 && other.imdbId > 0) { return imdbId == other.imdbId; + } else if (tmdbId > 0 && other.tmdbId > 0) { + return tmdbId == other.tmdbId; } return year == other.year && name.equals(other.name); @@ -58,7 +70,7 @@ public class Movie extends SearchResult { @Override public int hashCode() { - return imdbId; + return Arrays.hashCode(new Object[] { name, year }); } diff --git a/source/net/sourceforge/filebot/web/MovieFormat.java b/source/net/sourceforge/filebot/web/MovieFormat.java index fbddc7f0..1c358b43 100644 --- a/source/net/sourceforge/filebot/web/MovieFormat.java +++ b/source/net/sourceforge/filebot/web/MovieFormat.java @@ -18,14 +18,14 @@ public class MovieFormat extends Format { private final boolean includePartIndex; private final boolean smart; - + public MovieFormat(boolean includeYear, boolean includePartIndex, boolean smart) { this.includeYear = includeYear; this.includePartIndex = includePartIndex; this.smart = smart; } - + @Override public StringBuffer format(Object obj, StringBuffer sb, FieldPosition pos) { // format episode object, e.g. Avatar (2009), Part 1 @@ -50,11 +50,10 @@ public class MovieFormat extends Format { return sb; } - private final Pattern moviePattern = Pattern.compile("([^\\p{Punct}]+?)[\\p{Punct}\\s]+(\\d{4})(?:[\\p{Punct}\\s]+|$)"); private final Pattern partPattern = Pattern.compile("(?:Part|CD)\\D?(\\d)$", Pattern.CASE_INSENSITIVE); - + @Override public Movie parseObject(String source, ParsePosition pos) { String s = source; @@ -73,7 +72,7 @@ public class MovieFormat extends Format { String name = m.group(1).trim(); int year = Integer.parseInt(m.group(2)); - Movie movie = new Movie(name, year, -1); + Movie movie = new Movie(name, year, -1, -1); if (partIndex >= 0) { movie = new MoviePart(movie, partIndex, partCount); } @@ -88,7 +87,7 @@ public class MovieFormat extends Format { return null; } - + @Override public Movie parseObject(String source) throws ParseException { return (Movie) super.parseObject(source); diff --git a/source/net/sourceforge/filebot/web/MoviePart.java b/source/net/sourceforge/filebot/web/MoviePart.java index b0019e11..8fb29be8 100644 --- a/source/net/sourceforge/filebot/web/MoviePart.java +++ b/source/net/sourceforge/filebot/web/MoviePart.java @@ -9,17 +9,17 @@ public class MoviePart extends Movie { public MoviePart(MoviePart obj) { - this(obj.name, obj.year, obj.imdbId, obj.partIndex, obj.partCount); + this(obj.name, obj.year, obj.imdbId, obj.tmdbId, obj.partIndex, obj.partCount); } public MoviePart(Movie movie, int partIndex, int partCount) { - this(movie.name, movie.year, movie.imdbId, partIndex, partCount); + this(movie.name, movie.year, movie.imdbId, movie.tmdbId, partIndex, partCount); } - public MoviePart(String name, int year, int imdbId, int partIndex, int partCount) { - super(name, year, imdbId); + public MoviePart(String name, int year, int imdbId, int tmdbId, int partIndex, int partCount) { + super(name, year, imdbId, tmdbId); this.partIndex = partIndex; this.partCount = partCount; } diff --git a/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java b/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java index dca2d84c..4baeadcf 100644 --- a/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java +++ b/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java @@ -143,7 +143,7 @@ public class OpenSubtitlesXmlRpc { String name = matcher.group(1).replaceAll("\"", "").trim(); int year = Integer.parseInt(matcher.group(2)); - movies.add(new Movie(name, year, Integer.parseInt(imdbid))); + movies.add(new Movie(name, year, Integer.parseInt(imdbid), -1)); } catch (Exception e) { Logger.getLogger(OpenSubtitlesXmlRpc.class.getName()).log(Level.FINE, String.format("Ignore movie [%s]: %s", movie, e.getMessage())); } @@ -238,7 +238,7 @@ public class OpenSubtitlesXmlRpc { int year = Integer.parseInt(info.get("MovieYear")); int imdb = Integer.parseInt(info.get("MovieImdbID")); - movieHashMap.put(hash, new Movie(name, year, imdb)); + movieHashMap.put(hash, new Movie(name, year, imdb, -1)); } } @@ -261,7 +261,7 @@ public class OpenSubtitlesXmlRpc { String name = data.get("title"); int year = Integer.parseInt(data.get("year")); - return new Movie(name, year, imdbid); + return new Movie(name, year, imdbid, -1); } catch (RuntimeException e) { // ignore, invalid response Logger.getLogger(getClass().getName()).log(Level.WARNING, String.format("Failed to lookup movie by imdbid %s: %s", imdbid, e.getMessage())); diff --git a/source/net/sourceforge/filebot/web/SublightSubtitleClient.java b/source/net/sourceforge/filebot/web/SublightSubtitleClient.java index 302aaa88..a57a46da 100644 --- a/source/net/sourceforge/filebot/web/SublightSubtitleClient.java +++ b/source/net/sourceforge/filebot/web/SublightSubtitleClient.java @@ -97,7 +97,7 @@ public class SublightSubtitleClient implements SubtitleProvider, VideoHashSubtit // remove classifier (e.g. tt0436992 -> 0436992) int id = Integer.parseInt(imdb.getId().substring(2)); - results.add(new Movie(imdb.getTitle(), imdb.getYear(), id)); + results.add(new Movie(imdb.getTitle(), imdb.getYear(), id, -1)); } } @@ -120,6 +120,7 @@ public class SublightSubtitleClient implements SubtitleProvider, VideoHashSubtit } + @Override public Map> getSubtitleList(File[] files, final String languageName) throws Exception { Map> subtitles = new HashMap>(files.length); diff --git a/source/net/sourceforge/filebot/web/TMDbClient.java b/source/net/sourceforge/filebot/web/TMDbClient.java index 43c23eb4..f35f880e 100644 --- a/source/net/sourceforge/filebot/web/TMDbClient.java +++ b/source/net/sourceforge/filebot/web/TMDbClient.java @@ -5,17 +5,18 @@ package net.sourceforge.filebot.web; import static java.util.Arrays.*; import static java.util.Collections.*; import static net.sourceforge.filebot.web.WebRequest.*; -import static net.sourceforge.tuned.XPathUtilities.*; import java.io.File; import java.io.IOException; -import java.io.Reader; import java.io.Serializable; -import java.net.MalformedURLException; import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.AbstractList; import java.util.ArrayList; import java.util.Collection; import java.util.EnumMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -27,19 +28,18 @@ import java.util.logging.Logger; import javax.swing.Icon; import net.sourceforge.filebot.ResourceManager; -import net.sourceforge.filebot.web.TMDbClient.Artwork.ArtworkProperty; import net.sourceforge.filebot.web.TMDbClient.MovieInfo.MovieProperty; import net.sourceforge.filebot.web.TMDbClient.Person.PersonProperty; -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.xml.sax.SAXException; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; public class TMDbClient implements MovieIdentificationService { private static final String host = "api.themoviedb.org"; - private static final String version = "2.1"; + private static final String version = "3"; private static final FloodLimit SEARCH_LIMIT = new FloodLimit(10, 12, TimeUnit.SECONDS); private static final FloodLimit REQUEST_LIMIT = new FloodLimit(30, 12, TimeUnit.SECONDS); @@ -66,40 +66,41 @@ public class TMDbClient implements MovieIdentificationService { @Override public List searchMovie(String query, Locale locale) throws IOException { - try { - return getMovies("Movie.search", encode(query), locale, SEARCH_LIMIT); - } catch (SAXException e) { - // TMDb output is sometimes malformed xml - Logger.getLogger(getClass().getName()).log(Level.WARNING, e.getMessage()); - return emptyList(); + JSONObject response = request("search/movie", singletonMap("query", query), locale, SEARCH_LIMIT); + List result = new ArrayList(); + + for (JSONObject it : jsonList(response.get("results"))) { + // e.g. {"id":16320,"title":"冲出宁静号","release_date":"2005-09-30","original_title":"Serenity"} + String title = (String) it.get("title"); + if (title == null || title.isEmpty()) { + title = (String) it.get("original_title"); + } + + try { + long id = (Long) it.get("id"); + int year = -1; + try { + String release = (String) it.get("release_date"); + year = new Scanner(release).useDelimiter("\\D+").nextInt(); + } catch (Exception e) { + throw new IllegalArgumentException("Missing data: year"); + } + result.add(new Movie(title, year, -1, (int) id)); + } catch (Exception e) { + Logger.getLogger(TMDbClient.class.getName()).log(Level.FINE, String.format("Ignore movie [%s]: %s", title, e.getMessage())); + } } - } - - - public List searchMovie(String hash, long bytesize, Locale locale) throws IOException, SAXException { - return getMovies("Media.getInfo", hash + "/" + bytesize, locale, SEARCH_LIMIT); + return result; } @Override - public Movie getMovieDescriptor(int imdbid, Locale locale) throws Exception { - Document dom = fetchResource("Movie.imdbLookup", String.format("tt%07d", imdbid), locale, REQUEST_LIMIT); - Node movie = selectNode("//movie", dom); - - if (movie == null) - return null; - - String name = getTextContent("name", movie); - String released = getTextContent("released", movie); - int year = -1; - - try { - year = new Scanner(released).useDelimiter("\\D+").nextInt(); - } catch (Exception e) { - Logger.getLogger(getClass().getName()).log(Level.WARNING, "Illegal release year: " + released); + public Movie getMovieDescriptor(int imdbid, Locale locale) throws IOException { + MovieInfo info = getMovieInfo(String.format("tt%07d", imdbid), locale, false); + if (info != null) { + return new Movie(info.getName(), info.getReleased().getYear(), info.getImdbId(), info.getId()); } - - return new Movie(name, year, imdbid); + return null; } @@ -109,159 +110,173 @@ public class TMDbClient implements MovieIdentificationService { } - protected List getMovies(String method, String parameter, Locale locale, FloodLimit limit) throws IOException, SAXException { - Document dom = fetchResource(method, parameter, locale, limit); - List result = new ArrayList(); - - for (Node node : selectNodes("//movie", dom)) { - String name = getTextContent("name", node); - try { - // release date format will be YYYY-MM-DD, but we only care about the year - int year = -1; - try { - year = new Scanner(getTextContent("released", node)).useDelimiter("\\D+").nextInt(); - } catch (RuntimeException e) { - throw new IllegalArgumentException("Missing data: year"); - } - - // imdb id will be tt1234567, but we only care about the number - int imdbid = -1; - try { - imdbid = new Scanner(getTextContent("imdb_id", node)).useDelimiter("\\D+").nextInt(); - } catch (RuntimeException e) { - // ignore - } - - result.add(new Movie(name, year, imdbid)); - } catch (Exception e) { - Logger.getLogger(TMDbClient.class.getName()).log(Level.FINE, String.format("Ignore movie [%s]: %s", name, e.getMessage())); - } - } - - return result; - } - - - protected URL getResourceLocation(String method, String parameter, Locale locale) throws MalformedURLException { - // e.g. http://api.themoviedb.org/2.1/Movie.search/en/xml/{apikey}/serenity - return new URL("http", host, "/" + version + "/" + method + "/" + locale.getLanguage() + "/xml/" + apikey + "/" + parameter); - } - - - protected Document fetchResource(String method, String parameter, Locale locale, final FloodLimit limit) throws IOException, SAXException { - return getDocument(new CachedPage(getResourceLocation(method, parameter, locale)) { - - @Override - protected Reader openConnection(URL url) throws IOException { - try { - if (limit != null) { - limit.acquirePermit(); - } - return super.openConnection(url); - } catch (InterruptedException e) { - throw new IOException(e); - } - }; - }.get()); - } - - - public MovieInfo getMovieInfo(Movie movie, Locale locale) throws Exception { - if (movie.getImdbId() >= 0) { - return getMovieInfoByIMDbID(movie.getImdbId(), Locale.ENGLISH); + public MovieInfo getMovieInfo(Movie movie, Locale locale) throws IOException { + if (movie.getTmdbId() >= 0) { + return getMovieInfo(String.valueOf(movie.getTmdbId()), locale, true); + } else if (movie.getImdbId() >= 0) { + return getMovieInfo(String.format("tt%07d", movie.getImdbId()), locale, true); } else { - return getMovieInfoByName(movie.getName(), movie.getYear(), Locale.ENGLISH); - } - } - - - public MovieInfo getMovieInfoByName(String name, int year, Locale locale) throws Exception { - for (Movie it : searchMovie(name, locale)) { - if (name.equalsIgnoreCase(it.getName()) && year == it.getYear()) { - return getMovieInfo(it, locale); + for (Movie it : searchMovie(movie.getName(), locale)) { + if (movie.getName().equalsIgnoreCase(it.getName()) && movie.getYear() == it.getYear()) { + return getMovieInfo(String.valueOf(movie.getTmdbId()), locale, true); + } } } - return null; } - public MovieInfo getMovieInfoByIMDbID(int imdbid, Locale locale) throws Exception { - if (imdbid < 0) - throw new IllegalArgumentException("Illegal IMDb ID: " + imdbid); + public MovieInfo getMovieInfo(String id, Locale locale, boolean extendedInfo) throws IOException { + JSONObject response = request("movie/" + id, null, locale, REQUEST_LIMIT); - // resolve imdbid to tmdbid - Document dom = fetchResource("Movie.imdbLookup", String.format("tt%07d", imdbid), locale, REQUEST_LIMIT); - - String tmdbid = selectString("//movie/id", dom); - if (tmdbid == null || tmdbid.isEmpty()) { - throw new IllegalArgumentException("Unable to lookup tmdb entry: " + String.format("tt%07d", imdbid)); + Map fields = new EnumMap(MovieProperty.class); + for (MovieProperty key : MovieProperty.values()) { + Object value = response.get(key.name()); + if (value != null) { + fields.put(key, value.toString()); + } } - // get complete movie info via tmdbid lookup - dom = fetchResource("Movie.getInfo", tmdbid, locale, REQUEST_LIMIT); - - // select info from xml - Node node = selectNode("//movie", dom); - - Map movieProperties = new EnumMap(MovieProperty.class); - for (MovieProperty property : MovieProperty.values()) { - movieProperties.put(property, getTextContent(property.name(), node)); + try { + JSONObject collection = (JSONObject) response.get("belongs_to_collection"); + fields.put(MovieProperty.collection, (String) collection.get("name")); + } catch (Exception e) { + // ignore } List genres = new ArrayList(); - for (Node category : selectNodes("//category[@type='genre']", node)) { - genres.add(getAttribute("name", category)); + for (JSONObject it : jsonList(response.get("genres"))) { + genres.add((String) it.get("name")); } - List artwork = new ArrayList(); - for (Node image : selectNodes("//image", node)) { - Map artworkProperties = new EnumMap(ArtworkProperty.class); - for (ArtworkProperty property : ArtworkProperty.values()) { - artworkProperties.put(property, getAttribute(property.name(), image)); + List spokenLanguages = new ArrayList(); + for (JSONObject it : jsonList(response.get("spoken_languages"))) { + spokenLanguages.add((String) it.get("iso_639_1")); + } + + if (extendedInfo) { + JSONObject releases = request("movie/" + fields.get(MovieProperty.id) + "/releases", null, null, REQUEST_LIMIT); + for (JSONObject it : jsonList(releases.get("countries"))) { + if ("US".equals(it.get("iso_3166_1"))) { + fields.put(MovieProperty.certification, (String) it.get("certification")); + } } - artwork.add(new Artwork(artworkProperties)); } List cast = new ArrayList(); - for (Node image : selectNodes("//person", node)) { - Map personProperties = new EnumMap(PersonProperty.class); - for (PersonProperty property : PersonProperty.values()) { - personProperties.put(property, getAttribute(property.name(), image)); + if (extendedInfo) { + JSONObject castResponse = request("movie/" + fields.get(MovieProperty.id) + "/casts", null, null, REQUEST_LIMIT); + for (String section : new String[] { "cast", "crew" }) { + for (JSONObject it : jsonList(castResponse.get(section))) { + Map person = new EnumMap(PersonProperty.class); + for (PersonProperty key : PersonProperty.values()) { + Object value = it.get(key.name()); + if (value != null) { + person.put(key, value.toString()); + } + } + cast.add(new Person(person)); + } } - cast.add(new Person(personProperties)); } - return new MovieInfo(movieProperties, genres, cast, artwork); + return new MovieInfo(fields, genres, spokenLanguages, cast); + } + + + public List getArtwork(String id) throws IOException { + // http://api.themoviedb.org/3/movie/11/images + JSONObject config = request("configuration", null, null, REQUEST_LIMIT); + String baseUrl = (String) ((JSONObject) config.get("images")).get("base_url"); + + JSONObject images = request("movie/" + id + "/images", null, null, REQUEST_LIMIT); + List artwork = new ArrayList(); + + for (String section : new String[] { "backdrops", "posters" }) { + for (JSONObject it : jsonList(images.get(section))) { + try { + String url = baseUrl + "original" + (String) it.get("file_path"); + long width = (Long) it.get("width"); + long height = (Long) it.get("height"); + String lang = (String) it.get("iso_639_1"); + artwork.add(new Artwork(section, new URL(url), (int) width, (int) height, lang)); + } catch (Exception e) { + Logger.getLogger(getClass().getName()).log(Level.WARNING, "Invalid artwork: " + it, e); + } + } + } + + return artwork; + } + + + public JSONObject request(String resource, Map parameters, Locale locale, final FloodLimit limit) throws IOException { + // default parameters + LinkedHashMap data = new LinkedHashMap(); + if (parameters != null) { + data.putAll(parameters); + } + if (locale != null && !locale.getLanguage().isEmpty()) { + data.put("language", locale.getLanguage()); + } + data.put("api_key", apikey); + + URL url = new URL("http", host, "/" + version + "/" + resource + "?" + encodeParameters(data)); + + CachedResource json = new CachedResource(url.toString(), String.class, 7 * 24 * 60 * 60 * 1000) { + + @Override + public String process(ByteBuffer data) throws Exception { + return Charset.forName("UTF-8").decode(data).toString(); + } + + + @Override + protected ByteBuffer fetchData(URL url, long lastModified) throws IOException { + if (limit != null) { + try { + limit.acquirePermit(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return super.fetchData(url, lastModified); + } + }; + + return (JSONObject) JSONValue.parse(json.get()); + } + + + protected List jsonList(final Object array) { + return new AbstractList() { + + @Override + public JSONObject get(int index) { + return (JSONObject) ((JSONArray) array).get(index); + } + + + @Override + public int size() { + return ((JSONArray) array).size(); + } + }; } public static class MovieInfo implements Serializable { public static enum MovieProperty { - translated, - adult, - language, - original_name, - name, - type, - id, - imdb_id, - url, - overview, - votes, - rating, - tagline, - certification, - released, - runtime + adult, backdrop_path, budget, homepage, id, imdb_id, original_title, overview, popularity, poster_path, release_date, revenue, runtime, tagline, title, vote_average, vote_count, certification, collection } - protected Map fields; + protected String[] genres; - protected Person[] cast; - protected Artwork[] images; + protected String[] spokenLanguages; + + protected Person[] people; protected MovieInfo() { @@ -269,11 +284,11 @@ public class TMDbClient implements MovieIdentificationService { } - protected MovieInfo(Map fields, List genres, List cast, List images) { + protected MovieInfo(Map fields, List genres, List spokenLanguages, List people) { this.fields = new EnumMap(fields); this.genres = genres.toArray(new String[0]); - this.cast = cast.toArray(new Person[0]); - this.images = images.toArray(new Artwork[0]); + this.spokenLanguages = spokenLanguages.toArray(new String[0]); + this.people = people.toArray(new Person[0]); } @@ -287,19 +302,18 @@ public class TMDbClient implements MovieIdentificationService { } - public boolean isTranslated() { - return Boolean.valueOf(get(MovieProperty.translated)); - } - - public boolean isAdult() { return Boolean.valueOf(get(MovieProperty.adult)); } - public Locale getLanguage() { + public List getSpokenLanguages() { try { - return new Locale(get(MovieProperty.language)); + List locales = new ArrayList(); + for (String it : spokenLanguages) { + locales.add(new Locale(it)); + } + return locales; } catch (Exception e) { return null; } @@ -307,17 +321,12 @@ public class TMDbClient implements MovieIdentificationService { public String getOriginalName() { - return get(MovieProperty.original_name); + return get(MovieProperty.original_title); } public String getName() { - return get(MovieProperty.name); - } - - - public String getType() { - return get(MovieProperty.type); + return get(MovieProperty.title); } @@ -340,9 +349,9 @@ public class TMDbClient implements MovieIdentificationService { } - public URL getUrl() { + public URL getHomepage() { try { - return new URL(get(MovieProperty.url)); + return new URL(get(MovieProperty.homepage)); } catch (Exception e) { return null; } @@ -356,7 +365,7 @@ public class TMDbClient implements MovieIdentificationService { public Integer getVotes() { try { - return new Integer(get(MovieProperty.votes)); + return new Integer(get(MovieProperty.vote_count)); } catch (Exception e) { return null; } @@ -365,7 +374,7 @@ public class TMDbClient implements MovieIdentificationService { public Double getRating() { try { - return new Double(get(MovieProperty.rating)); + return new Double(get(MovieProperty.vote_average)); } catch (Exception e) { return null; } @@ -383,9 +392,15 @@ public class TMDbClient implements MovieIdentificationService { } + public String getCollection() { + // e.g. Star Wars Collection + return get(MovieProperty.collection); + } + + public Date getReleased() { // e.g. 2005-09-30 - return Date.parse(get(MovieProperty.released), "yyyy-MM-dd"); + return Date.parse(get(MovieProperty.release_date), "yyyy-MM-dd"); } @@ -404,12 +419,12 @@ public class TMDbClient implements MovieIdentificationService { public List getCast() { - return unmodifiableList(asList(cast)); + return unmodifiableList(asList(people)); } public String getDirector() { - for (Person person : cast) { + for (Person person : people) { if (person.isDirector()) return person.getName(); } @@ -419,7 +434,7 @@ public class TMDbClient implements MovieIdentificationService { public List getActors() { List actors = new ArrayList(); - for (Person person : cast) { + for (Person person : people) { if (person.isActor()) { actors.add(person.getName()); } @@ -428,99 +443,6 @@ public class TMDbClient implements MovieIdentificationService { } - public List getImages() { - return unmodifiableList(asList(images)); - } - - - @Override - public String toString() { - return fields.toString(); - } - } - - - public static class Artwork implements Serializable { - - public static enum ArtworkProperty { - type, - url, - size, - width, - height - } - - - protected Map fields; - - - protected Artwork() { - // used by serializer - } - - - public Artwork(Map fields) { - this.fields = new EnumMap(fields); - } - - - public Artwork(String type, String url, String size, String width, String height) { - fields = new EnumMap(ArtworkProperty.class); - fields.put(ArtworkProperty.type, type); - fields.put(ArtworkProperty.url, url); - fields.put(ArtworkProperty.size, size); - fields.put(ArtworkProperty.width, width); - fields.put(ArtworkProperty.height, height); - } - - - public String get(Object key) { - return fields.get(ArtworkProperty.valueOf(key.toString())); - } - - - public String get(ArtworkProperty key) { - return fields.get(key); - } - - - public String getType() { - return get(ArtworkProperty.type); - } - - - public URL getUrl() { - try { - return new URL(get(ArtworkProperty.url)); - } catch (Exception e) { - return null; - } - } - - - public String getSize() { - return get(ArtworkProperty.size); - } - - - public Integer getWidth() { - try { - return new Integer(get(ArtworkProperty.width)); - } catch (Exception e) { - return null; - } - } - - - public Integer getHeight() { - try { - return new Integer(get(ArtworkProperty.height)); - } catch (Exception e) { - return null; - } - } - - @Override public String toString() { return fields.toString(); @@ -531,14 +453,9 @@ public class TMDbClient implements MovieIdentificationService { public static class Person implements Serializable { public static enum PersonProperty { - name, - character, - job, - thumb, - department + name, character, job } - protected Map fields; @@ -552,13 +469,11 @@ public class TMDbClient implements MovieIdentificationService { } - public Person(String name, String character, String job, String thumb, String department) { + public Person(String name, String character, String job) { fields = new EnumMap(PersonProperty.class); fields.put(PersonProperty.name, name); fields.put(PersonProperty.character, character); fields.put(PersonProperty.job, job); - fields.put(PersonProperty.thumb, thumb); - fields.put(PersonProperty.department, department); } @@ -587,22 +502,8 @@ public class TMDbClient implements MovieIdentificationService { } - public String getDepartment() { - return get(PersonProperty.department); - } - - - public URL getThumb() { - try { - return new URL(get(PersonProperty.thumb)); - } catch (Exception e) { - return null; - } - } - - public boolean isActor() { - return "Actor".equals(getJob()); + return getJob() == null; } @@ -617,4 +518,56 @@ public class TMDbClient implements MovieIdentificationService { } } + + public static class Artwork { + + private String category; + private String language; + + private int width; + private int height; + + private URL url; + + + public Artwork(String category, URL url, int width, int height, String language) { + this.category = category; + this.url = url; + this.width = width; + this.height = height; + this.language = language; + } + + + public String getCategory() { + return category; + } + + + public String getLanguage() { + return language; + } + + + public int getWidth() { + return width; + } + + + public int getHeight() { + return height; + } + + + public URL getUrl() { + return url; + } + + + @Override + public String toString() { + return String.format("{category: %s, width: %s, height: %s, language: %s, url: %s}", category, width, height, language, url); + } + } + } diff --git a/test/net/sourceforge/filebot/web/SublightSubtitleClientTest.java b/test/net/sourceforge/filebot/web/SublightSubtitleClientTest.java index 45e04da0..719ce688 100644 --- a/test/net/sourceforge/filebot/web/SublightSubtitleClientTest.java +++ b/test/net/sourceforge/filebot/web/SublightSubtitleClientTest.java @@ -9,12 +9,12 @@ import java.util.List; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import net.sublight.webservice.Subtitle; + import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import net.sublight.webservice.Subtitle; - public class SublightSubtitleClientTest { @@ -46,7 +46,7 @@ public class SublightSubtitleClientTest { @Test public void getSubtitleListEnglish() { - List list = client.getSubtitleList(new Movie("Heroes", 2006, 813715), "English"); + List list = client.getSubtitleList(new Movie("Heroes", 2006, 813715, -1), "English"); SubtitleDescriptor sample = list.get(0); assertEquals("English", sample.getLanguageName()); @@ -58,7 +58,7 @@ public class SublightSubtitleClientTest { @Test public void getSubtitleListAllLanguages() { - List list = client.getSubtitleList(new Movie("Terminator 2", 1991, 103064), "Croatian"); + List list = client.getSubtitleList(new Movie("Terminator 2", 1991, 103064, -1), "Croatian"); SubtitleDescriptor sample = list.get(0); diff --git a/test/net/sourceforge/filebot/web/TMDbClientTest.java b/test/net/sourceforge/filebot/web/TMDbClientTest.java index f7ce2f8b..c12e6efc 100644 --- a/test/net/sourceforge/filebot/web/TMDbClientTest.java +++ b/test/net/sourceforge/filebot/web/TMDbClientTest.java @@ -8,10 +8,11 @@ import static org.junit.Assert.*; import java.util.List; import java.util.Locale; -import org.junit.Test; - +import net.sourceforge.filebot.web.TMDbClient.Artwork; import net.sourceforge.filebot.web.TMDbClient.MovieInfo; +import org.junit.Test; + public class TMDbClientTest { @@ -25,18 +26,8 @@ public class TMDbClientTest { assertEquals("冲出宁静号", movie.getName()); assertEquals(2005, movie.getYear()); - assertEquals(379786, movie.getImdbId()); - } - - - @Test - public void searchByHash() throws Exception { - List results = tmdb.searchMovie("907172e7fe51ba57", 742086656, Locale.ENGLISH); - Movie movie = results.get(0); - - assertEquals("Sin City", movie.getName()); - assertEquals(2005, movie.getYear(), 0); - assertEquals(401792, movie.getImdbId(), 0); + assertEquals(-1, movie.getImdbId()); + assertEquals(16320, movie.getTmdbId()); } @@ -47,20 +38,31 @@ public class TMDbClientTest { assertEquals("Transformers", movie.getName()); assertEquals(2007, movie.getYear(), 0); assertEquals(418279, movie.getImdbId(), 0); + assertEquals(1858, movie.getTmdbId(), 0); } @Test public void getMovieInfo() throws Exception { - MovieInfo movie = tmdb.getMovieInfo(new Movie(null, 0, 418279), Locale.ENGLISH); + MovieInfo movie = tmdb.getMovieInfo(new Movie(null, 0, 418279, -1), Locale.ENGLISH); assertEquals("Transformers", movie.getName()); assertEquals("2007-07-03", movie.getReleased().toString()); - assertEquals("Adventure", movie.getGenres().get(0)); - assertEquals("Deborah Lynn Scott", movie.getCast().get(0).getName()); - assertEquals("Costume Design", movie.getCast().get(0).getJob()); - assertEquals("thumb", movie.getImages().get(0).getSize()); - assertEquals("http://cf2.imgobject.com/t/p/w92/bgSHbGEA1OM6qDs3Qba4VlSZsNG.jpg", movie.getImages().get(0).getUrl().toString()); + assertEquals("PG-13", movie.getCertification()); + assertEquals("[Action, Adventure, Science Fiction, Thriller]", movie.getGenres().toString()); + assertEquals("[en]", movie.getSpokenLanguages().toString()); + assertEquals("Shia LaBeouf", movie.getActors().get(0)); + assertEquals("Michael Bay", movie.getDirector()); + assertEquals("Paul Rubell", movie.getCast().get(30).getName()); + assertEquals("Editor", movie.getCast().get(30).getJob()); + } + + + @Test + public void getArtwork() throws Exception { + List artwork = tmdb.getArtwork("tt0418279"); + assertEquals("backdrops", artwork.get(0).getCategory()); + assertEquals("http://cf2.imgobject.com/t/p/original/p4OHBbXfxToWF4e36uEhQMSidWu.jpg", artwork.get(0).getUrl().toString()); } @@ -68,7 +70,7 @@ public class TMDbClientTest { public void floodLimit() throws Exception { for (Locale it : Locale.getAvailableLocales()) { List results = tmdb.searchMovie("Serenity", it); - assertEquals(379786, results.get(0).getImdbId()); + assertEquals(16320, results.get(0).getTmdbId()); } } diff --git a/website/scripts/artwork.tmdb.groovy b/website/scripts/artwork.tmdb.groovy index 93ba78fe..3e52b506 100644 --- a/website/scripts/artwork.tmdb.groovy +++ b/website/scripts/artwork.tmdb.groovy @@ -1,56 +1,11 @@ -// filebot -script "http://filebot.sf.net/scripts/artwork.tmdb.groovy" -trust-script /path/to/media/ +// filebot -script fn:artwork.tmdb /path/to/movies/ /* * Fetch movie artwork. The movie is determined using the parent folders name. */ -def fetchArtwork(outputFile, movieInfo, artworkType, artworkSize) { - // select and fetch artwork - def artwork = movieInfo.images.find { it.type == artworkType && it.size == artworkSize } - if (artwork == null) { - println "Artwork not found: $outputFile" - return null - } - - println "Fetching $outputFile => $artwork" - return artwork.url.saveAs(outputFile) -} - - -def fetchNfo(outputFile, movieInfo) { - movieInfo.applyXmlTemplate(''' - $name - $released.year - $rating - $votes - $overview - $runtime - $certification - ${!genres.empty ? genres[0] : ''} - tt${imdbId.pad(7)} - - ''') - .replaceAll(/\t|\r|\n/, '') // xbmc can't handle leading/trailing whitespace properly - .saveAs(outputFile) -} - - -def fetchMovieArtworkAndNfo(movieDir, movie) { - println "Fetch nfo and artwork for $movie" - def movieInfo = TheMovieDB.getMovieInfo(movie, Locale.ENGLISH) - - println movieInfo - movieInfo.images.each { - println "Available artwork: $it.url => $it" - } - - // fetch nfo - fetchNfo(movieDir['movie.nfo'], movieInfo) - - // fetch series banner, fanart, posters, etc - fetchArtwork(movieDir['folder.jpg'], movieInfo, 'poster', 'original') - fetchArtwork(movieDir['backdrop.jpg'], movieInfo, 'backdrop', 'original') -} +// xbmc artwork/nfo utility +include("fn:lib/xbmc") args.eachMediaFolder { dir -> @@ -72,7 +27,7 @@ args.eachMediaFolder { dir -> // sort by relevance options = options.sortBySimilarity(query, { it.name }) - // auto-select series + // auto-select movie def movie = options[0] // maybe require user input diff --git a/website/scripts/lib/xbmc.groovy b/website/scripts/lib/xbmc.groovy index 86df55ff..d7865717 100644 --- a/website/scripts/lib/xbmc.groovy +++ b/website/scripts/lib/xbmc.groovy @@ -93,15 +93,16 @@ def fetchSeriesArtworkAndNfo(seriesDir, seasonDir, series, season, locale = _arg // functions for TheMovieDB artwork/nfo -def fetchMovieArtwork(outputFile, movieInfo, artworkType, artworkSize) { +def fetchMovieArtwork(outputFile, movieInfo, category, language) { // select and fetch artwork - def artwork = movieInfo.images.find { it.type == artworkType && it.size == artworkSize } - if (artwork == null) { + def artwork = TheMovieDB.getArtwork(movieInfo.id as String) + def selection = [language, 'en', null].findResult{ l -> artwork.find{ (l == it.language || l == null) && it.category == category } } + if (selection == null) { println "Artwork not found: $outputFile" return null } - println "Fetching $outputFile => $artwork" - return artwork.url.saveAs(outputFile) + println "Fetching $outputFile => $selection" + return selection.url.saveAs(outputFile) } def fetchMovieNfo(outputFile, movieInfo) { @@ -124,12 +125,13 @@ def fetchMovieNfo(outputFile, movieInfo) { def fetchMovieArtworkAndNfo(movieDir, movie, locale = _args.locale) { try { def movieInfo = TheMovieDB.getMovieInfo(movie, locale) + // fetch nfo fetchMovieNfo(movieDir['movie.nfo'], movieInfo) // fetch series banner, fanart, posters, etc - fetchMovieArtwork(movieDir['folder.jpg'], movieInfo, 'poster', 'original') - fetchMovieArtwork(movieDir['backdrop.jpg'], movieInfo, 'backdrop', 'original') + fetchMovieArtwork(movieDir['folder.jpg'], movieInfo, 'posters', locale.language) + fetchMovieArtwork(movieDir['backdrop.jpg'], movieInfo, 'backdrops', locale.language) } catch(e) { println "${e.class.simpleName}: ${e.message}" }