moparisthebest
/
filebot
mirror of https://github.com/mitb-archive/filebot
1
0
Fork 0
Code Issues Releases Wiki Activity

* match Movie object for nfo files directly via nfo content 

* enable caching for TMDb
This commit is contained in:
Reinhard Pointner 2012-02-14 14:16:13 +00:00
parent 2bfef63d3c
commit 38ea14d86f
9 changed files with 163 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
source/net/sourceforge/filebot/WebServices.java
View File

@ -27,7 +27,6 @@ public final class WebServices {
// episode dbs
public static final TVRageClient TVRage = new TVRageClient();
public static final AnidbClient AniDB = new AnidbClient(getApplicationName().toLowerCase(), 2);
public static final IMDbClient IMDb = new IMDbClient();
public static final TheTVDBClient TheTVDB = new TheTVDBClient(getApplicationProperty("thetvdb.apikey"));
public static final SerienjunkiesClient Serienjunkies = new SerienjunkiesClient(getApplicationProperty("serienjunkies.apikey"));
@ -37,6 +36,7 @@ public final class WebServices {
public static final SubsceneSubtitleClient Subscene = new SubsceneSubtitleClient();
// movie dbs
public static final IMDbClient IMDb = new IMDbClient();
public static final TMDbClient TMDb = new TMDbClient(getApplicationProperty("themoviedb.apikey"));

100
source/net/sourceforge/filebot/cli/CmdlineOperations.java
View File

@ -29,6 +29,7 @@ import java.util.ListIterator;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
@ -266,15 +267,17 @@ public class CmdlineOperations implements CmdlineInterface {
// handle movie files
List<File> movieFiles = filter(files, VIDEO_FILES);
List<File> nfoFiles = filter(files, MediaTypes.getDefaultFilter("application/nfo"));
List<File> standaloneFiles = new ArrayList<File>(files);
standaloneFiles.removeAll(movieFiles);
List<File> orphanedFiles = new ArrayList<File>(filter(files, FILES));
orphanedFiles.removeAll(movieFiles);
orphanedFiles.removeAll(nfoFiles);
Map<File, List<File>> derivatesByMovieFile = new HashMap<File, List<File>>();
for (File movieFile : movieFiles) {
derivatesByMovieFile.put(movieFile, new ArrayList<File>());
}
for (File file : standaloneFiles) {
for (File file : orphanedFiles) {
for (File movieFile : movieFiles) {
if (isDerived(file, movieFile)) {
derivatesByMovieFile.get(movieFile).add(file);
@ -283,41 +286,49 @@ public class CmdlineOperations implements CmdlineInterface {
}
}
for (List<File> derivates : derivatesByMovieFile.values()) {
standaloneFiles.removeAll(derivates);
orphanedFiles.removeAll(derivates);
}
List<File> movieMatchFiles = new ArrayList<File>();
movieMatchFiles.addAll(movieFiles);
movieMatchFiles.addAll(filter(files, new ReleaseInfo().getDiskFolderFilter()));
movieMatchFiles.addAll(filter(standaloneFiles, SUBTITLE_FILES));
// map movies to (possibly multiple) files (in natural order)
Map<Movie, SortedSet<File>> filesByMovie = new HashMap<Movie, SortedSet<File>>();
// match movie hashes online
Map<File, Movie> movieByFile = new HashMap<File, Movie>();
if (query == null && movieFiles.size() > 0) {
try {
CLILogger.fine(format("Looking up movie by filehash via [%s]", service.getName()));
Map<File, Movie> hashLookup = service.getMovieDescriptors(movieFiles, locale);
movieByFile.putAll(hashLookup);
Analytics.trackEvent(service.getName(), "HashLookup", "Movie", hashLookup.size()); // number of positive hash lookups
} catch (UnsupportedOperationException e) {
CLILogger.fine(format("%s: Hash lookup not supported", service.getName()));
final Map<File, Movie> movieByFile = new HashMap<File, Movie>();
if (query == null) {
if (movieFiles.size() > 0) {
try {
CLILogger.fine(format("Looking up movie by filehash via [%s]", service.getName()));
Map<File, Movie> hashLookup = service.getMovieDescriptors(movieFiles, locale);
movieByFile.putAll(hashLookup);
Analytics.trackEvent(service.getName(), "HashLookup", "Movie", hashLookup.size()); // number of positive hash lookups
} catch (UnsupportedOperationException e) {
CLILogger.fine(format("%s: Hash lookup not supported", service.getName()));
}
}
}
if (query != null) {
for (File nfo : nfoFiles) {
try {
movieByFile.put(nfo, grepMovie(nfo, service, locale));
} catch (NoSuchElementException e) {
CLILogger.warning("Failed to grep IMDbID: " + nfo.getName());
}
}
} else {
CLILogger.fine(format("Looking up movie by query [%s]", query));
Movie result = (Movie) selectSearchResult(query, service.searchMovie(query, locale), strict).get(0);
// force all mappings
for (File file : movieMatchFiles) {
for (File file : files) {
movieByFile.put(file, result);
}
}
List<File> movieMatchFiles = new ArrayList<File>();
movieMatchFiles.addAll(movieFiles);
movieMatchFiles.addAll(nfoFiles);
movieMatchFiles.addAll(filter(files, new ReleaseInfo().getDiskFolderFilter()));
movieMatchFiles.addAll(filter(orphanedFiles, SUBTITLE_FILES)); // run movie detection only on orphaned subtitle files
// map movies to (possibly multiple) files (in natural order)
Map<Movie, SortedSet<File>> filesByMovie = new HashMap<Movie, SortedSet<File>>();
// map all files by movie
for (File file : movieMatchFiles) {
for (final File file : movieMatchFiles) {
Movie movie = movieByFile.get(file);
// unknown hash, try via imdb id from nfo file
@ -345,29 +356,26 @@ public class CmdlineOperations implements CmdlineInterface {
}
}
// collect all File / MoviePart matches
// collect all File/MoviePart matches
List<Match<File, ?>> matches = new ArrayList<Match<File, ?>>();
for (Entry<Movie, SortedSet<File>> entry : filesByMovie.entrySet()) {
Movie movie = entry.getKey();
int partIndex = 0;
int partCount = entry.getValue().size();
// add all movie parts
for (File file : entry.getValue()) {
Movie part = movie;
if (partCount > 1) {
part = new MoviePart(movie, ++partIndex, partCount);
}
matches.add(new Match<File, Movie>(file, part));
// automatically add matches for derivates
List<File> derivates = derivatesByMovieFile.get(file);
if (derivates != null) {
for (File derivate : derivates) {
matches.add(new Match<File, Movie>(derivate, part));
for (List<File> fileSet : mapByExtension(entry.getValue()).values()) {
// resolve movie parts
for (int i = 0; i < fileSet.size(); i++) {
Movie moviePart = entry.getKey();
if (fileSet.size() > 1) {
moviePart = new MoviePart(moviePart, i + 1, fileSet.size());
}
matches.add(new Match<File, Movie>(fileSet.get(i), moviePart));
// automatically add matches for derivate files
List<File> derivates = derivatesByMovieFile.get(fileSet.get(i));
if (derivates != null) {
for (File derivate : derivates) {
matches.add(new Match<File, Movie>(derivate, moviePart));
}
}
}
}

6
source/net/sourceforge/filebot/format/MediaBindingBean.java
View File

@ -378,6 +378,12 @@ public class MediaBindingBean {
}
@Define("folder")
public File getMediaParentFolder() {
return mediaFile.getParentFile();
}
@Define("home")
public File getUserHome() throws IOException {
return new File(System.getProperty("user.home"));

11
source/net/sourceforge/filebot/media/MediaDetection.java
View File

@ -42,6 +42,7 @@ import net.sourceforge.filebot.web.AnidbClient.AnidbSearchResult;
import net.sourceforge.filebot.web.Movie;
import net.sourceforge.filebot.web.MovieIdentificationService;
import net.sourceforge.filebot.web.SearchResult;
import net.sourceforge.filebot.web.TheTVDBClient.SeriesInfo;
import net.sourceforge.filebot.web.TheTVDBClient.TheTVDBSearchResult;
@ -442,6 +443,16 @@ public class MediaDetection {
}
public static Movie grepMovie(File nfo, MovieIdentificationService resolver, Locale locale) throws Exception {
return resolver.getMovieDescriptor(grepImdbId(new String(readFile(nfo), "UTF-8")).iterator().next(), locale);
}
public static SeriesInfo grepSeries(File nfo, Locale locale) throws Exception {
return WebServices.TheTVDB.getSeriesInfoByID(grepTheTvdbId(new String(readFile(nfo), "UTF-8")).iterator().next(), locale);
}
@SuppressWarnings("unchecked")
public static Comparator<String> getLenientCollator(Locale locale) {
// use maximum strength collator by default

91
source/net/sourceforge/filebot/ui/rename/MovieHashMatcher.java
View File

@ -21,6 +21,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@ -30,11 +31,14 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.RunnableFuture;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.Action;
import javax.swing.SwingUtilities;
import net.sourceforge.filebot.Analytics;
import net.sourceforge.filebot.MediaTypes;
import net.sourceforge.filebot.media.ReleaseInfo;
import net.sourceforge.filebot.similarity.Match;
import net.sourceforge.filebot.similarity.NameSimilarityMetric;
@ -60,15 +64,17 @@ class MovieHashMatcher implements AutoCompleteMatcher {
public List<Match<File, ?>> match(final List<File> files, final SortOrder sortOrder, final Locale locale, final boolean autodetect, final Component parent) throws Exception {
// handle movie files
List<File> movieFiles = filter(files, VIDEO_FILES);
List<File> nfoFiles = filter(files, MediaTypes.getDefaultFilter("application/nfo"));
List<File> standaloneFiles = new ArrayList<File>(files);
standaloneFiles.removeAll(movieFiles);
List<File> orphanedFiles = new ArrayList<File>(filter(files, FILES));
orphanedFiles.removeAll(movieFiles);
orphanedFiles.removeAll(nfoFiles);
Map<File, List<File>> derivatesByMovieFile = new HashMap<File, List<File>>();
for (File movieFile : movieFiles) {
derivatesByMovieFile.put(movieFile, new ArrayList<File>());
}
for (File file : standaloneFiles) {
for (File file : orphanedFiles) {
for (File movieFile : movieFiles) {
if (isDerived(file, movieFile)) {
derivatesByMovieFile.get(movieFile).add(file);
@ -77,22 +83,11 @@ class MovieHashMatcher implements AutoCompleteMatcher {
}
}
for (List<File> derivates : derivatesByMovieFile.values()) {
standaloneFiles.removeAll(derivates);
orphanedFiles.removeAll(derivates);
}
List<File> movieMatchFiles = new ArrayList<File>();
movieMatchFiles.addAll(movieFiles);
movieMatchFiles.addAll(filter(files, new ReleaseInfo().getDiskFolderFilter()));
movieMatchFiles.addAll(filter(standaloneFiles, SUBTITLE_FILES));
// map movies to (possibly multiple) files (in natural order)
Map<Movie, SortedSet<File>> filesByMovie = new HashMap<Movie, SortedSet<File>>();
// match remaining movies file by file in parallel
List<Callable<Entry<File, Movie>>> grabMovieJobs = new ArrayList<Callable<Entry<File, Movie>>>();
// match movie hashes online
Map<File, Movie> movieByFile = new HashMap<File, Movie>();
final Map<File, Movie> movieByFile = new HashMap<File, Movie>();
if (movieFiles.size() > 0) {
try {
Map<File, Movie> hashLookup = service.getMovieDescriptors(movieFiles, locale);
@ -102,28 +97,45 @@ class MovieHashMatcher implements AutoCompleteMatcher {
// ignore
}
}
for (File nfo : nfoFiles) {
try {
movieByFile.put(nfo, grepMovie(nfo, service, locale));
} catch (NoSuchElementException e) {
Logger.getLogger(getClass().getName()).log(Level.WARNING, "Failed to grep IMDbID: " + nfo.getName());
}
}
// match remaining movies file by file in parallel
List<Callable<Entry<File, Movie>>> grabMovieJobs = new ArrayList<Callable<Entry<File, Movie>>>();
List<File> movieMatchFiles = new ArrayList<File>();
movieMatchFiles.addAll(movieFiles);
movieMatchFiles.addAll(nfoFiles);
movieMatchFiles.addAll(filter(files, new ReleaseInfo().getDiskFolderFilter()));
movieMatchFiles.addAll(filter(orphanedFiles, SUBTITLE_FILES)); // run movie detection only on orphaned subtitle files
// map all files by movie
for (final File file : movieMatchFiles) {
final Movie movie = movieByFile.get(file);
grabMovieJobs.add(new Callable<Entry<File, Movie>>() {
@Override
public Entry<File, Movie> call() throws Exception {
// unknown hash, try via imdb id from nfo file
if (movie == null || !autodetect) {
Movie result = grabMovieName(file, locale, autodetect, parent, movie);
if (!movieByFile.containsKey(file) || !autodetect) {
Movie result = grabMovieName(file, locale, autodetect, parent, movieByFile.get(file));
if (result != null) {
Analytics.trackEvent(service.getName(), "SearchMovie", result.toString(), 1);
}
return new SimpleEntry<File, Movie>(file, result);
}
return new SimpleEntry<File, Movie>(file, null);
return new SimpleEntry<File, Movie>(file, movieByFile.get(file));
}
});
}
// map movies to (possibly multiple) files (in natural order)
Map<Movie, SortedSet<File>> filesByMovie = new HashMap<Movie, SortedSet<File>>();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
try {
for (Future<Entry<File, Movie>> it : executor.invokeAll(grabMovieJobs)) {
@ -134,12 +146,10 @@ class MovieHashMatcher implements AutoCompleteMatcher {
// get file list for movie
if (movie != null) {
SortedSet<File> movieParts = filesByMovie.get(movie);
if (movieParts == null) {
movieParts = new TreeSet<File>();
filesByMovie.put(movie, movieParts);
}
movieParts.add(file);
}
}
@ -151,25 +161,22 @@ class MovieHashMatcher implements AutoCompleteMatcher {
List<Match<File, ?>> matches = new ArrayList<Match<File, ?>>();
for (Entry<Movie, SortedSet<File>> entry : filesByMovie.entrySet()) {
Movie movie = entry.getKey();
int partIndex = 0;
int partCount = entry.getValue().size();
// add all movie parts
for (File file : entry.getValue()) {
Movie part = movie;
if (partCount > 1) {
part = new MoviePart(movie, ++partIndex, partCount);
}
matches.add(new Match<File, Movie>(file, part));
// automatically add matches for derivates
List<File> derivates = derivatesByMovieFile.get(file);
if (derivates != null) {
for (File derivate : derivates) {
matches.add(new Match<File, Movie>(derivate, part));
for (List<File> fileSet : mapByExtension(entry.getValue()).values()) {
// resolve movie parts
for (int i = 0; i < fileSet.size(); i++) {
Movie moviePart = entry.getKey();
if (fileSet.size() > 1) {
moviePart = new MoviePart(moviePart, i + 1, fileSet.size());
}
matches.add(new Match<File, Movie>(fileSet.get(i), moviePart));
// automatically add matches for derivate files
List<File> derivates = derivatesByMovieFile.get(fileSet.get(i));
if (derivates != null) {
for (File derivate : derivates) {
matches.add(new Match<File, Movie>(derivate, moviePart));
}
}
}
}

39
source/net/sourceforge/filebot/web/TMDbClient.java
View File

@ -5,6 +5,7 @@ package net.sourceforge.filebot.web;
import static java.util.Arrays.*;
import static java.util.Collections.*;
import static net.sourceforge.filebot.web.WebRequest.*;
import static net.sourceforge.tuned.FileUtilities.*;
import static net.sourceforge.tuned.XPathUtilities.*;
import java.io.File;
@ -28,6 +29,9 @@ import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.SAXException;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
import net.sourceforge.filebot.ResourceManager;
import net.sourceforge.filebot.web.TMDbClient.Artwork.ArtworkProperty;
import net.sourceforge.filebot.web.TMDbClient.MovieInfo.MovieProperty;
@ -78,8 +82,8 @@ public class TMDbClient implements MovieIdentificationService {
@Override
public Movie getMovieDescriptor(int imdbid, Locale locale) throws Exception {
URL resource = getResource("Movie.imdbLookup", String.format("tt%07d", imdbid), locale);
Node movie = selectNode("//movie", getDocument(resource));
Document dom = fetchResource("Movie.imdbLookup", String.format("tt%07d", imdbid), locale);
Node movie = selectNode("//movie", dom);
if (movie == null)
return null;
@ -98,9 +102,10 @@ public class TMDbClient implements MovieIdentificationService {
protected List<Movie> getMovies(String method, String parameter, Locale locale) throws IOException, SAXException {
Document dom = fetchResource(method, parameter, locale);
List<Movie> result = new ArrayList<Movie>();
for (Node node : selectNodes("//movie", getDocument(getResource(method, parameter, locale)))) {
for (Node node : selectNodes("//movie", dom)) {
try {
String name = getTextContent("name", node);
@ -120,12 +125,29 @@ public class TMDbClient implements MovieIdentificationService {
}
protected URL getResource(String method, String parameter, Locale locale) throws MalformedURLException {
protected URL getResourceLocation(String method, String parameter, Locale locale) throws MalformedURLException {
// e.g. http://api.themoviedb.org/2.1/Movie.search/en/xml/{apikey}/serenity
return new URL("http", host, "/" + version + "/" + method + "/" + locale.getLanguage() + "/xml/" + apikey + "/" + parameter);
}
protected Document fetchResource(String method, String parameter, Locale locale) throws IOException, SAXException {
URL url = getResourceLocation(method, parameter, locale);
Cache cache = CacheManager.getInstance().getCache("web-persistent-datasource");
Element element = cache.get(url.toString());
if (element != null) {
return WebRequest.getDocument((String) element.getValue());
}
String xml = readAll(WebRequest.getReader(url.openConnection()));
Document dom = getDocument(xml);
cache.put(new Element(url.toString(), xml));
return dom;
}
public MovieInfo getMovieInfo(Movie movie, Locale locale) throws Exception {
if (movie.getImdbId() >= 0) {
return getMovieInfoByIMDbID(movie.getImdbId(), Locale.ENGLISH);
@ -150,15 +172,14 @@ public class TMDbClient implements MovieIdentificationService {
if (imdbid < 0)
throw new IllegalArgumentException("Illegal IMDb ID: " + imdbid);
URL resource = getResource("Movie.imdbLookup", String.format("tt%07d", imdbid), locale);
Document dom = getDocument(resource);
// resolve imdbid to tmdbid
Document dom = fetchResource("Movie.imdbLookup", String.format("tt%07d", imdbid), locale);
// get complete movie info via tmdbid lookup
resource = getResource("Movie.getInfo", selectString("//movie/id", dom), locale);
dom = getDocument(resource);
dom = fetchResource("Movie.getInfo", selectString("//movie/id", dom), locale);
// select info from xml
Node node = selectNode("//movie", getDocument(resource));
Node node = selectNode("//movie", dom);
Map<MovieProperty, String> movieProperties = new EnumMap<MovieProperty, String>(MovieProperty.class);
for (MovieProperty property : MovieProperty.values()) {

5
source/net/sourceforge/filebot/web/TheTVDBClient.java
View File

@ -382,6 +382,11 @@ public class TheTVDBClient extends AbstractEpisodeListProvider {
}
public SeriesInfo getSeriesInfoByID(int thetvdbid, Locale locale) throws Exception {
return getSeriesInfo(new TheTVDBSearchResult(null, thetvdbid), locale);
}
public SeriesInfo getSeriesInfoByIMDbID(int imdbid, Locale locale) throws Exception {
return getSeriesInfo(lookupByIMDbID(imdbid, locale), locale);
}

6
source/net/sourceforge/filebot/web/WebRequest.java
View File

@ -7,6 +7,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
@ -96,6 +97,11 @@ public final class WebRequest {
}
public static Document getDocument(String xml) throws IOException, SAXException {
return getDocument(new InputSource(new StringReader(xml)));
}
public static Document getDocument(InputSource source) throws IOException, SAXException {
try {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();

4
source/net/sourceforge/tuned/FileUtilities.java
View File

@ -253,9 +253,7 @@ public final class FileUtilities {
public static boolean isDerived(String derivate, File prime) {
String withoutTypeSuffix = getNameWithoutExtension(prime.getName());
String withoutPartSuffix = getNameWithoutExtension(withoutTypeSuffix);
String base = (withoutPartSuffix.length() > 2 ? withoutPartSuffix : withoutTypeSuffix).trim().toLowerCase();
String base = getName(prime).trim().toLowerCase();
derivate = derivate.trim().toLowerCase();
return derivate.startsWith(base);
}