mirror of
https://github.com/mitb-archive/filebot
synced 2024-12-24 00:38:52 -05:00
* massive performance improvements
* proper parallel processing in movie mode
This commit is contained in:
parent
9c8e720f2a
commit
9e6883b646
@ -25,9 +25,7 @@ import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.WeakHashMap;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.regex.Matcher;
|
||||
@ -206,39 +204,27 @@ public class MediaDetection {
|
||||
|
||||
private static List<Movie> matchMovieName(final List<String> files, final Locale locale, final boolean strict) throws Exception {
|
||||
// cross-reference file / folder name with movie list
|
||||
final SeriesNameMatcher nameMatcher = new SeriesNameMatcher(String.CASE_INSENSITIVE_ORDER); // use simple comparator for speed (2-3x faster)
|
||||
|
||||
final Map<Movie, String> matchMap = synchronizedMap(new HashMap<Movie, String>());
|
||||
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
||||
final HighPerformanceMatcher nameMatcher = new HighPerformanceMatcher();
|
||||
final Map<Movie, String> matchMap = new HashMap<Movie, String>();
|
||||
|
||||
for (final Movie movie : releaseInfo.getMovieList()) {
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
for (String name : files) {
|
||||
String movieIdentifier = movie.getName();
|
||||
String commonName = nameMatcher.matchByFirstCommonWordSequence(name, movieIdentifier);
|
||||
if (commonName != null && commonName.length() >= movieIdentifier.length()) {
|
||||
String strictMovieIdentifier = movie.getName() + " " + movie.getYear();
|
||||
String strictCommonName = nameMatcher.matchByFirstCommonWordSequence(name, strictMovieIdentifier);
|
||||
if (strictCommonName != null && strictCommonName.length() >= strictMovieIdentifier.length()) {
|
||||
// prefer strict match
|
||||
matchMap.put(movie, strictCommonName);
|
||||
} else if (!strict) {
|
||||
// make sure the common identifier is not just the year
|
||||
matchMap.put(movie, commonName);
|
||||
}
|
||||
}
|
||||
for (String name : files) {
|
||||
String movieIdentifier = movie.getName();
|
||||
String commonName = nameMatcher.matchByFirstCommonWordSequence(name, movieIdentifier);
|
||||
if (commonName != null && commonName.length() >= movieIdentifier.length()) {
|
||||
String strictMovieIdentifier = movie.getName() + " " + movie.getYear();
|
||||
String strictCommonName = nameMatcher.matchByFirstCommonWordSequence(name, strictMovieIdentifier);
|
||||
if (strictCommonName != null && strictCommonName.length() >= strictMovieIdentifier.length()) {
|
||||
// prefer strict match
|
||||
matchMap.put(movie, strictCommonName);
|
||||
} else if (!strict) {
|
||||
// make sure the common identifier is not just the year
|
||||
matchMap.put(movie, commonName);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// wait for last task to finish
|
||||
executor.shutdown();
|
||||
executor.awaitTermination(1, TimeUnit.MINUTES);
|
||||
|
||||
// sort by length of name match (descending)
|
||||
List<Movie> results = new ArrayList<Movie>(matchMap.keySet());
|
||||
sort(results, new Comparator<Movie>() {
|
||||
@ -365,6 +351,7 @@ public class MediaDetection {
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public static Comparator<String> getLenientCollator(Locale locale) {
|
||||
// use maximum strength collator by default
|
||||
final Collator collator = Collator.getInstance(locale);
|
||||
@ -373,4 +360,30 @@ public class MediaDetection {
|
||||
|
||||
return (Comparator) collator;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Heavy-duty name matcher used for matching a file to or more movies (out of a list of ~50k)
|
||||
*/
|
||||
private static class HighPerformanceMatcher extends SeriesNameMatcher {
|
||||
|
||||
private static final Map<String, String> transformCache = synchronizedMap(new WeakHashMap<String, String>(65536));
|
||||
|
||||
|
||||
public HighPerformanceMatcher() {
|
||||
super(String.CASE_INSENSITIVE_ORDER); // 3-4x faster than a Collator
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected String normalize(String source) {
|
||||
String value = transformCache.get(source);
|
||||
if (value == null) {
|
||||
value = super.normalize(source);
|
||||
transformCache.put(source, value);
|
||||
}
|
||||
return transformCache.get(source);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -11,6 +11,7 @@ import static net.sourceforge.tuned.ui.TunedUtilities.*;
|
||||
|
||||
import java.awt.Component;
|
||||
import java.io.File;
|
||||
import java.util.AbstractMap.SimpleEntry;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
@ -26,6 +27,9 @@ import java.util.Set;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.FutureTask;
|
||||
import java.util.concurrent.RunnableFuture;
|
||||
|
||||
@ -53,7 +57,7 @@ class MovieHashMatcher implements AutoCompleteMatcher {
|
||||
|
||||
|
||||
@Override
|
||||
public List<Match<File, ?>> match(final List<File> files, Locale locale, boolean autodetect, Component parent) throws Exception {
|
||||
public List<Match<File, ?>> match(final List<File> files, final Locale locale, final boolean autodetect, final Component parent) throws Exception {
|
||||
// handle movie files
|
||||
File[] movieFiles = filter(files, VIDEO_FILES).toArray(new File[0]);
|
||||
File[] subtitleFiles = filter(files, SUBTITLE_FILES).toArray(new File[0]);
|
||||
@ -73,31 +77,51 @@ class MovieHashMatcher implements AutoCompleteMatcher {
|
||||
// map movies to (possibly multiple) files (in natural order)
|
||||
Map<Movie, SortedSet<File>> filesByMovie = new HashMap<Movie, SortedSet<File>>();
|
||||
|
||||
// match remaining movies file by file in parallel
|
||||
List<Callable<Entry<File, Movie>>> grabMovieJobs = new ArrayList<Callable<Entry<File, Movie>>>();
|
||||
|
||||
// map all files by movie
|
||||
for (int i = 0; i < movieFiles.length; i++) {
|
||||
Movie movie = movieByFileHash[i];
|
||||
final Movie movie = movieByFileHash[i];
|
||||
final File file = movieFiles[i];
|
||||
grabMovieJobs.add(new Callable<Entry<File, Movie>>() {
|
||||
|
||||
// unknown hash, try via imdb id from nfo file
|
||||
if (movie == null || !autodetect) {
|
||||
movie = grabMovieName(movieFiles[i], locale, autodetect, parent, movie);
|
||||
@Override
|
||||
public Entry<File, Movie> call() throws Exception {
|
||||
// unknown hash, try via imdb id from nfo file
|
||||
if (movie == null || !autodetect) {
|
||||
Movie result = grabMovieName(file, locale, autodetect, parent, movie);
|
||||
if (result != null) {
|
||||
Analytics.trackEvent(service.getName(), "SearchMovie", result.toString(), 1);
|
||||
}
|
||||
return new SimpleEntry<File, Movie>(file, result);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (movie != null) {
|
||||
Analytics.trackEvent(service.getName(), "SearchMovie", movie.toString(), 1);
|
||||
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
||||
try {
|
||||
for (Future<Entry<File, Movie>> it : executor.invokeAll(grabMovieJobs)) {
|
||||
// check if we managed to lookup the movie descriptor
|
||||
if (it.get() != null) {
|
||||
File file = it.get().getKey();
|
||||
Movie movie = it.get().getValue();
|
||||
|
||||
// get file list for movie
|
||||
SortedSet<File> movieParts = filesByMovie.get(movie);
|
||||
|
||||
if (movieParts == null) {
|
||||
movieParts = new TreeSet<File>();
|
||||
filesByMovie.put(movie, movieParts);
|
||||
}
|
||||
|
||||
movieParts.add(file);
|
||||
}
|
||||
}
|
||||
|
||||
// check if we managed to lookup the movie descriptor
|
||||
if (movie != null) {
|
||||
// get file list for movie
|
||||
SortedSet<File> movieParts = filesByMovie.get(movie);
|
||||
|
||||
if (movieParts == null) {
|
||||
movieParts = new TreeSet<File>();
|
||||
filesByMovie.put(movie, movieParts);
|
||||
}
|
||||
|
||||
movieParts.add(movieFiles[i]);
|
||||
}
|
||||
} finally {
|
||||
executor.shutdown();
|
||||
}
|
||||
|
||||
// collect all File/MoviePart matches
|
||||
|
Loading…
Reference in New Issue
Block a user