moparisthebest/filebot
1
0
Fork 0
mirror of https://github.com/mitb-archive/filebot synced 2024-12-23 16:28:51 -05:00
Code Issues Releases Wiki Activity

* massive performance improvements

Browse Source 
* proper parallel processing in movie mode
This commit is contained in:
Reinhard Pointner 2012-01-02 09:33:50 +00:00
parent 9c8e720f2a
commit 9e6883b646
2 changed files with 88 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
source/net/sourceforge/filebot/media/MediaDetection.java
View File

@ -25,9 +25,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.WeakHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
@ -206,39 +204,27 @@ public class MediaDetection {
private static List<Movie> matchMovieName(final List<String> files, final Locale locale, final boolean strict) throws Exception {
// cross-reference file / folder name with movie list
final SeriesNameMatcher nameMatcher = new SeriesNameMatcher(String.CASE_INSENSITIVE_ORDER); // use simple comparator for speed (2-3x faster)
final Map<Movie, String> matchMap = synchronizedMap(new HashMap<Movie, String>());
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
final HighPerformanceMatcher nameMatcher = new HighPerformanceMatcher();
final Map<Movie, String> matchMap = new HashMap<Movie, String>();
for (final Movie movie : releaseInfo.getMovieList()) {
executor.submit(new Runnable() {
@Override
public void run() {
for (String name : files) {
String movieIdentifier = movie.getName();
String commonName = nameMatcher.matchByFirstCommonWordSequence(name, movieIdentifier);
if (commonName != null && commonName.length() >= movieIdentifier.length()) {
String strictMovieIdentifier = movie.getName() + " " + movie.getYear();
String strictCommonName = nameMatcher.matchByFirstCommonWordSequence(name, strictMovieIdentifier);
if (strictCommonName != null && strictCommonName.length() >= strictMovieIdentifier.length()) {
// prefer strict match
matchMap.put(movie, strictCommonName);
} else if (!strict) {
// make sure the common identifier is not just the year
matchMap.put(movie, commonName);
}
}
for (String name : files) {
String movieIdentifier = movie.getName();
String commonName = nameMatcher.matchByFirstCommonWordSequence(name, movieIdentifier);
if (commonName != null && commonName.length() >= movieIdentifier.length()) {
String strictMovieIdentifier = movie.getName() + " " + movie.getYear();
String strictCommonName = nameMatcher.matchByFirstCommonWordSequence(name, strictMovieIdentifier);
if (strictCommonName != null && strictCommonName.length() >= strictMovieIdentifier.length()) {
// prefer strict match
matchMap.put(movie, strictCommonName);
} else if (!strict) {
// make sure the common identifier is not just the year
matchMap.put(movie, commonName);
}
}
});
}
}
// wait for last task to finish
executor.shutdown();
executor.awaitTermination(1, TimeUnit.MINUTES);
// sort by length of name match (descending)
List<Movie> results = new ArrayList<Movie>(matchMap.keySet());
sort(results, new Comparator<Movie>() {
@ -365,6 +351,7 @@ public class MediaDetection {
}
@SuppressWarnings("unchecked")
public static Comparator<String> getLenientCollator(Locale locale) {
// use maximum strength collator by default
final Collator collator = Collator.getInstance(locale);
@ -373,4 +360,30 @@ public class MediaDetection {
return (Comparator) collator;
}
/*
* Heavy-duty name matcher used for matching a file to or more movies (out of a list of ~50k)
*/
private static class HighPerformanceMatcher extends SeriesNameMatcher {
private static final Map<String, String> transformCache = synchronizedMap(new WeakHashMap<String, String>(65536));
public HighPerformanceMatcher() {
super(String.CASE_INSENSITIVE_ORDER); // 3-4x faster than a Collator
}
@Override
protected String normalize(String source) {
String value = transformCache.get(source);
if (value == null) {
value = super.normalize(source);
transformCache.put(source, value);
}
return transformCache.get(source);
}
}
}

66
source/net/sourceforge/filebot/ui/rename/MovieHashMatcher.java
View File

@ -11,6 +11,7 @@ import static net.sourceforge.tuned.ui.TunedUtilities.*;
import java.awt.Component;
import java.io.File;
import java.util.AbstractMap.SimpleEntry;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -26,6 +27,9 @@ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.RunnableFuture;
@ -53,7 +57,7 @@ class MovieHashMatcher implements AutoCompleteMatcher {
@Override
public List<Match<File, ?>> match(final List<File> files, Locale locale, boolean autodetect, Component parent) throws Exception {
public List<Match<File, ?>> match(final List<File> files, final Locale locale, final boolean autodetect, final Component parent) throws Exception {
// handle movie files
File[] movieFiles = filter(files, VIDEO_FILES).toArray(new File[0]);
File[] subtitleFiles = filter(files, SUBTITLE_FILES).toArray(new File[0]);
@ -73,31 +77,51 @@ class MovieHashMatcher implements AutoCompleteMatcher {
// map movies to (possibly multiple) files (in natural order)
Map<Movie, SortedSet<File>> filesByMovie = new HashMap<Movie, SortedSet<File>>();
// match remaining movies file by file in parallel
List<Callable<Entry<File, Movie>>> grabMovieJobs = new ArrayList<Callable<Entry<File, Movie>>>();
// map all files by movie
for (int i = 0; i < movieFiles.length; i++) {
Movie movie = movieByFileHash[i];
// unknown hash, try via imdb id from nfo file
if (movie == null || !autodetect) {
movie = grabMovieName(movieFiles[i], locale, autodetect, parent, movie);
final Movie movie = movieByFileHash[i];
final File file = movieFiles[i];
grabMovieJobs.add(new Callable<Entry<File, Movie>>() {
if (movie != null) {
Analytics.trackEvent(service.getName(), "SearchMovie", movie.toString(), 1);
@Override
public Entry<File, Movie> call() throws Exception {
// unknown hash, try via imdb id from nfo file
if (movie == null || !autodetect) {
Movie result = grabMovieName(file, locale, autodetect, parent, movie);
if (result != null) {
Analytics.trackEvent(service.getName(), "SearchMovie", result.toString(), 1);
}
return new SimpleEntry<File, Movie>(file, result);
}
return null;
}
});
}
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
try {
for (Future<Entry<File, Movie>> it : executor.invokeAll(grabMovieJobs)) {
// check if we managed to lookup the movie descriptor
if (it.get() != null) {
File file = it.get().getKey();
Movie movie = it.get().getValue();
// get file list for movie
SortedSet<File> movieParts = filesByMovie.get(movie);
if (movieParts == null) {
movieParts = new TreeSet<File>();
filesByMovie.put(movie, movieParts);
}
movieParts.add(file);
}
}
// check if we managed to lookup the movie descriptor
if (movie != null) {
// get file list for movie
SortedSet<File> movieParts = filesByMovie.get(movie);
if (movieParts == null) {
movieParts = new TreeSet<File>();
filesByMovie.put(movie, movieParts);
}
movieParts.add(movieFiles[i]);
}
} finally {
executor.shutdown();
}
// collect all File/MoviePart matches