* lots of fixes and improvement for movie renaming (cmd & gui)

This commit is contained in:
Reinhard Pointner 2011-12-30 15:34:02 +00:00
parent a097daf079
commit f818ca5124
7 changed files with 242 additions and 124 deletions

View File

@ -3,6 +3,7 @@ package net.sourceforge.filebot.cli;
import static java.lang.String.*;
import static java.util.Arrays.*;
import static java.util.Collections.*;
import static net.sourceforge.filebot.MediaTypes.*;
import static net.sourceforge.filebot.WebServices.*;
@ -21,7 +22,6 @@ import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
@ -31,6 +31,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.Callable;
@ -51,6 +52,7 @@ import net.sourceforge.filebot.similarity.Match;
import net.sourceforge.filebot.similarity.Matcher;
import net.sourceforge.filebot.similarity.NameSimilarityMetric;
import net.sourceforge.filebot.similarity.SeriesNameMatcher;
import net.sourceforge.filebot.similarity.SimilarityComparator;
import net.sourceforge.filebot.similarity.SimilarityMetric;
import net.sourceforge.filebot.similarity.StrictEpisodeMetrics;
import net.sourceforge.filebot.subtitle.SubtitleFormat;
@ -62,6 +64,7 @@ import net.sourceforge.filebot.web.EpisodeFormat;
import net.sourceforge.filebot.web.EpisodeListProvider;
import net.sourceforge.filebot.web.Movie;
import net.sourceforge.filebot.web.MovieIdentificationService;
import net.sourceforge.filebot.web.MoviePart;
import net.sourceforge.filebot.web.SearchResult;
import net.sourceforge.filebot.web.SubtitleDescriptor;
import net.sourceforge.filebot.web.SubtitleProvider;
@ -257,81 +260,112 @@ public class CmdlineOperations implements CmdlineInterface {
}
public List<File> renameMovie(Collection<File> mediaFiles, String query, ExpressionFormat format, MovieIdentificationService db, Locale locale, boolean strict) throws Exception {
CLILogger.config(format("Rename movies using [%s]", db.getName()));
public List<File> renameMovie(Collection<File> mediaFiles, String query, ExpressionFormat format, MovieIdentificationService service, Locale locale, boolean strict) throws Exception {
CLILogger.config(format("Rename movies using [%s]", service.getName()));
// handle movie files
File[] movieFiles = filter(mediaFiles, VIDEO_FILES).toArray(new File[0]);
File[] subtitleFiles = filter(mediaFiles, SUBTITLE_FILES).toArray(new File[0]);
Movie[] movieDescriptors;
Movie[] movieByFileHash = null;
if (movieFiles.length > 0) {
if (movieFiles.length > 0 && query == null) {
// match movie hashes online
CLILogger.fine(format("Looking up movie by filehash via [%s]", db.getName()));
movieDescriptors = db.getMovieDescriptors(movieFiles, locale);
} else {
// allow subtitles without video files
movieDescriptors = new Movie[subtitleFiles.length];
CLILogger.fine(format("Looking up movie by filehash via [%s]", service.getName()));
movieByFileHash = service.getMovieDescriptors(movieFiles, locale);
Analytics.trackEvent(service.getName(), "HashLookup", "Movie", movieByFileHash.length - frequency(asList(movieByFileHash), null)); // number of positive hash lookups
}
if (subtitleFiles.length > 0 && movieFiles.length == 0) {
// special handling if there is only subtitle files
movieByFileHash = new Movie[subtitleFiles.length];
movieFiles = subtitleFiles;
subtitleFiles = new File[0];
}
// fill in movie information from nfo file imdb when necessary
if (query != null) {
CLILogger.fine(format("Looking up movie by query [%s]", query));
Movie result = (Movie) selectSearchResult(query, service.searchMovie(query, locale), strict).get(0);
fill(movieByFileHash, result);
}
// map movies to (possibly multiple) files (in natural order)
Map<Movie, SortedSet<File>> filesByMovie = new HashMap<Movie, SortedSet<File>>();
// map all files by movie
for (int i = 0; i < movieFiles.length; i++) {
if (movieDescriptors[i] == null) {
Set<Integer> imdbid = grepImdbIdFor(movieFiles[i]);
if (imdbid.size() > 1) {
CLILogger.warning(String.format("Multiple imdb ids found for %s: %s", movieFiles[i].getName(), imdbid));
Movie movie = movieByFileHash[i];
// unknown hash, try via imdb id from nfo file
if (movie == null) {
Collection<Movie> results = detectMovie(movieFiles[i], service, locale, strict);
movie = (Movie) selectSearchResult(query, results, strict).get(0);
if (movie != null) {
Analytics.trackEvent(service.getName(), "SearchMovie", movie.toString(), 1);
}
}
// check if we managed to lookup the movie descriptor
if (movie != null) {
// get file list for movie
SortedSet<File> movieParts = filesByMovie.get(movie);
if (movieParts == null) {
movieParts = new TreeSet<File>();
filesByMovie.put(movie, movieParts);
}
if (imdbid.size() == 1 || (imdbid.size() > 1 && !strict)) {
movieDescriptors[i] = db.getMovieDescriptor(imdbid.iterator().next(), locale);
CLILogger.fine(String.format("Identified %s as %s via imdb id", movieFiles[i].getName(), movieDescriptors[i]));
}
movieParts.add(movieFiles[i]);
}
}
// use user query if search by hash did not return any results, only one query for one movie though
if (query != null && movieDescriptors.length == 1 && movieDescriptors[0] == null) {
CLILogger.fine(format("Looking up movie by query [%s]", query));
movieDescriptors[0] = (Movie) selectSearchResult(query, db.searchMovie(query, locale), strict).get(0);
// collect all File / MoviePart matches
List<Match<File, ?>> matches = new ArrayList<Match<File, ?>>();
for (Entry<Movie, SortedSet<File>> entry : filesByMovie.entrySet()) {
Movie movie = entry.getKey();
int partIndex = 0;
int partCount = entry.getValue().size();
// add all movie parts
for (File file : entry.getValue()) {
Movie part = movie;
if (partCount > 1) {
part = new MoviePart(movie, ++partIndex, partCount);
}
matches.add(new Match<File, Movie>(file, part));
}
}
// handle subtitle files
for (File subtitle : subtitleFiles) {
// check if subtitle corresponds to a movie file (same name, different extension)
for (Match<File, ?> movieMatch : matches) {
if (isDerived(subtitle, movieMatch.getValue())) {
matches.add(new Match<File, Object>(subtitle, movieMatch.getCandidate()));
// movie match found, we're done
break;
}
}
}
// map old files to new paths by applying formatting and validating filenames
Map<File, File> renameMap = new LinkedHashMap<File, File>();
for (int i = 0; i < movieFiles.length; i++) {
if (movieDescriptors[i] != null) {
Movie movie = movieDescriptors[i];
File file = movieFiles[i];
String newName = (format != null) ? format.format(new MediaBindingBean(movie, file)) : movie.toString();
File newFile = new File(newName + "." + getExtension(file));
if (isInvalidFilePath(newFile)) {
CLILogger.config("Stripping invalid characters from new path: " + newName);
newFile = validateFilePath(newFile);
}
renameMap.put(file, newFile);
} else {
CLILogger.warning("No matching movie: " + movieFiles[i]);
}
}
// handle subtitle files
for (File subtitleFile : subtitleFiles) {
// check if subtitle corresponds to a movie file (same name, different extension)
for (int i = 0; i < movieDescriptors.length; i++) {
if (movieDescriptors[i] != null) {
if (isDerived(subtitleFile, movieFiles[i])) {
File movieDestination = renameMap.get(movieFiles[i]);
File subtitleDestination = new File(movieDestination.getParentFile(), getName(movieDestination) + "." + getExtension(subtitleFile));
renameMap.put(subtitleFile, subtitleDestination);
// movie match found, we're done
break;
}
}
for (Match<File, ?> match : matches) {
File file = match.getValue();
Object movie = match.getCandidate();
String newName = (format != null) ? format.format(new MediaBindingBean(movie, file)) : movie.toString();
File newFile = new File(newName + "." + getExtension(file));
if (isInvalidFilePath(newFile)) {
CLILogger.config("Stripping invalid characters from new path: " + newName);
newFile = validateFilePath(newFile);
}
renameMap.put(file, newFile);
}
// rename movies
@ -340,7 +374,7 @@ public class CmdlineOperations implements CmdlineInterface {
}
private List<File> renameAll(Map<File, File> renameMap) throws Exception {
public List<File> renameAll(Map<File, File> renameMap) throws Exception {
// rename files
final List<Entry<File, File>> renameLog = new ArrayList<Entry<File, File>>();
@ -389,8 +423,9 @@ public class CmdlineOperations implements CmdlineInterface {
// new file names
List<File> destinationList = new ArrayList<File>();
for (Entry<File, File> it : renameLog)
for (Entry<File, File> it : renameLog) {
destinationList.add(it.getValue());
}
return destinationList;
}
@ -615,21 +650,12 @@ public class CmdlineOperations implements CmdlineInterface {
// sort results by similarity to query
List<SearchResult> results = new ArrayList<SearchResult>(probableMatches.values());
sort(results, new Comparator<SearchResult>() {
@Override
public int compare(SearchResult o1, SearchResult o2) {
float f1 = metric.getSimilarity(o1.getName(), query);
float f2 = metric.getSimilarity(o2.getName(), query);
return f1 > f2 ? f1 == f2 ? 0 : -1 : 1;
}
});
sort(results, new SimilarityComparator(query));
return results;
}
private List<SearchResult> selectSearchResult(String query, Iterable<? extends SearchResult> searchResults, boolean strict) throws Exception {
public List<SearchResult> selectSearchResult(String query, Iterable<? extends SearchResult> searchResults, boolean strict) throws Exception {
List<SearchResult> probableMatches = findProbableMatches(query, searchResults);
if (probableMatches.isEmpty() || (strict && probableMatches.size() != 1)) {

View File

@ -2,10 +2,13 @@
package net.sourceforge.filebot.media;
import static java.util.Arrays.*;
import static java.util.Collections.*;
import static net.sourceforge.filebot.MediaTypes.*;
import static net.sourceforge.tuned.FileUtilities.*;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
@ -29,6 +32,9 @@ import java.util.regex.Pattern;
import net.sourceforge.filebot.MediaTypes;
import net.sourceforge.filebot.WebServices;
import net.sourceforge.filebot.similarity.SeriesNameMatcher;
import net.sourceforge.filebot.similarity.SimilarityComparator;
import net.sourceforge.filebot.web.Movie;
import net.sourceforge.filebot.web.MovieIdentificationService;
import net.sourceforge.filebot.web.SearchResult;
import net.sourceforge.filebot.web.TheTVDBClient.TheTVDBSearchResult;
@ -125,7 +131,7 @@ public class MediaDetection {
Collection<String> matches = new SeriesNameMatcher().matchAll(files.toArray(new File[files.size()]));
try {
matches = new ReleaseInfo().cleanRG(matches);
matches = stripReleaseInfo(matches);
} catch (Exception e) {
Logger.getLogger(MediaDetection.class.getClass().getName()).log(Level.WARNING, "Failed to clean matches: " + e.getMessage(), e);
}
@ -138,6 +144,48 @@ public class MediaDetection {
}
public static Collection<Movie> detectMovie(File movieFile, MovieIdentificationService service, Locale locale, boolean strict) throws Exception {
Set<Movie> options = new LinkedHashSet<Movie>();
// try to grep imdb id from nfo files
for (int imdbid : grepImdbIdFor(movieFile)) {
Movie movie = service.getMovieDescriptor(imdbid, locale);
if (movie != null) {
options.add(movie);
}
}
// search by file name or folder name
Collection<String> searchQueries = new LinkedHashSet<String>();
searchQueries.add(getName(movieFile));
searchQueries.add(getName(movieFile.getParentFile()));
// remove blacklisted terms
searchQueries = stripReleaseInfo(searchQueries);
if (!strict && options.isEmpty()) {
for (String query : searchQueries) {
Movie[] results = service.searchMovie(query, locale).toArray(new Movie[0]);
sort(results, new SimilarityComparator(query)); // sort by similarity to original query
addAll(options, results);
}
}
return options;
}
public static String stripReleaseInfo(String name) throws IOException {
return new ReleaseInfo().cleanRelease(name);
}
public static List<String> stripReleaseInfo(Collection<String> names) throws IOException {
return new ReleaseInfo().cleanRelease(names);
}
public static Set<Integer> grepImdbIdFor(File file) throws Exception {
Set<Integer> collection = new LinkedHashSet<Integer>();

View File

@ -13,6 +13,9 @@ import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -50,23 +53,13 @@ public class ReleaseInfo {
}
public List<String> clean(Iterable<String> items) throws IOException {
return clean(items, getVideoSourcePattern(), getCodecPattern());
public List<String> cleanRelease(Iterable<String> items) throws IOException {
return clean(items, getReleaseGroupPattern(), getLanguageSuffixPattern(), getVideoSourcePattern(), getCodecPattern(), getResolutionPattern());
}
public String clean(String item) throws IOException {
return clean(item, getVideoSourcePattern(), getCodecPattern());
}
public List<String> cleanRG(Iterable<String> items) throws IOException {
return clean(items, getReleaseGroupPattern(), getVideoSourcePattern(), getCodecPattern());
}
public String cleanRG(String item) throws IOException {
return clean(item, getReleaseGroupPattern(), getVideoSourcePattern(), getCodecPattern());
public String cleanRelease(String item) throws IOException {
return clean(item, getReleaseGroupPattern(), getLanguageSuffixPattern(), getVideoSourcePattern(), getCodecPattern(), getResolutionPattern());
}
@ -89,6 +82,30 @@ public class ReleaseInfo {
}
public Pattern getLanguageSuffixPattern() {
Set<String> tokens = new TreeSet<String>();
for (String code : Locale.getISOLanguages()) {
Locale locale = new Locale(code);
tokens.add(locale.getLanguage());
tokens.add(locale.getISO3Language());
tokens.add(locale.getDisplayLanguage(Locale.ENGLISH));
}
// remove illegal tokens
tokens.remove("");
// .{language}[.srt]
return compile("(?<=[.])(" + join(tokens, "|") + ")(?=$)", CASE_INSENSITIVE);
}
public Pattern getResolutionPattern() {
// match screen resolutions 640x480, 1280x720, etc
return compile("(?<!\\p{Alnum})(\\d{4}|[6-9]\\d{2})x(\\d{4}|[4-9]\\d{2})(?!\\p{Alnum})");
}
public Pattern getCodecPattern() {
// pattern matching any video source name
String pattern = getBundle(getClass().getName()).getString("pattern.codec");

View File

@ -0,0 +1,36 @@
package net.sourceforge.filebot.similarity;
import java.util.Comparator;
public class SimilarityComparator implements Comparator<Object> {
private SimilarityMetric metric;
private Object paragon;
public SimilarityComparator(SimilarityMetric metric, Object paragon) {
this.metric = metric;
this.paragon = paragon;
}
public SimilarityComparator(String paragon) {
this(new NameSimilarityMetric(), paragon);
}
@Override
public int compare(Object o1, Object o2) {
float f1 = metric.getSimilarity(o1, paragon);
float f2 = metric.getSimilarity(o2, paragon);
if (f1 == f2)
return 0;
return f1 > f2 ? -1 : 1;
}
}

View File

@ -33,7 +33,6 @@ import javax.swing.Action;
import javax.swing.SwingUtilities;
import net.sourceforge.filebot.Analytics;
import net.sourceforge.filebot.media.ReleaseInfo;
import net.sourceforge.filebot.similarity.EpisodeMetrics;
import net.sourceforge.filebot.similarity.Match;
import net.sourceforge.filebot.similarity.Matcher;
@ -234,7 +233,7 @@ class EpisodeListMatcher implements AutoCompleteMatcher {
String suggestion = new SeriesNameMatcher().matchBySeasonEpisodePattern(getName(files.get(0)));
if (suggestion != null) {
// clean media info / release group info / etc
suggestion = new ReleaseInfo().cleanRG(suggestion);
suggestion = stripReleaseInfo(suggestion);
} else {
// use folder name
suggestion = files.get(0).getParentFile().getName();

View File

@ -16,10 +16,12 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.Callable;
@ -30,7 +32,6 @@ import javax.swing.Action;
import javax.swing.SwingUtilities;
import net.sourceforge.filebot.Analytics;
import net.sourceforge.filebot.media.ReleaseInfo;
import net.sourceforge.filebot.similarity.Match;
import net.sourceforge.filebot.ui.SelectDialog;
import net.sourceforge.filebot.web.Movie;
@ -52,10 +53,19 @@ class MovieHashMatcher implements AutoCompleteMatcher {
public List<Match<File, ?>> match(final List<File> files, Locale locale, boolean autodetect, Component parent) throws Exception {
// handle movie files
File[] movieFiles = filter(files, VIDEO_FILES).toArray(new File[0]);
File[] subtitleFiles = filter(files, SUBTITLE_FILES).toArray(new File[0]);
Movie[] movieByFileHash = null;
// match movie hashes online
Movie[] movieByFileHash = service.getMovieDescriptors(movieFiles, locale);
Analytics.trackEvent(service.getName(), "HashLookup", "Movie", movieByFileHash.length - frequency(asList(movieByFileHash), null)); // number of positive hash lookups
if (movieFiles.length > 0) {
// match movie hashes online
movieByFileHash = service.getMovieDescriptors(movieFiles, locale);
Analytics.trackEvent(service.getName(), "HashLookup", "Movie", movieByFileHash.length - frequency(asList(movieByFileHash), null)); // number of positive hash lookups
} else if (subtitleFiles.length > 0) {
// special handling if there is only subtitle files
movieByFileHash = new Movie[subtitleFiles.length];
movieFiles = subtitleFiles;
subtitleFiles = new File[0];
}
// map movies to (possibly multiple) files (in natural order)
Map<Movie, SortedSet<File>> filesByMovie = new HashMap<Movie, SortedSet<File>>();
@ -108,10 +118,10 @@ class MovieHashMatcher implements AutoCompleteMatcher {
}
// handle subtitle files
for (File subtitle : filter(files, SUBTITLE_FILES)) {
for (File subtitle : subtitleFiles) {
// check if subtitle corresponds to a movie file (same name, different extension)
for (Match<File, ?> movieMatch : matches) {
if (isDerived(getName(subtitle), movieMatch.getValue())) {
if (isDerived(subtitle, movieMatch.getValue())) {
matches.add(new Match<File, Object>(subtitle, movieMatch.getCandidate()));
// movie match found, we're done
break;
@ -133,7 +143,7 @@ class MovieHashMatcher implements AutoCompleteMatcher {
protected Movie grabMovieName(File movieFile, Locale locale, boolean autodetect, Component parent, Movie... suggestions) throws Exception {
List<Movie> options = new ArrayList<Movie>();
Set<Movie> options = new LinkedHashSet<Movie>();
// add default value if any
for (Movie it : suggestions) {
@ -142,52 +152,33 @@ class MovieHashMatcher implements AutoCompleteMatcher {
}
}
// try to grep imdb id from nfo files
for (int imdbid : grepImdbIdFor(movieFile)) {
Movie movie = service.getMovieDescriptor(imdbid, locale);
if (movie != null) {
options.add(movie);
}
}
// search by file name or folder name
Collection<String> searchQueries = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
searchQueries.add(getName(movieFile));
searchQueries.add(getName(movieFile.getParentFile()));
// remove blacklisted terms
searchQueries = new ReleaseInfo().cleanRG(searchQueries);
for (String query : searchQueries) {
if (autodetect && options.isEmpty()) {
options = service.searchMovie(query, locale);
}
}
// auto-detect movie from nfo or folder / file name
options.addAll(detectMovie(movieFile, service, locale, false));
// allow manual user input
if (options.isEmpty() || !autodetect) {
String suggestion = options.isEmpty() ? searchQueries.iterator().next() : options.get(0).getName();
String suggestion = options.isEmpty() ? stripReleaseInfo(getName(movieFile)) : options.iterator().next().getName();
String input = null;
synchronized (this) {
input = showInputDialog("Enter movie name:", suggestion, searchQueries.iterator().next(), parent);
input = showInputDialog("Enter movie name:", suggestion, movieFile.getPath(), parent);
}
// we only care about results from manual input from here on out
options.clear();
if (input != null) {
options = service.searchMovie(input, locale);
} else {
options.clear(); // cancel search
options.addAll(service.searchMovie(input, locale));
}
}
return options.isEmpty() ? null : selectMovie(options, parent);
return options.isEmpty() ? null : selectMovie(movieFile, options, parent);
}
protected Movie selectMovie(final List<Movie> options, final Component parent) throws Exception {
protected Movie selectMovie(final File movieFile, final Collection<Movie> options, final Component parent) throws Exception {
if (options.size() == 1) {
return options.get(0);
return options.iterator().next();
}
// show selection dialog on EDT
@ -198,7 +189,8 @@ class MovieHashMatcher implements AutoCompleteMatcher {
// multiple results have been found, user must select one
SelectDialog<Movie> selectDialog = new SelectDialog<Movie>(parent, options);
selectDialog.getHeaderLabel().setText("Select Movie:");
selectDialog.setTitle(movieFile.getPath());
selectDialog.getHeaderLabel().setText(String.format("Movies matching '%s':", stripReleaseInfo(getName(movieFile))));
selectDialog.getCancelAction().putValue(Action.NAME, "Ignore");
// show dialog

View File

@ -134,7 +134,7 @@ public class OpenSubtitlesXmlRpc {
try {
String imdbid = movie.get("id");
if (!imdbid.matches("\\d{1,7}"))
throw new IllegalArgumentException("Illegal IMDbID");
throw new IllegalArgumentException("Illegal IMDb movie ID");
// match movie name and movie year from search result
Matcher matcher = pattern.matcher(movie.get("title"));