mirror of
https://github.com/mitb-archive/filebot
synced 2024-11-14 05:15:03 -05:00
95ff15d475
TEST-DATA: [www.Cpasbien.me] Django.Unchained.2012.FRENCH.BDRip.XviD-AYMO\[www.Cpasbien.me] aymo-django.xvid.cd1.avi
470 lines
15 KiB
Java
470 lines
15 KiB
Java
|
|
package net.sourceforge.filebot.media;
|
|
|
|
|
|
import static java.util.Arrays.*;
|
|
import static java.util.Collections.*;
|
|
import static java.util.ResourceBundle.*;
|
|
import static java.util.regex.Pattern.*;
|
|
import static net.sourceforge.filebot.similarity.Normalization.*;
|
|
import static net.sourceforge.tuned.StringUtilities.*;
|
|
|
|
import java.io.File;
|
|
import java.io.FileFilter;
|
|
import java.io.IOException;
|
|
import java.nio.ByteBuffer;
|
|
import java.nio.charset.Charset;
|
|
import java.text.Collator;
|
|
import java.text.Normalizer;
|
|
import java.text.Normalizer.Form;
|
|
import java.util.ArrayList;
|
|
import java.util.Collection;
|
|
import java.util.Comparator;
|
|
import java.util.HashMap;
|
|
import java.util.HashSet;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.List;
|
|
import java.util.Locale;
|
|
import java.util.Map;
|
|
import java.util.Scanner;
|
|
import java.util.Set;
|
|
import java.util.TreeMap;
|
|
import java.util.regex.Matcher;
|
|
import java.util.regex.Pattern;
|
|
import java.util.zip.GZIPInputStream;
|
|
|
|
import net.sourceforge.filebot.web.AnidbClient.AnidbSearchResult;
|
|
import net.sourceforge.filebot.web.CachedResource;
|
|
import net.sourceforge.filebot.web.Movie;
|
|
import net.sourceforge.filebot.web.TheTVDBClient.TheTVDBSearchResult;
|
|
import net.sourceforge.tuned.ByteBufferInputStream;
|
|
|
|
|
|
public class ReleaseInfo {
|
|
|
|
public String getVideoSource(String... strings) {
|
|
// check parent and itself for group names
|
|
return matchLast(getVideoSourcePattern(), getBundle(getClass().getName()).getString("pattern.video.source").split("[|]"), strings);
|
|
}
|
|
|
|
|
|
public String getReleaseGroup(String... strings) throws IOException {
|
|
// check file and folder for release group names
|
|
String[] groups = releaseGroupResource.get();
|
|
|
|
// try case-sensitive match
|
|
String match = matchLast(getReleaseGroupPattern(true), groups, strings);
|
|
|
|
// try case-insensitive match as fallback
|
|
if (match == null) {
|
|
match = matchLast(getReleaseGroupPattern(false), groups, strings);
|
|
}
|
|
|
|
return match;
|
|
}
|
|
|
|
|
|
public Locale getLanguageSuffix(String name) {
|
|
// match locale identifier and lookup Locale object
|
|
Map<String, Locale> languages = getLanguageMap(Locale.ENGLISH, Locale.getDefault());
|
|
|
|
String lang = matchLast(getLanguageSuffixPattern(languages.keySet(), false), null, name);
|
|
if (lang == null)
|
|
return null;
|
|
|
|
return languages.get(lang);
|
|
}
|
|
|
|
|
|
protected String matchLast(Pattern pattern, String[] standardValues, CharSequence... sequence) {
|
|
String lastMatch = null;
|
|
|
|
// match last occurrence
|
|
for (CharSequence name : sequence) {
|
|
if (name == null)
|
|
continue;
|
|
|
|
Matcher matcher = pattern.matcher(name);
|
|
while (matcher.find()) {
|
|
lastMatch = matcher.group();
|
|
}
|
|
}
|
|
|
|
// prefer standard value over matched value
|
|
if (lastMatch != null && standardValues != null) {
|
|
for (String standard : standardValues) {
|
|
if (standard.equalsIgnoreCase(lastMatch)) {
|
|
return standard;
|
|
}
|
|
}
|
|
}
|
|
|
|
return lastMatch;
|
|
}
|
|
|
|
// cached patterns
|
|
private final Map<Boolean, Pattern[]> stopwords = new HashMap<Boolean, Pattern[]>(2);
|
|
private final Map<Boolean, Pattern[]> blacklist = new HashMap<Boolean, Pattern[]>(2);
|
|
|
|
|
|
public List<String> cleanRelease(Collection<String> items, boolean strict) throws IOException {
|
|
Pattern[] stopwords;
|
|
Pattern[] blacklist;
|
|
|
|
// initialize cached patterns
|
|
synchronized (this.stopwords) {
|
|
stopwords = this.stopwords.get(strict);
|
|
blacklist = this.blacklist.get(strict);
|
|
|
|
if (stopwords == null || blacklist == null) {
|
|
Set<String> languages = getLanguageMap(Locale.ENGLISH, Locale.getDefault()).keySet();
|
|
Pattern clutterBracket = getClutterBracketPattern(strict);
|
|
Pattern releaseGroup = getReleaseGroupPattern(strict);
|
|
Pattern languageSuffix = getLanguageSuffixPattern(languages, strict);
|
|
Pattern languageTag = getLanguageTagPattern(languages);
|
|
Pattern videoSource = getVideoSourcePattern();
|
|
Pattern videoFormat = getVideoFormatPattern();
|
|
Pattern resolution = getResolutionPattern();
|
|
Pattern queryBlacklist = getBlacklistPattern();
|
|
|
|
stopwords = new Pattern[] { languageTag, videoSource, videoFormat, resolution, languageSuffix };
|
|
blacklist = new Pattern[] { queryBlacklist, languageTag, clutterBracket, releaseGroup, videoSource, videoFormat, resolution, languageSuffix };
|
|
|
|
// cache compiled patterns for common usage
|
|
this.stopwords.put(strict, stopwords);
|
|
this.blacklist.put(strict, blacklist);
|
|
}
|
|
}
|
|
|
|
List<String> output = new ArrayList<String>(items.size());
|
|
for (String it : items) {
|
|
it = strict ? clean(it, stopwords) : substringBefore(it, stopwords);
|
|
it = normalizePunctuation(clean(it, blacklist));
|
|
|
|
// ignore empty values
|
|
if (it.length() > 0) {
|
|
output.add(it);
|
|
}
|
|
}
|
|
|
|
return output;
|
|
}
|
|
|
|
|
|
public String clean(String item, Pattern... blacklisted) {
|
|
for (Pattern it : blacklisted) {
|
|
item = it.matcher(item).replaceAll("");
|
|
}
|
|
return item;
|
|
}
|
|
|
|
|
|
public String substringBefore(String item, Pattern... stopwords) {
|
|
for (Pattern it : stopwords) {
|
|
Matcher matcher = it.matcher(item);
|
|
if (matcher.find()) {
|
|
String substring = item.substring(0, matcher.start()); // use substring before the matched stopword
|
|
if (normalizePunctuation(substring).length() >= 3) {
|
|
item = substring; // make sure that the substring has enough data
|
|
}
|
|
}
|
|
}
|
|
return item;
|
|
}
|
|
|
|
|
|
public Pattern getLanguageTagPattern(Collection<String> languages) {
|
|
// [en]
|
|
return compile("(?<=[-\\[{(])(" + join(quoteAll(languages), "|") + ")(?=\\p{Punct})", CASE_INSENSITIVE | UNICODE_CASE);
|
|
}
|
|
|
|
|
|
public Pattern getLanguageSuffixPattern(Collection<String> languages, boolean strict) {
|
|
// .en.srt
|
|
return compile("(?<=" + (strict ? "[.]" : "[\\p{Punct}\\p{Space}]") + ")(" + join(quoteAll(languages), "|") + ")(?=[._ ]*$)", (strict ? 0 : CASE_INSENSITIVE) | UNICODE_CASE);
|
|
}
|
|
|
|
|
|
public Pattern getResolutionPattern() {
|
|
// match screen resolutions 640x480, 1280x720, etc
|
|
return compile("(?<!\\p{Alnum})(\\d{4}|[6-9]\\d{2})x(\\d{4}|[4-9]\\d{2})(?!\\p{Alnum})");
|
|
}
|
|
|
|
|
|
public Pattern getVideoFormatPattern() {
|
|
// pattern matching any video source name
|
|
String pattern = getBundle(getClass().getName()).getString("pattern.video.format");
|
|
return compile("(?<!\\p{Alnum})(" + pattern + ")(?!\\p{Alnum})", CASE_INSENSITIVE);
|
|
}
|
|
|
|
|
|
public Pattern getVideoSourcePattern() {
|
|
// pattern matching any video source name
|
|
String pattern = getBundle(getClass().getName()).getString("pattern.video.source");
|
|
return compile("(?<!\\p{Alnum})(" + pattern + ")(?!\\p{Alnum})", CASE_INSENSITIVE);
|
|
}
|
|
|
|
|
|
public Pattern getClutterBracketPattern(boolean strict) {
|
|
// match patterns like [Action, Drama] or {ENG-XViD-MP3-DVDRiP} etc
|
|
String contentFilter = strict ? "[\\p{Space}\\p{Punct}&&[^\\[\\]]]" : "\\p{Alpha}";
|
|
return compile("(?:\\[([^\\[\\]]+?" + contentFilter + "[^\\[\\]]+?)\\])|(?:\\{([^\\{\\}]+?" + contentFilter + "[^\\{\\}]+?)\\})|(?:\\(([^\\(\\)]+?" + contentFilter + "[^\\(\\)]+?)\\))");
|
|
}
|
|
|
|
|
|
public Pattern getReleaseGroupPattern(boolean strict) throws IOException {
|
|
// pattern matching any release group name enclosed in separators
|
|
return compile("(?<!\\p{Alnum})(" + join(releaseGroupResource.get(), "|") + ")(?!\\p{Alnum})", strict ? 0 : CASE_INSENSITIVE | UNICODE_CASE);
|
|
}
|
|
|
|
|
|
public Pattern getBlacklistPattern() throws IOException {
|
|
// pattern matching any release group name enclosed in separators
|
|
return compile("(?<!\\p{Alnum})(" + join(queryBlacklistResource.get(), "|") + ")(?!\\p{Alnum})", CASE_INSENSITIVE | UNICODE_CASE);
|
|
}
|
|
|
|
|
|
public Pattern getExcludePattern() throws IOException {
|
|
// pattern matching any release group name enclosed in separators
|
|
return compile(join(excludeBlacklistResource.get(), "|"), CASE_INSENSITIVE | UNICODE_CASE);
|
|
}
|
|
|
|
|
|
public Movie[] getMovieList() throws IOException {
|
|
return movieListResource.get();
|
|
}
|
|
|
|
|
|
public TheTVDBSearchResult[] getTheTVDBIndex() throws IOException {
|
|
return tvdbIndexResource.get();
|
|
}
|
|
|
|
|
|
public AnidbSearchResult[] getAnidbIndex() throws IOException {
|
|
return anidbIndexResource.get();
|
|
}
|
|
|
|
|
|
public Map<Pattern, String> getSeriesDirectMappings() throws IOException {
|
|
Map<Pattern, String> mappings = new LinkedHashMap<Pattern, String>();
|
|
for (String line : seriesDirectMappingsResource.get()) {
|
|
String[] tsv = line.split("\t", 2);
|
|
if (tsv.length == 2) {
|
|
mappings.put(compile("(?<!\\p{Alnum})(" + tsv[0] + ")(?!\\p{Alnum})", CASE_INSENSITIVE | UNICODE_CASE), tsv[1]);
|
|
}
|
|
}
|
|
return mappings;
|
|
}
|
|
|
|
|
|
public FileFilter getDiskFolderFilter() {
|
|
return new FolderEntryFilter(compile(getBundle(getClass().getName()).getString("pattern.diskfolder.entry")));
|
|
}
|
|
|
|
|
|
public FileFilter getClutterFileFilter() throws IOException {
|
|
return new ClutterFileFilter(getExcludePattern(), 262144000); // only files smaller than 250 MB may be considered clutter
|
|
}
|
|
|
|
// fetch release group names online and try to update the data every other day
|
|
protected final CachedResource<String[]> releaseGroupResource = new PatternResource(getBundle(getClass().getName()).getString("url.release-groups"));
|
|
protected final CachedResource<String[]> queryBlacklistResource = new PatternResource(getBundle(getClass().getName()).getString("url.query-blacklist"));
|
|
protected final CachedResource<String[]> excludeBlacklistResource = new PatternResource(getBundle(getClass().getName()).getString("url.exclude-blacklist"));
|
|
protected final CachedResource<Movie[]> movieListResource = new MovieResource(getBundle(getClass().getName()).getString("url.movie-list"));
|
|
protected final CachedResource<String[]> seriesDirectMappingsResource = new PatternResource(getBundle(getClass().getName()).getString("url.series-mappings"));
|
|
protected final CachedResource<TheTVDBSearchResult[]> tvdbIndexResource = new TheTVDBIndexResource(getBundle(getClass().getName()).getString("url.thetvdb-index"));
|
|
protected final CachedResource<AnidbSearchResult[]> anidbIndexResource = new AnidbIndexResource(getBundle(getClass().getName()).getString("url.anidb-index"));
|
|
|
|
|
|
protected static class PatternResource extends CachedResource<String[]> {
|
|
|
|
public PatternResource(String resource) {
|
|
super(resource, String[].class, 24 * 60 * 60 * 1000); // 24h update interval
|
|
}
|
|
|
|
|
|
@Override
|
|
public String[] process(ByteBuffer data) {
|
|
return compile("\\n").split(Charset.forName("UTF-8").decode(data));
|
|
}
|
|
}
|
|
|
|
|
|
protected static class MovieResource extends CachedResource<Movie[]> {
|
|
|
|
public MovieResource(String resource) {
|
|
super(resource, Movie[].class, 7 * 24 * 60 * 60 * 1000); // check for updates once a week
|
|
}
|
|
|
|
|
|
@Override
|
|
public Movie[] process(ByteBuffer data) throws IOException {
|
|
Scanner scanner = new Scanner(new GZIPInputStream(new ByteBufferInputStream(data)), "UTF-8").useDelimiter("\t|\n");
|
|
|
|
List<Movie> movies = new ArrayList<Movie>();
|
|
while (scanner.hasNext()) {
|
|
int imdbid = scanner.nextInt();
|
|
String name = scanner.next().trim();
|
|
int year = scanner.nextInt();
|
|
movies.add(new Movie(name, year, imdbid, -1));
|
|
}
|
|
|
|
return movies.toArray(new Movie[0]);
|
|
}
|
|
}
|
|
|
|
|
|
protected static class TheTVDBIndexResource extends CachedResource<TheTVDBSearchResult[]> {
|
|
|
|
public TheTVDBIndexResource(String resource) {
|
|
super(resource, TheTVDBSearchResult[].class, 7 * 24 * 60 * 60 * 1000); // check for updates once a week
|
|
}
|
|
|
|
|
|
@Override
|
|
public TheTVDBSearchResult[] process(ByteBuffer data) throws IOException {
|
|
Scanner scanner = new Scanner(new GZIPInputStream(new ByteBufferInputStream(data)), "UTF-8").useDelimiter("\t|\n");
|
|
|
|
List<TheTVDBSearchResult> tvshows = new ArrayList<TheTVDBSearchResult>();
|
|
while (scanner.hasNext() && scanner.hasNextInt()) {
|
|
int id = scanner.nextInt();
|
|
String name = scanner.next().trim();
|
|
tvshows.add(new TheTVDBSearchResult(name, id));
|
|
}
|
|
|
|
return tvshows.toArray(new TheTVDBSearchResult[0]);
|
|
}
|
|
}
|
|
|
|
|
|
protected static class AnidbIndexResource extends CachedResource<AnidbSearchResult[]> {
|
|
|
|
public AnidbIndexResource(String resource) {
|
|
super(resource, AnidbSearchResult[].class, 7 * 24 * 60 * 60 * 1000); // check for updates once a week
|
|
}
|
|
|
|
|
|
@Override
|
|
public AnidbSearchResult[] process(ByteBuffer data) throws IOException {
|
|
Scanner scanner = new Scanner(new GZIPInputStream(new ByteBufferInputStream(data)), "UTF-8").useDelimiter("\t|\n");
|
|
|
|
List<AnidbSearchResult> anime = new ArrayList<AnidbSearchResult>();
|
|
while (scanner.hasNext() && scanner.hasNextInt()) {
|
|
int aid = scanner.nextInt();
|
|
String primaryTitle = scanner.next().trim();
|
|
String englishTitle = scanner.next().trim();
|
|
|
|
if (englishTitle.isEmpty() || englishTitle.equals(primaryTitle)) {
|
|
anime.add(new AnidbSearchResult(aid, primaryTitle, null));
|
|
} else {
|
|
anime.add(new AnidbSearchResult(aid, primaryTitle, singletonMap("en", englishTitle)));
|
|
}
|
|
}
|
|
|
|
return anime.toArray(new AnidbSearchResult[0]);
|
|
}
|
|
}
|
|
|
|
|
|
protected static class FolderEntryFilter implements FileFilter {
|
|
|
|
private final Pattern entryPattern;
|
|
|
|
|
|
public FolderEntryFilter(Pattern entryPattern) {
|
|
this.entryPattern = entryPattern;
|
|
}
|
|
|
|
|
|
@Override
|
|
public boolean accept(File dir) {
|
|
if (dir.isDirectory()) {
|
|
for (String entry : dir.list()) {
|
|
if (entryPattern.matcher(entry).matches()) {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
}
|
|
|
|
|
|
public static class FileFolderNameFilter implements FileFilter {
|
|
|
|
private final Pattern namePattern;
|
|
|
|
|
|
public FileFolderNameFilter(Pattern namePattern) {
|
|
this.namePattern = namePattern;
|
|
}
|
|
|
|
|
|
@Override
|
|
public boolean accept(File file) {
|
|
return (namePattern.matcher(file.getName()).find() || (file.isFile() && namePattern.matcher(file.getParentFile().getName()).find()));
|
|
}
|
|
}
|
|
|
|
|
|
public static class ClutterFileFilter extends FileFolderNameFilter {
|
|
|
|
private long maxFileSize;
|
|
|
|
|
|
public ClutterFileFilter(Pattern namePattern, long maxFileSize) {
|
|
super(namePattern);
|
|
this.maxFileSize = maxFileSize;
|
|
}
|
|
|
|
|
|
@Override
|
|
public boolean accept(File file) {
|
|
return super.accept(file) && file.isFile() && file.length() < maxFileSize;
|
|
}
|
|
}
|
|
|
|
|
|
private Collection<String> quoteAll(Collection<String> strings) {
|
|
List<String> patterns = new ArrayList<String>(strings.size());
|
|
for (String it : strings) {
|
|
patterns.add(Pattern.quote(it));
|
|
}
|
|
return patterns;
|
|
}
|
|
|
|
|
|
private Map<String, Locale> getLanguageMap(Locale... supportedDisplayLocale) {
|
|
// use maximum strength collator by default
|
|
Collator collator = Collator.getInstance(Locale.ROOT);
|
|
collator.setDecomposition(Collator.FULL_DECOMPOSITION);
|
|
collator.setStrength(Collator.PRIMARY);
|
|
|
|
@SuppressWarnings("unchecked")
|
|
Comparator<String> order = (Comparator) collator;
|
|
Map<String, Locale> languageMap = languageMap = new TreeMap<String, Locale>(order);
|
|
|
|
for (String code : Locale.getISOLanguages()) {
|
|
Locale locale = new Locale(code);
|
|
languageMap.put(locale.getLanguage(), locale);
|
|
languageMap.put(locale.getISO3Language(), locale);
|
|
|
|
// map display language names for given locales
|
|
for (Locale language : new HashSet<Locale>(asList(supportedDisplayLocale))) {
|
|
// make sure language name is properly normalized so accents and whatever don't break the regex pattern syntax
|
|
String languageName = Normalizer.normalize(locale.getDisplayLanguage(language), Form.NFKD);
|
|
languageMap.put(languageName.toLowerCase(), locale);
|
|
}
|
|
}
|
|
|
|
// remove illegal tokens
|
|
languageMap.remove("");
|
|
languageMap.remove("II");
|
|
languageMap.remove("III");
|
|
languageMap.remove("hi"); // hi => hearing-impaired subtitles, NOT hindi language
|
|
|
|
Map<String, Locale> result = unmodifiableMap(languageMap);
|
|
return result;
|
|
}
|
|
}
|