package net.sourceforge.filebot.media; import static java.lang.Integer.*; import static java.util.Arrays.*; import static java.util.Collections.*; import static java.util.ResourceBundle.*; import static java.util.regex.Pattern.*; import static net.sourceforge.filebot.similarity.Normalization.*; import static net.sourceforge.tuned.FileUtilities.*; import static net.sourceforge.tuned.StringUtilities.*; import java.io.File; import java.io.FileFilter; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.text.Collator; import java.text.Normalizer; import java.text.Normalizer.Form; import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; import net.sourceforge.filebot.web.AnidbSearchResult; import net.sourceforge.filebot.web.CachedResource; import net.sourceforge.filebot.web.Movie; import net.sourceforge.filebot.web.TheTVDBSearchResult; import net.sourceforge.tuned.ByteBufferInputStream; import net.sourceforge.tuned.FileUtilities.RegexFileFilter; import org.tukaani.xz.XZInputStream; public class ReleaseInfo { public String getVideoSource(String... strings) { // check parent and itself for group names return matchLast(getVideoSourcePattern(), getBundle(getClass().getName()).getString("pattern.video.source").split("[|]"), strings); } public String getReleaseGroup(String... strings) throws IOException { // check file and folder for release group names String[] groups = releaseGroupResource.get(); // try case-sensitive match String match = matchLast(getReleaseGroupPattern(true), groups, strings); // try case-insensitive match as fallback if (match == null) { match = matchLast(getReleaseGroupPattern(false), groups, strings); } return match; } public Locale getLanguageSuffix(String name) { // match locale identifier and lookup Locale object Map languages = getLanguageMap(Locale.ENGLISH, Locale.getDefault()); String lang = matchLast(getLanguageSuffixPattern(languages.keySet(), false), null, name); if (lang == null) return null; return languages.get(lang); } protected String matchLast(Pattern pattern, String[] standardValues, CharSequence... sequence) { String lastMatch = null; // match last occurrence for (CharSequence name : sequence) { if (name == null) continue; Matcher matcher = pattern.matcher(name); while (matcher.find()) { lastMatch = matcher.group(); } } // prefer standard value over matched value if (lastMatch != null && standardValues != null) { for (String standard : standardValues) { if (standard.equalsIgnoreCase(lastMatch)) { return standard; } } } return lastMatch; } // cached patterns private final Map stopwords = new HashMap(2); private final Map blacklist = new HashMap(2); public List cleanRelease(Collection items, boolean strict) throws IOException { Pattern[] stopwords; Pattern[] blacklist; // initialize cached patterns synchronized (this.stopwords) { stopwords = this.stopwords.get(strict); blacklist = this.blacklist.get(strict); if (stopwords == null || blacklist == null) { Set languages = getLanguageMap(Locale.ENGLISH, Locale.getDefault()).keySet(); Pattern clutterBracket = getClutterBracketPattern(strict); Pattern releaseGroup = getReleaseGroupPattern(strict); Pattern languageSuffix = getLanguageSuffixPattern(languages, strict); Pattern languageTag = getLanguageTagPattern(languages); Pattern videoSource = getVideoSourcePattern(); Pattern videoFormat = getVideoFormatPattern(); Pattern resolution = getResolutionPattern(); Pattern queryBlacklist = getBlacklistPattern(); stopwords = new Pattern[] { languageTag, videoSource, videoFormat, resolution, languageSuffix }; blacklist = new Pattern[] { queryBlacklist, languageTag, clutterBracket, releaseGroup, videoSource, videoFormat, resolution, languageSuffix }; // cache compiled patterns for common usage this.stopwords.put(strict, stopwords); this.blacklist.put(strict, blacklist); } } List output = new ArrayList(items.size()); for (String it : items) { it = strict ? clean(it, stopwords) : substringBefore(it, stopwords); it = normalizePunctuation(clean(it, blacklist)); // ignore empty values if (it.length() > 0) { output.add(it); } } return output; } public String clean(String item, Pattern... blacklisted) { for (Pattern it : blacklisted) { item = it.matcher(item).replaceAll(""); } return item; } public String substringBefore(String item, Pattern... stopwords) { for (Pattern it : stopwords) { Matcher matcher = it.matcher(item); if (matcher.find()) { String substring = item.substring(0, matcher.start()); // use substring before the matched stopword if (normalizePunctuation(substring).length() >= 3) { item = substring; // make sure that the substring has enough data } } } return item; } // cached patterns private Pattern structureRootFolderPattern; public Pattern getStructureRootPattern() throws IOException { if (structureRootFolderPattern == null) { List folders = new ArrayList(); for (String it : queryBlacklistResource.get()) { if (it.startsWith("^")) { folders.add(it); } } structureRootFolderPattern = compile(join(folders, "|"), CASE_INSENSITIVE | UNICODE_CASE); } return structureRootFolderPattern; } public Pattern getLanguageTagPattern(Collection languages) { // [en] return compile("(?<=[-\\[{(])(" + join(quoteAll(languages), "|") + ")(?=\\p{Punct})", CASE_INSENSITIVE | UNICODE_CASE); } public Pattern getLanguageSuffixPattern(Collection languages, boolean strict) { // .en.srt return compile("(?<=[.])(" + join(quoteAll(languages), "|") + ")(?=[._ ]*$)", (strict ? 0 : CASE_INSENSITIVE) | UNICODE_CASE); } public Pattern getResolutionPattern() { // match screen resolutions 640x480, 1280x720, etc return compile("(? getSeriesDirectMappings() throws IOException { Map mappings = new LinkedHashMap(); for (String line : seriesDirectMappingsResource.get()) { String[] tsv = line.split("\t", 2); if (tsv.length == 2) { mappings.put(compile("(? releaseGroupResource = new PatternResource(getBundle(getClass().getName()).getString("url.release-groups")); protected final CachedResource queryBlacklistResource = new PatternResource(getBundle(getClass().getName()).getString("url.query-blacklist")); protected final CachedResource excludeBlacklistResource = new PatternResource(getBundle(getClass().getName()).getString("url.exclude-blacklist")); protected final CachedResource movieListResource = new MovieResource(getBundle(getClass().getName()).getString("url.movie-list")); protected final CachedResource seriesDirectMappingsResource = new PatternResource(getBundle(getClass().getName()).getString("url.series-mappings")); protected final CachedResource tvdbIndexResource = new TheTVDBIndexResource(getBundle(getClass().getName()).getString("url.thetvdb-index")); protected final CachedResource anidbIndexResource = new AnidbIndexResource(getBundle(getClass().getName()).getString("url.anidb-index")); protected static class PatternResource extends CachedResource { public PatternResource(String resource) { super(resource, String[].class, ONE_WEEK); // 1 week update interval } @Override public String[] process(ByteBuffer data) { return compile("\\n").split(Charset.forName("UTF-8").decode(data)); } } protected static class MovieResource extends CachedResource { public MovieResource(String resource) { super(resource, Movie[].class, ONE_MONTH); // check for updates every month } @Override public Movie[] process(ByteBuffer data) throws IOException { List rows = readCSV(new XZInputStream(new ByteBufferInputStream(data)), "UTF-8", "\t"); List movies = new ArrayList(rows.size()); for (String[] row : rows) { int imdbid = parseInt(row[0]); int year = parseInt(row[1]); String name = row[2]; String[] aliasNames = copyOfRange(row, 3, row.length); movies.add(new Movie(name, aliasNames, year, imdbid, -1)); } return movies.toArray(new Movie[0]); } } protected static class TheTVDBIndexResource extends CachedResource { public TheTVDBIndexResource(String resource) { super(resource, TheTVDBSearchResult[].class, ONE_MONTH); // check for updates once a week } @Override public TheTVDBSearchResult[] process(ByteBuffer data) throws IOException { List rows = readCSV(new XZInputStream(new ByteBufferInputStream(data)), "UTF-8", "\t"); List tvshows = new ArrayList(rows.size()); for (String[] row : rows) { int id = parseInt(row[0]); String name = row[1]; String[] aliasNames = copyOfRange(row, 2, row.length); tvshows.add(new TheTVDBSearchResult(name, aliasNames, id)); } return tvshows.toArray(new TheTVDBSearchResult[0]); } } protected static class AnidbIndexResource extends CachedResource { public AnidbIndexResource(String resource) { super(resource, AnidbSearchResult[].class, ONE_MONTH); // check for updates once a week } @Override public AnidbSearchResult[] process(ByteBuffer data) throws IOException { List rows = readCSV(new XZInputStream(new ByteBufferInputStream(data)), "UTF-8", "\t"); List anime = new ArrayList(rows.size()); for (String[] row : rows) { int aid = parseInt(row[0]); String primaryTitle = row[1]; String[] aliasNames = copyOfRange(row, 2, row.length); anime.add(new AnidbSearchResult(aid, primaryTitle, aliasNames)); } return anime.toArray(new AnidbSearchResult[0]); } } protected static class FolderEntryFilter implements FileFilter { private final Pattern entryPattern; public FolderEntryFilter(Pattern entryPattern) { this.entryPattern = entryPattern; } @Override public boolean accept(File dir) { if (dir.isDirectory()) { for (String entry : dir.list()) { if (entryPattern.matcher(entry).matches()) { return true; } } } return false; } } public static class FileFolderNameFilter implements FileFilter { private final Pattern namePattern; public FileFolderNameFilter(Pattern namePattern) { this.namePattern = namePattern; } @Override public boolean accept(File file) { return (namePattern.matcher(file.getName()).find() || (file.isFile() && namePattern.matcher(file.getParentFile().getName()).find())); } } public static class ClutterFileFilter extends FileFolderNameFilter { private long maxFileSize; public ClutterFileFilter(Pattern namePattern, long maxFileSize) { super(namePattern); this.maxFileSize = maxFileSize; } @Override public boolean accept(File file) { return super.accept(file) && file.isFile() && file.length() < maxFileSize; } } private Collection quoteAll(Collection strings) { List patterns = new ArrayList(strings.size()); for (String it : strings) { patterns.add(Pattern.quote(it)); } return patterns; } private Map getLanguageMap(Locale... supportedDisplayLocale) { // use maximum strength collator by default Collator collator = Collator.getInstance(Locale.ROOT); collator.setDecomposition(Collator.FULL_DECOMPOSITION); collator.setStrength(Collator.PRIMARY); @SuppressWarnings("unchecked") Comparator order = (Comparator) collator; Map languageMap = languageMap = new TreeMap(order); for (String code : Locale.getISOLanguages()) { Locale locale = new Locale(code); languageMap.put(locale.getLanguage(), locale); languageMap.put(locale.getISO3Language(), locale); // map display language names for given locales for (Locale language : new HashSet(asList(supportedDisplayLocale))) { // make sure language name is properly normalized so accents and whatever don't break the regex pattern syntax String languageName = Normalizer.normalize(locale.getDisplayLanguage(language), Form.NFKD); languageMap.put(languageName.toLowerCase(), locale); } } // remove illegal tokens languageMap.remove(""); languageMap.remove("II"); languageMap.remove("III"); languageMap.remove("hi"); // hi => hearing-impaired subtitles, NOT hindi language Map result = unmodifiableMap(languageMap); return result; } }