diff --git a/source/net/sourceforge/filebot/web/OpenSubtitlesClient.java b/source/net/sourceforge/filebot/web/OpenSubtitlesClient.java index 5557e0d1..58fa4bf5 100644 --- a/source/net/sourceforge/filebot/web/OpenSubtitlesClient.java +++ b/source/net/sourceforge/filebot/web/OpenSubtitlesClient.java @@ -17,6 +17,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -134,6 +135,29 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS @Override public Map> getSubtitleList(File[] files, String languageName) throws Exception { + Map> results = new HashMap>(files.length); + Set remainingFiles = new LinkedHashSet(asList(files)); + + // lookup subtitles by hash + if (remainingFiles.size() > 0) { + results.putAll(getSubtitleListByHash(remainingFiles.toArray(new File[0]), languageName)); + } + + for (Entry> it : results.entrySet()) { + if (it.getValue().size() > 0) { + remainingFiles.remove(it.getKey()); + } + } + + // lookup subtitles by tag + if (remainingFiles.size() > 0) { + results.putAll(getSubtitleListByTag(remainingFiles.toArray(new File[0]), languageName)); + } + + return results; + } + + public Map> getSubtitleListByHash(File[] files, String languageName) throws Exception { // singleton array with or empty array String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName) } : new String[0]; @@ -142,10 +166,10 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS Map> resultMap = new HashMap>(files.length); // create hash query for each file - List queryList = new ArrayList(files.length); + List hashQueryList = new ArrayList(files.length); for (File file : files) { - // add query + // add hash query if (file.length() > HASH_CHUNK_SIZE) { String movieHash = computeHash(file); Query query = Query.forHash(movieHash, file.length(), languageFilter); @@ -153,7 +177,7 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS // check hash List cachedResults = getCache().getSubtitleDescriptorList(query, languageName); if (cachedResults == null) { - queryList.add(query); + hashQueryList.add(query); hashMap.put(query, file); } else { resultMap.put(file, cachedResults); @@ -166,22 +190,26 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS } } - if (queryList.size() > 0) { + if (hashQueryList.size() > 0) { // require login login(); // dispatch query for all hashes int batchSize = 50; - for (int bn = 0; bn < ceil((float) queryList.size() / batchSize); bn++) { - List batch = queryList.subList(bn * batchSize, min((bn * batchSize) + batchSize, queryList.size())); + for (int bn = 0; bn < ceil((float) hashQueryList.size() / batchSize); bn++) { + List batch = hashQueryList.subList(bn * batchSize, min((bn * batchSize) + batchSize, hashQueryList.size())); // submit query and map results to given files for (OpenSubtitlesSubtitleDescriptor subtitle : xmlrpc.searchSubtitles(batch)) { // get file for hash - File file = hashMap.get(Query.forHash(subtitle.getMovieHash(), subtitle.getMovieByteSize(), languageFilter)); + File file = hashMap.get((batch.get(subtitle.getQueryNumber()))); // add subtitle - resultMap.get(file).add(subtitle); + if (file != null) { + resultMap.get(file).add(subtitle); + } else { + Logger.getLogger(getClass().getName()).log(Level.WARNING, "Unable to map hash to file: " + subtitle.getMovieHash()); + } } for (Query query : batch) { @@ -193,6 +221,68 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS return resultMap; } + public Map> getSubtitleListByTag(File[] files, String languageName) throws Exception { + // singleton array with or empty array + String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName) } : new String[0]; + + // remember tag for each file + Map tagMap = new HashMap(files.length); + Map> resultMap = new HashMap>(files.length); + + // create tag query for each file + List tagQueryList = new ArrayList(files.length); + + for (File file : files) { + // add tag query + String tag = getNameWithoutExtension(file.getName()); + Query query = Query.forTag(tag, languageFilter); + + // check tag + List cachedResults = getCache().getSubtitleDescriptorList(query, languageName); + if (cachedResults == null) { + tagQueryList.add(query); + tagMap.put(query, file); + } else { + resultMap.put(file, cachedResults); + } + + // prepare result map + if (resultMap.get(file) == null) { + resultMap.put(file, new LinkedList()); + } + } + + if (tagQueryList.size() > 0) { + // require login + login(); + + // dispatch query for all hashes + int batchSize = 50; + for (int bn = 0; bn < ceil((float) tagQueryList.size() / batchSize); bn++) { + List batch = tagQueryList.subList(bn * batchSize, min((bn * batchSize) + batchSize, tagQueryList.size())); + + // submit query and map results to given files + for (OpenSubtitlesSubtitleDescriptor subtitle : xmlrpc.searchSubtitles(batch)) { + // get file for tag + File file = tagMap.get(batch.get(subtitle.getQueryNumber())); + + // add subtitle + if (file != null) { + resultMap.get(file).add(subtitle); + } else { + Logger.getLogger(getClass().getName()).log(Level.WARNING, "Unable to map release name to file: " + subtitle.getMovieReleaseName()); + } + } + + for (Query query : batch) { + getCache().putSubtitleDescriptorList(query, languageName, resultMap.get(tagMap.get(query))); + } + } + } + + return resultMap; + } + @Override public CheckResult checkSubtitle(File videoFile, File subtitleFile) throws Exception { // subhash (md5 of subtitles), subfilename, moviehash, moviebytesize, moviefilename diff --git a/source/net/sourceforge/filebot/web/OpenSubtitlesSubtitleDescriptor.java b/source/net/sourceforge/filebot/web/OpenSubtitlesSubtitleDescriptor.java index 2a28087d..8d4c6d2f 100644 --- a/source/net/sourceforge/filebot/web/OpenSubtitlesSubtitleDescriptor.java +++ b/source/net/sourceforge/filebot/web/OpenSubtitlesSubtitleDescriptor.java @@ -23,7 +23,7 @@ import net.sourceforge.tuned.FileUtilities; public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor, Serializable { public static enum Property { - IDSubtitle, IDSubtitleFile, IDSubMovieFile, IDMovie, IDMovieImdb, SubFileName, SubFormat, SubHash, SubSize, MovieHash, MovieByteSize, MovieName, MovieNameEng, MovieYear, MovieReleaseName, MovieTimeMS, MovieFPS, MovieImdbRating, MovieKind, SeriesSeason, SeriesEpisode, SeriesIMDBParent, SubLanguageID, ISO639, LanguageName, UserID, UserRank, UserNickName, SubAddDate, SubAuthorComment, SubFeatured, SubComments, SubDownloadsCnt, SubHearingImpaired, SubRating, SubHD, SubBad, SubActualCD, SubSumCD, MatchedBy, SubtitlesLink, SubDownloadLink, ZipDownloadLink; + IDSubtitle, IDSubtitleFile, IDSubMovieFile, IDMovie, IDMovieImdb, SubFileName, SubFormat, SubHash, SubSize, MovieHash, MovieByteSize, MovieName, MovieNameEng, MovieYear, MovieReleaseName, MovieTimeMS, MovieFPS, MovieImdbRating, MovieKind, SeriesSeason, SeriesEpisode, SeriesIMDBParent, SubLanguageID, ISO639, LanguageName, UserID, UserRank, UserNickName, SubAddDate, SubAuthorComment, SubFeatured, SubComments, SubDownloadsCnt, SubHearingImpaired, SubRating, SubHD, SubBad, SubActualCD, SubSumCD, MatchedBy, QueryNumber, SubtitlesLink, SubDownloadLink, ZipDownloadLink; public static EnumMap asEnumMap(Map stringMap) { EnumMap enumMap = new EnumMap(Property.class); @@ -47,6 +47,10 @@ public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor, Seri this.properties = properties; } + public Map getProperties() { + return properties; + } + public String getProperty(Property key) { return properties.get(key); } @@ -84,6 +88,14 @@ public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor, Seri return Long.parseLong(getProperty(Property.MovieByteSize)); } + public String getMovieReleaseName() { + return getProperty(Property.MovieReleaseName); + } + + public int getQueryNumber() { + return Integer.parseInt(getProperty(Property.QueryNumber)); + } + @Override public ByteBuffer fetch() throws Exception { URL resource = new URL(getProperty(Property.SubDownloadLink)); diff --git a/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java b/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java index abc3faf1..2b51d73c 100644 --- a/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java +++ b/source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java @@ -377,23 +377,27 @@ public class OpenSubtitlesXmlRpc { public static final class Query extends HashMap implements Serializable { - private Query(String imdbid, String... sublanguageids) { - put("imdbid", imdbid); - put("sublanguageid", join(sublanguageids, ",")); - } - - private Query(String moviehash, String moviebytesize, String... sublanguageids) { - put("moviehash", moviehash); - put("moviebytesize", moviebytesize); + private Query(String... sublanguageids) { put("sublanguageid", join(sublanguageids, ",")); } public static Query forHash(String moviehash, long moviebytesize, String... sublanguageids) { - return new Query(moviehash, Long.toString(moviebytesize), sublanguageids); + Query query = new Query(sublanguageids); + query.put("moviehash", moviehash); + query.put("moviebytesize", Long.toString(moviebytesize)); + return query; + } + + public static Query forTag(String tag, String... sublanguageids) { + Query query = new Query(sublanguageids); + query.put("tag", tag); + return query; } public static Query forImdbId(int imdbid, String... sublanguageids) { - return new Query(Integer.toString(imdbid), sublanguageids); + Query query = new Query(sublanguageids); + query.put("imdbid", Integer.toString(imdbid)); + return query; } } diff --git a/test/net/sourceforge/filebot/web/OpenSubtitlesXmlRpcTest.java b/test/net/sourceforge/filebot/web/OpenSubtitlesXmlRpcTest.java index 1f12839d..ecf69a9c 100644 --- a/test/net/sourceforge/filebot/web/OpenSubtitlesXmlRpcTest.java +++ b/test/net/sourceforge/filebot/web/OpenSubtitlesXmlRpcTest.java @@ -1,7 +1,5 @@ - package net.sourceforge.filebot.web; - import static java.util.Collections.*; import static net.sourceforge.filebot.Settings.*; import static org.junit.Assert.*; @@ -19,81 +17,74 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; - public class OpenSubtitlesXmlRpcTest { - + private static OpenSubtitlesXmlRpc xmlrpc = new OpenSubtitlesXmlRpc(String.format("%s %s", getApplicationName(), getApplicationVersion())); - - + @BeforeClass public static void login() throws Exception { // login manually xmlrpc.loginAnonymous(); } - - + @Test public void search() throws Exception { List list = xmlrpc.searchMoviesOnIMDB("babylon 5"); Movie sample = list.get(0); - + // check sample entry assertEquals("Babylon 5", sample.getName()); assertEquals(1994, sample.getYear()); assertEquals(105946, sample.getImdbId()); } - - + @Test(expected = IndexOutOfBoundsException.class) public void searchOST() throws Exception { List list = xmlrpc.searchMoviesOnIMDB("Linkin.Park.New.Divide.1280-720p.Transformers.Revenge.of.the.Fallen.ost"); - + // seek to OST entry, expect to fail - for (int i = 0; !list.get(i).getName().contains("Linkin.Park"); i++); + for (int i = 0; !list.get(i).getName().contains("Linkin.Park"); i++) + ; } - - + @Test public void getSubtitleListEnglish() throws Exception { List list = xmlrpc.searchSubtitles(361256, "eng"); - + SubtitleDescriptor sample = list.get(0); - + assertTrue(sample.getName().startsWith("Wonderfalls")); assertEquals("English", sample.getLanguageName()); - + // check size assertTrue(list.size() > 20); } - - + @Test public void getSubtitleListAllLanguages() throws Exception { List list = xmlrpc.searchSubtitles(361256); - + OpenSubtitlesSubtitleDescriptor sample = list.get(75); - + assertEquals("\"Wonderfalls\" Wound-up Penguin", sample.getProperty(Property.MovieName)); assertEquals("German", sample.getProperty(Property.LanguageName)); assertEquals("imdbid", sample.getProperty(Property.MatchedBy)); - + // check size assertTrue(list.size() > 70); } - - + @Test public void getSubtitleListMovieHash() throws Exception { List list = xmlrpc.searchSubtitles(singleton(Query.forHash("2bba5c34b007153b", 717565952, "eng"))); - + OpenSubtitlesSubtitleDescriptor sample = list.get(0); - + assertEquals("firefly.s01e01.serenity.pilot.dvdrip.xvid.srt", sample.getProperty(Property.SubFileName)); assertEquals("English", sample.getProperty(Property.LanguageName)); assertEquals("moviehash", sample.getProperty(Property.MatchedBy)); } - - + @Test public void tryUploadSubtitles() throws Exception { SubFile subtitle = new SubFile(); @@ -102,103 +93,94 @@ public class OpenSubtitlesXmlRpcTest { subtitle.setMovieFileName("firefly.s01e01.serenity.pilot.dvdrip.xvid.avi"); subtitle.setMovieHash("2bba5c34b007153b"); subtitle.setMovieByteSize(717565952); - + TryUploadResponse response = xmlrpc.tryUploadSubtitles(subtitle); - + assertFalse(response.isUploadRequired()); assertEquals("4513264", response.getSubtitleData().get(0).get(Property.IDSubtitle.toString())); assertEquals("eng", response.getSubtitleData().get(0).get(Property.SubLanguageID.toString())); } - - + @Test public void checkSubHash() throws Exception { Map subHashMap = xmlrpc.checkSubHash(singleton("e12715f466ee73c86694b7ab9f311285")); - + assertEquals("247060", subHashMap.values().iterator().next().toString()); assertTrue(1 == subHashMap.size()); } - - + @Test public void checkSubHashInvalid() throws Exception { Map subHashMap = xmlrpc.checkSubHash(singleton("0123456789abcdef0123456789abcdef")); - + assertEquals("0", subHashMap.values().iterator().next().toString()); assertTrue(1 == subHashMap.size()); } - - + @Test public void checkMovieHash() throws Exception { Map results = xmlrpc.checkMovieHash(singleton("d7aa0275cace4410"), 0); Movie movie = results.get("d7aa0275cace4410"); - + assertEquals("Iron Man", movie.getName()); assertEquals(2008, movie.getYear()); assertEquals(371746, movie.getImdbId()); } - - + @Test public void checkMovieHashInvalid() throws Exception { Map results = xmlrpc.checkMovieHash(singleton("0123456789abcdef"), 0); - + // no movie info assertTrue(results.isEmpty()); } - - + @Test public void getIMDBMovieDetails() throws Exception { Movie movie = xmlrpc.getIMDBMovieDetails(371746); - + assertEquals("Iron Man", movie.getName()); assertEquals(2008, movie.getYear()); assertEquals(371746, movie.getImdbId()); } - - + @Test public void getIMDBMovieDetailsInvalid() throws Exception { Movie movie = xmlrpc.getIMDBMovieDetails(371746); - + assertEquals("Iron Man", movie.getName()); assertEquals(2008, movie.getYear()); assertEquals(371746, movie.getImdbId()); } - - + @Test public void detectLanguage() throws Exception { String text = "Only those that are prepared to fire should be fired at."; - + List languages = xmlrpc.detectLanguage(text.getBytes("UTF-8")); - + assertEquals("eng", languages.get(0)); assertTrue(1 == languages.size()); } - - + @Test public void fetchSubtitle() throws Exception { List list = xmlrpc.searchSubtitles(361256, "eng"); - + // check format assertEquals("srt", list.get(0).getType()); - + // fetch subtitle file ByteBuffer data = list.get(0).fetch(); - + // check size assertEquals(48717, data.remaining(), 0); } - - + @AfterClass public static void logout() throws Exception { // logout manually xmlrpc.logout(); } - + }