* support OpenSubtitles TAG lookup (i.e. filename) as part of hash lookup if actual hash lookup does not yield any results
This commit is contained in:
Reinhard Pointner
parent
6e732e8987
commit
5fadfbe3e3
|
|
@ -17,6 +17,7 @@ import java.util.Arrays;
|
|||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
|
@ -134,6 +135,29 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
|
|||
|
||||
@Override
|
||||
public Map<File, List<SubtitleDescriptor>> getSubtitleList(File[] files, String languageName) throws Exception {
|
||||
Map<File, List<SubtitleDescriptor>> results = new HashMap<File, List<SubtitleDescriptor>>(files.length);
|
||||
Set<File> remainingFiles = new LinkedHashSet<File>(asList(files));
|
||||
|
||||
// lookup subtitles by hash
|
||||
if (remainingFiles.size() > 0) {
|
||||
results.putAll(getSubtitleListByHash(remainingFiles.toArray(new File[0]), languageName));
|
||||
}
|
||||
|
||||
for (Entry<File, List<SubtitleDescriptor>> it : results.entrySet()) {
|
||||
if (it.getValue().size() > 0) {
|
||||
remainingFiles.remove(it.getKey());
|
||||
}
|
||||
}
|
||||
|
||||
// lookup subtitles by tag
|
||||
if (remainingFiles.size() > 0) {
|
||||
results.putAll(getSubtitleListByTag(remainingFiles.toArray(new File[0]), languageName));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
public Map<File, List<SubtitleDescriptor>> getSubtitleListByHash(File[] files, String languageName) throws Exception {
|
||||
// singleton array with or empty array
|
||||
String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName) } : new String[0];
|
||||
|
||||
|
|
@ -142,10 +166,10 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
|
|||
Map<File, List<SubtitleDescriptor>> resultMap = new HashMap<File, List<SubtitleDescriptor>>(files.length);
|
||||
|
||||
// create hash query for each file
|
||||
List<Query> queryList = new ArrayList<Query>(files.length);
|
||||
List<Query> hashQueryList = new ArrayList<Query>(files.length);
|
||||
|
||||
for (File file : files) {
|
||||
// add query
|
||||
// add hash query
|
||||
if (file.length() > HASH_CHUNK_SIZE) {
|
||||
String movieHash = computeHash(file);
|
||||
Query query = Query.forHash(movieHash, file.length(), languageFilter);
|
||||
|
|
@ -153,7 +177,7 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
|
|||
// check hash
|
||||
List<SubtitleDescriptor> cachedResults = getCache().getSubtitleDescriptorList(query, languageName);
|
||||
if (cachedResults == null) {
|
||||
queryList.add(query);
|
||||
hashQueryList.add(query);
|
||||
hashMap.put(query, file);
|
||||
} else {
|
||||
resultMap.put(file, cachedResults);
|
||||
|
|
@ -166,22 +190,26 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
|
|||
}
|
||||
}
|
||||
|
||||
if (queryList.size() > 0) {
|
||||
if (hashQueryList.size() > 0) {
|
||||
// require login
|
||||
login();
|
||||
|
||||
// dispatch query for all hashes
|
||||
int batchSize = 50;
|
||||
for (int bn = 0; bn < ceil((float) queryList.size() / batchSize); bn++) {
|
||||
List<Query> batch = queryList.subList(bn * batchSize, min((bn * batchSize) + batchSize, queryList.size()));
|
||||
for (int bn = 0; bn < ceil((float) hashQueryList.size() / batchSize); bn++) {
|
||||
List<Query> batch = hashQueryList.subList(bn * batchSize, min((bn * batchSize) + batchSize, hashQueryList.size()));
|
||||
|
||||
// submit query and map results to given files
|
||||
for (OpenSubtitlesSubtitleDescriptor subtitle : xmlrpc.searchSubtitles(batch)) {
|
||||
// get file for hash
|
||||
File file = hashMap.get(Query.forHash(subtitle.getMovieHash(), subtitle.getMovieByteSize(), languageFilter));
|
||||
File file = hashMap.get((batch.get(subtitle.getQueryNumber())));
|
||||
|
||||
// add subtitle
|
||||
resultMap.get(file).add(subtitle);
|
||||
if (file != null) {
|
||||
resultMap.get(file).add(subtitle);
|
||||
} else {
|
||||
Logger.getLogger(getClass().getName()).log(Level.WARNING, "Unable to map hash to file: " + subtitle.getMovieHash());
|
||||
}
|
||||
}
|
||||
|
||||
for (Query query : batch) {
|
||||
|
|
@ -193,6 +221,68 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
|
|||
return resultMap;
|
||||
}
|
||||
|
||||
public Map<File, List<SubtitleDescriptor>> getSubtitleListByTag(File[] files, String languageName) throws Exception {
|
||||
// singleton array with or empty array
|
||||
String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName) } : new String[0];
|
||||
|
||||
// remember tag for each file
|
||||
Map<Query, File> tagMap = new HashMap<Query, File>(files.length);
|
||||
Map<File, List<SubtitleDescriptor>> resultMap = new HashMap<File, List<SubtitleDescriptor>>(files.length);
|
||||
|
||||
// create tag query for each file
|
||||
List<Query> tagQueryList = new ArrayList<Query>(files.length);
|
||||
|
||||
for (File file : files) {
|
||||
// add tag query
|
||||
String tag = getNameWithoutExtension(file.getName());
|
||||
Query query = Query.forTag(tag, languageFilter);
|
||||
|
||||
// check tag
|
||||
List<SubtitleDescriptor> cachedResults = getCache().getSubtitleDescriptorList(query, languageName);
|
||||
if (cachedResults == null) {
|
||||
tagQueryList.add(query);
|
||||
tagMap.put(query, file);
|
||||
} else {
|
||||
resultMap.put(file, cachedResults);
|
||||
}
|
||||
|
||||
// prepare result map
|
||||
if (resultMap.get(file) == null) {
|
||||
resultMap.put(file, new LinkedList<SubtitleDescriptor>());
|
||||
}
|
||||
}
|
||||
|
||||
if (tagQueryList.size() > 0) {
|
||||
// require login
|
||||
login();
|
||||
|
||||
// dispatch query for all hashes
|
||||
int batchSize = 50;
|
||||
for (int bn = 0; bn < ceil((float) tagQueryList.size() / batchSize); bn++) {
|
||||
List<Query> batch = tagQueryList.subList(bn * batchSize, min((bn * batchSize) + batchSize, tagQueryList.size()));
|
||||
|
||||
// submit query and map results to given files
|
||||
for (OpenSubtitlesSubtitleDescriptor subtitle : xmlrpc.searchSubtitles(batch)) {
|
||||
// get file for tag
|
||||
File file = tagMap.get(batch.get(subtitle.getQueryNumber()));
|
||||
|
||||
// add subtitle
|
||||
if (file != null) {
|
||||
resultMap.get(file).add(subtitle);
|
||||
} else {
|
||||
Logger.getLogger(getClass().getName()).log(Level.WARNING, "Unable to map release name to file: " + subtitle.getMovieReleaseName());
|
||||
}
|
||||
}
|
||||
|
||||
for (Query query : batch) {
|
||||
getCache().putSubtitleDescriptorList(query, languageName, resultMap.get(tagMap.get(query)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return resultMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CheckResult checkSubtitle(File videoFile, File subtitleFile) throws Exception {
|
||||
// subhash (md5 of subtitles), subfilename, moviehash, moviebytesize, moviefilename
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import net.sourceforge.tuned.FileUtilities;
|
|||
public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor, Serializable {
|
||||
|
||||
public static enum Property {
|
||||
IDSubtitle, IDSubtitleFile, IDSubMovieFile, IDMovie, IDMovieImdb, SubFileName, SubFormat, SubHash, SubSize, MovieHash, MovieByteSize, MovieName, MovieNameEng, MovieYear, MovieReleaseName, MovieTimeMS, MovieFPS, MovieImdbRating, MovieKind, SeriesSeason, SeriesEpisode, SeriesIMDBParent, SubLanguageID, ISO639, LanguageName, UserID, UserRank, UserNickName, SubAddDate, SubAuthorComment, SubFeatured, SubComments, SubDownloadsCnt, SubHearingImpaired, SubRating, SubHD, SubBad, SubActualCD, SubSumCD, MatchedBy, SubtitlesLink, SubDownloadLink, ZipDownloadLink;
|
||||
IDSubtitle, IDSubtitleFile, IDSubMovieFile, IDMovie, IDMovieImdb, SubFileName, SubFormat, SubHash, SubSize, MovieHash, MovieByteSize, MovieName, MovieNameEng, MovieYear, MovieReleaseName, MovieTimeMS, MovieFPS, MovieImdbRating, MovieKind, SeriesSeason, SeriesEpisode, SeriesIMDBParent, SubLanguageID, ISO639, LanguageName, UserID, UserRank, UserNickName, SubAddDate, SubAuthorComment, SubFeatured, SubComments, SubDownloadsCnt, SubHearingImpaired, SubRating, SubHD, SubBad, SubActualCD, SubSumCD, MatchedBy, QueryNumber, SubtitlesLink, SubDownloadLink, ZipDownloadLink;
|
||||
|
||||
public static <V> EnumMap<Property, V> asEnumMap(Map<String, V> stringMap) {
|
||||
EnumMap<Property, V> enumMap = new EnumMap<Property, V>(Property.class);
|
||||
|
|
@ -47,6 +47,10 @@ public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor, Seri
|
|||
this.properties = properties;
|
||||
}
|
||||
|
||||
public Map<Property, String> getProperties() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
public String getProperty(Property key) {
|
||||
return properties.get(key);
|
||||
}
|
||||
|
|
@ -84,6 +88,14 @@ public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor, Seri
|
|||
return Long.parseLong(getProperty(Property.MovieByteSize));
|
||||
}
|
||||
|
||||
public String getMovieReleaseName() {
|
||||
return getProperty(Property.MovieReleaseName);
|
||||
}
|
||||
|
||||
public int getQueryNumber() {
|
||||
return Integer.parseInt(getProperty(Property.QueryNumber));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteBuffer fetch() throws Exception {
|
||||
URL resource = new URL(getProperty(Property.SubDownloadLink));
|
||||
|
|
|
|||
|
|
@ -377,23 +377,27 @@ public class OpenSubtitlesXmlRpc {
|
|||
|
||||
public static final class Query extends HashMap<String, Object> implements Serializable {
|
||||
|
||||
private Query(String imdbid, String... sublanguageids) {
|
||||
put("imdbid", imdbid);
|
||||
put("sublanguageid", join(sublanguageids, ","));
|
||||
}
|
||||
|
||||
private Query(String moviehash, String moviebytesize, String... sublanguageids) {
|
||||
put("moviehash", moviehash);
|
||||
put("moviebytesize", moviebytesize);
|
||||
private Query(String... sublanguageids) {
|
||||
put("sublanguageid", join(sublanguageids, ","));
|
||||
}
|
||||
|
||||
public static Query forHash(String moviehash, long moviebytesize, String... sublanguageids) {
|
||||
return new Query(moviehash, Long.toString(moviebytesize), sublanguageids);
|
||||
Query query = new Query(sublanguageids);
|
||||
query.put("moviehash", moviehash);
|
||||
query.put("moviebytesize", Long.toString(moviebytesize));
|
||||
return query;
|
||||
}
|
||||
|
||||
public static Query forTag(String tag, String... sublanguageids) {
|
||||
Query query = new Query(sublanguageids);
|
||||
query.put("tag", tag);
|
||||
return query;
|
||||
}
|
||||
|
||||
public static Query forImdbId(int imdbid, String... sublanguageids) {
|
||||
return new Query(Integer.toString(imdbid), sublanguageids);
|
||||
Query query = new Query(sublanguageids);
|
||||
query.put("imdbid", Integer.toString(imdbid));
|
||||
return query;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
|
||||
package net.sourceforge.filebot.web;
|
||||
|
||||
|
||||
import static java.util.Collections.*;
|
||||
import static net.sourceforge.filebot.Settings.*;
|
||||
import static org.junit.Assert.*;
|
||||
|
|
@ -19,81 +17,74 @@ import org.junit.AfterClass;
|
|||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
|
||||
public class OpenSubtitlesXmlRpcTest {
|
||||
|
||||
|
||||
private static OpenSubtitlesXmlRpc xmlrpc = new OpenSubtitlesXmlRpc(String.format("%s %s", getApplicationName(), getApplicationVersion()));
|
||||
|
||||
|
||||
|
||||
@BeforeClass
|
||||
public static void login() throws Exception {
|
||||
// login manually
|
||||
xmlrpc.loginAnonymous();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void search() throws Exception {
|
||||
List<Movie> list = xmlrpc.searchMoviesOnIMDB("babylon 5");
|
||||
Movie sample = list.get(0);
|
||||
|
||||
|
||||
// check sample entry
|
||||
assertEquals("Babylon 5", sample.getName());
|
||||
assertEquals(1994, sample.getYear());
|
||||
assertEquals(105946, sample.getImdbId());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test(expected = IndexOutOfBoundsException.class)
|
||||
public void searchOST() throws Exception {
|
||||
List<Movie> list = xmlrpc.searchMoviesOnIMDB("Linkin.Park.New.Divide.1280-720p.Transformers.Revenge.of.the.Fallen.ost");
|
||||
|
||||
|
||||
// seek to OST entry, expect to fail
|
||||
for (int i = 0; !list.get(i).getName().contains("Linkin.Park"); i++);
|
||||
for (int i = 0; !list.get(i).getName().contains("Linkin.Park"); i++)
|
||||
;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void getSubtitleListEnglish() throws Exception {
|
||||
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(361256, "eng");
|
||||
|
||||
|
||||
SubtitleDescriptor sample = list.get(0);
|
||||
|
||||
|
||||
assertTrue(sample.getName().startsWith("Wonderfalls"));
|
||||
assertEquals("English", sample.getLanguageName());
|
||||
|
||||
|
||||
// check size
|
||||
assertTrue(list.size() > 20);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void getSubtitleListAllLanguages() throws Exception {
|
||||
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(361256);
|
||||
|
||||
|
||||
OpenSubtitlesSubtitleDescriptor sample = list.get(75);
|
||||
|
||||
|
||||
assertEquals("\"Wonderfalls\" Wound-up Penguin", sample.getProperty(Property.MovieName));
|
||||
assertEquals("German", sample.getProperty(Property.LanguageName));
|
||||
assertEquals("imdbid", sample.getProperty(Property.MatchedBy));
|
||||
|
||||
|
||||
// check size
|
||||
assertTrue(list.size() > 70);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void getSubtitleListMovieHash() throws Exception {
|
||||
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(singleton(Query.forHash("2bba5c34b007153b", 717565952, "eng")));
|
||||
|
||||
|
||||
OpenSubtitlesSubtitleDescriptor sample = list.get(0);
|
||||
|
||||
|
||||
assertEquals("firefly.s01e01.serenity.pilot.dvdrip.xvid.srt", sample.getProperty(Property.SubFileName));
|
||||
assertEquals("English", sample.getProperty(Property.LanguageName));
|
||||
assertEquals("moviehash", sample.getProperty(Property.MatchedBy));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void tryUploadSubtitles() throws Exception {
|
||||
SubFile subtitle = new SubFile();
|
||||
|
|
@ -102,103 +93,94 @@ public class OpenSubtitlesXmlRpcTest {
|
|||
subtitle.setMovieFileName("firefly.s01e01.serenity.pilot.dvdrip.xvid.avi");
|
||||
subtitle.setMovieHash("2bba5c34b007153b");
|
||||
subtitle.setMovieByteSize(717565952);
|
||||
|
||||
|
||||
TryUploadResponse response = xmlrpc.tryUploadSubtitles(subtitle);
|
||||
|
||||
|
||||
assertFalse(response.isUploadRequired());
|
||||
assertEquals("4513264", response.getSubtitleData().get(0).get(Property.IDSubtitle.toString()));
|
||||
assertEquals("eng", response.getSubtitleData().get(0).get(Property.SubLanguageID.toString()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void checkSubHash() throws Exception {
|
||||
Map<String, Integer> subHashMap = xmlrpc.checkSubHash(singleton("e12715f466ee73c86694b7ab9f311285"));
|
||||
|
||||
|
||||
assertEquals("247060", subHashMap.values().iterator().next().toString());
|
||||
assertTrue(1 == subHashMap.size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void checkSubHashInvalid() throws Exception {
|
||||
Map<String, Integer> subHashMap = xmlrpc.checkSubHash(singleton("0123456789abcdef0123456789abcdef"));
|
||||
|
||||
|
||||
assertEquals("0", subHashMap.values().iterator().next().toString());
|
||||
assertTrue(1 == subHashMap.size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void checkMovieHash() throws Exception {
|
||||
Map<String, Movie> results = xmlrpc.checkMovieHash(singleton("d7aa0275cace4410"), 0);
|
||||
Movie movie = results.get("d7aa0275cace4410");
|
||||
|
||||
|
||||
assertEquals("Iron Man", movie.getName());
|
||||
assertEquals(2008, movie.getYear());
|
||||
assertEquals(371746, movie.getImdbId());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void checkMovieHashInvalid() throws Exception {
|
||||
Map<String, Movie> results = xmlrpc.checkMovieHash(singleton("0123456789abcdef"), 0);
|
||||
|
||||
|
||||
// no movie info
|
||||
assertTrue(results.isEmpty());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void getIMDBMovieDetails() throws Exception {
|
||||
Movie movie = xmlrpc.getIMDBMovieDetails(371746);
|
||||
|
||||
|
||||
assertEquals("Iron Man", movie.getName());
|
||||
assertEquals(2008, movie.getYear());
|
||||
assertEquals(371746, movie.getImdbId());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void getIMDBMovieDetailsInvalid() throws Exception {
|
||||
Movie movie = xmlrpc.getIMDBMovieDetails(371746);
|
||||
|
||||
|
||||
assertEquals("Iron Man", movie.getName());
|
||||
assertEquals(2008, movie.getYear());
|
||||
assertEquals(371746, movie.getImdbId());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void detectLanguage() throws Exception {
|
||||
String text = "Only those that are prepared to fire should be fired at.";
|
||||
|
||||
|
||||
List<String> languages = xmlrpc.detectLanguage(text.getBytes("UTF-8"));
|
||||
|
||||
|
||||
assertEquals("eng", languages.get(0));
|
||||
assertTrue(1 == languages.size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void fetchSubtitle() throws Exception {
|
||||
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(361256, "eng");
|
||||
|
||||
|
||||
// check format
|
||||
assertEquals("srt", list.get(0).getType());
|
||||
|
||||
|
||||
// fetch subtitle file
|
||||
ByteBuffer data = list.get(0).fetch();
|
||||
|
||||
|
||||
// check size
|
||||
assertEquals(48717, data.remaining(), 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@AfterClass
|
||||
public static void logout() throws Exception {
|
||||
// logout manually
|
||||
xmlrpc.logout();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue