moparisthebest/filebot
1
0
Fork 0
mirror of https://github.com/mitb-archive/filebot synced 2024-11-02 08:25:02 -04:00
Code Issues Releases Wiki Activity

* support OpenSubtitles TAG lookup (i.e. filename) as part of hash lookup if actual hash lookup does not yield any results

Browse Source
This commit is contained in:
Reinhard Pointner 2014-01-27 08:36:30 +00:00
parent 6e732e8987
commit 5fadfbe3e3
4 changed files with 168 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
source/net/sourceforge/filebot/web/OpenSubtitlesClient.java
View File

@ -17,6 +17,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
@ -134,6 +135,29 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
@Override
public Map<File, List<SubtitleDescriptor>> getSubtitleList(File[] files, String languageName) throws Exception {
Map<File, List<SubtitleDescriptor>> results = new HashMap<File, List<SubtitleDescriptor>>(files.length);
Set<File> remainingFiles = new LinkedHashSet<File>(asList(files));
// lookup subtitles by hash
if (remainingFiles.size() > 0) {
results.putAll(getSubtitleListByHash(remainingFiles.toArray(new File[0]), languageName));
}
for (Entry<File, List<SubtitleDescriptor>> it : results.entrySet()) {
if (it.getValue().size() > 0) {
remainingFiles.remove(it.getKey());
}
}
// lookup subtitles by tag
if (remainingFiles.size() > 0) {
results.putAll(getSubtitleListByTag(remainingFiles.toArray(new File[0]), languageName));
}
return results;
}
public Map<File, List<SubtitleDescriptor>> getSubtitleListByHash(File[] files, String languageName) throws Exception {
// singleton array with or empty array
String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName) } : new String[0];
@ -142,10 +166,10 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
Map<File, List<SubtitleDescriptor>> resultMap = new HashMap<File, List<SubtitleDescriptor>>(files.length);
// create hash query for each file
List<Query> queryList = new ArrayList<Query>(files.length);
List<Query> hashQueryList = new ArrayList<Query>(files.length);
for (File file : files) {
// add query
// add hash query
if (file.length() > HASH_CHUNK_SIZE) {
String movieHash = computeHash(file);
Query query = Query.forHash(movieHash, file.length(), languageFilter);
@ -153,7 +177,7 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
// check hash
List<SubtitleDescriptor> cachedResults = getCache().getSubtitleDescriptorList(query, languageName);
if (cachedResults == null) {
queryList.add(query);
hashQueryList.add(query);
hashMap.put(query, file);
} else {
resultMap.put(file, cachedResults);
@ -166,22 +190,26 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
}
}
if (queryList.size() > 0) {
if (hashQueryList.size() > 0) {
// require login
login();
// dispatch query for all hashes
int batchSize = 50;
for (int bn = 0; bn < ceil((float) queryList.size() / batchSize); bn++) {
List<Query> batch = queryList.subList(bn * batchSize, min((bn * batchSize) + batchSize, queryList.size()));
for (int bn = 0; bn < ceil((float) hashQueryList.size() / batchSize); bn++) {
List<Query> batch = hashQueryList.subList(bn * batchSize, min((bn * batchSize) + batchSize, hashQueryList.size()));
// submit query and map results to given files
for (OpenSubtitlesSubtitleDescriptor subtitle : xmlrpc.searchSubtitles(batch)) {
// get file for hash
File file = hashMap.get(Query.forHash(subtitle.getMovieHash(), subtitle.getMovieByteSize(), languageFilter));
File file = hashMap.get((batch.get(subtitle.getQueryNumber())));
// add subtitle
resultMap.get(file).add(subtitle);
if (file != null) {
resultMap.get(file).add(subtitle);
} else {
Logger.getLogger(getClass().getName()).log(Level.WARNING, "Unable to map hash to file: " + subtitle.getMovieHash());
}
}
for (Query query : batch) {
@ -193,6 +221,68 @@ public class OpenSubtitlesClient implements SubtitleProvider, VideoHashSubtitleS
return resultMap;
}
public Map<File, List<SubtitleDescriptor>> getSubtitleListByTag(File[] files, String languageName) throws Exception {
// singleton array with or empty array
String[] languageFilter = languageName != null ? new String[] { getSubLanguageID(languageName) } : new String[0];
// remember tag for each file
Map<Query, File> tagMap = new HashMap<Query, File>(files.length);
Map<File, List<SubtitleDescriptor>> resultMap = new HashMap<File, List<SubtitleDescriptor>>(files.length);
// create tag query for each file
List<Query> tagQueryList = new ArrayList<Query>(files.length);
for (File file : files) {
// add tag query
String tag = getNameWithoutExtension(file.getName());
Query query = Query.forTag(tag, languageFilter);
// check tag
List<SubtitleDescriptor> cachedResults = getCache().getSubtitleDescriptorList(query, languageName);
if (cachedResults == null) {
tagQueryList.add(query);
tagMap.put(query, file);
} else {
resultMap.put(file, cachedResults);
}
// prepare result map
if (resultMap.get(file) == null) {
resultMap.put(file, new LinkedList<SubtitleDescriptor>());
}
}
if (tagQueryList.size() > 0) {
// require login
login();
// dispatch query for all hashes
int batchSize = 50;
for (int bn = 0; bn < ceil((float) tagQueryList.size() / batchSize); bn++) {
List<Query> batch = tagQueryList.subList(bn * batchSize, min((bn * batchSize) + batchSize, tagQueryList.size()));
// submit query and map results to given files
for (OpenSubtitlesSubtitleDescriptor subtitle : xmlrpc.searchSubtitles(batch)) {
// get file for tag
File file = tagMap.get(batch.get(subtitle.getQueryNumber()));
// add subtitle
if (file != null) {
resultMap.get(file).add(subtitle);
} else {
Logger.getLogger(getClass().getName()).log(Level.WARNING, "Unable to map release name to file: " + subtitle.getMovieReleaseName());
}
}
for (Query query : batch) {
getCache().putSubtitleDescriptorList(query, languageName, resultMap.get(tagMap.get(query)));
}
}
}
return resultMap;
}
@Override
public CheckResult checkSubtitle(File videoFile, File subtitleFile) throws Exception {
// subhash (md5 of subtitles), subfilename, moviehash, moviebytesize, moviefilename

14
source/net/sourceforge/filebot/web/OpenSubtitlesSubtitleDescriptor.java
View File

@ -23,7 +23,7 @@ import net.sourceforge.tuned.FileUtilities;
public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor, Serializable {
public static enum Property {
IDSubtitle, IDSubtitleFile, IDSubMovieFile, IDMovie, IDMovieImdb, SubFileName, SubFormat, SubHash, SubSize, MovieHash, MovieByteSize, MovieName, MovieNameEng, MovieYear, MovieReleaseName, MovieTimeMS, MovieFPS, MovieImdbRating, MovieKind, SeriesSeason, SeriesEpisode, SeriesIMDBParent, SubLanguageID, ISO639, LanguageName, UserID, UserRank, UserNickName, SubAddDate, SubAuthorComment, SubFeatured, SubComments, SubDownloadsCnt, SubHearingImpaired, SubRating, SubHD, SubBad, SubActualCD, SubSumCD, MatchedBy, SubtitlesLink, SubDownloadLink, ZipDownloadLink;
IDSubtitle, IDSubtitleFile, IDSubMovieFile, IDMovie, IDMovieImdb, SubFileName, SubFormat, SubHash, SubSize, MovieHash, MovieByteSize, MovieName, MovieNameEng, MovieYear, MovieReleaseName, MovieTimeMS, MovieFPS, MovieImdbRating, MovieKind, SeriesSeason, SeriesEpisode, SeriesIMDBParent, SubLanguageID, ISO639, LanguageName, UserID, UserRank, UserNickName, SubAddDate, SubAuthorComment, SubFeatured, SubComments, SubDownloadsCnt, SubHearingImpaired, SubRating, SubHD, SubBad, SubActualCD, SubSumCD, MatchedBy, QueryNumber, SubtitlesLink, SubDownloadLink, ZipDownloadLink;
public static <V> EnumMap<Property, V> asEnumMap(Map<String, V> stringMap) {
EnumMap<Property, V> enumMap = new EnumMap<Property, V>(Property.class);
@ -47,6 +47,10 @@ public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor, Seri
this.properties = properties;
}
public Map<Property, String> getProperties() {
return properties;
}
public String getProperty(Property key) {
return properties.get(key);
}
@ -84,6 +88,14 @@ public class OpenSubtitlesSubtitleDescriptor implements SubtitleDescriptor, Seri
return Long.parseLong(getProperty(Property.MovieByteSize));
}
public String getMovieReleaseName() {
return getProperty(Property.MovieReleaseName);
}
public int getQueryNumber() {
return Integer.parseInt(getProperty(Property.QueryNumber));
}
@Override
public ByteBuffer fetch() throws Exception {
URL resource = new URL(getProperty(Property.SubDownloadLink));

24
source/net/sourceforge/filebot/web/OpenSubtitlesXmlRpc.java
View File

@ -377,23 +377,27 @@ public class OpenSubtitlesXmlRpc {
public static final class Query extends HashMap<String, Object> implements Serializable {
private Query(String imdbid, String... sublanguageids) {
put("imdbid", imdbid);
put("sublanguageid", join(sublanguageids, ","));
}
private Query(String moviehash, String moviebytesize, String... sublanguageids) {
put("moviehash", moviehash);
put("moviebytesize", moviebytesize);
private Query(String... sublanguageids) {
put("sublanguageid", join(sublanguageids, ","));
}
public static Query forHash(String moviehash, long moviebytesize, String... sublanguageids) {
return new Query(moviehash, Long.toString(moviebytesize), sublanguageids);
Query query = new Query(sublanguageids);
query.put("moviehash", moviehash);
query.put("moviebytesize", Long.toString(moviebytesize));
return query;
}
public static Query forTag(String tag, String... sublanguageids) {
Query query = new Query(sublanguageids);
query.put("tag", tag);
return query;
}
public static Query forImdbId(int imdbid, String... sublanguageids) {
return new Query(Integer.toString(imdbid), sublanguageids);
Query query = new Query(sublanguageids);
query.put("imdbid", Integer.toString(imdbid));
return query;
}
}

104
test/net/sourceforge/filebot/web/OpenSubtitlesXmlRpcTest.java
View File

@ -1,7 +1,5 @@
package net.sourceforge.filebot.web;
import static java.util.Collections.*;
import static net.sourceforge.filebot.Settings.*;
import static org.junit.Assert.*;
@ -19,81 +17,74 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
public class OpenSubtitlesXmlRpcTest {
private static OpenSubtitlesXmlRpc xmlrpc = new OpenSubtitlesXmlRpc(String.format("%s %s", getApplicationName(), getApplicationVersion()));
@BeforeClass
public static void login() throws Exception {
// login manually
xmlrpc.loginAnonymous();
}
@Test
public void search() throws Exception {
List<Movie> list = xmlrpc.searchMoviesOnIMDB("babylon 5");
Movie sample = list.get(0);
// check sample entry
assertEquals("Babylon 5", sample.getName());
assertEquals(1994, sample.getYear());
assertEquals(105946, sample.getImdbId());
}
@Test(expected = IndexOutOfBoundsException.class)
public void searchOST() throws Exception {
List<Movie> list = xmlrpc.searchMoviesOnIMDB("Linkin.Park.New.Divide.1280-720p.Transformers.Revenge.of.the.Fallen.ost");
// seek to OST entry, expect to fail
for (int i = 0; !list.get(i).getName().contains("Linkin.Park"); i++);
for (int i = 0; !list.get(i).getName().contains("Linkin.Park"); i++)
;
}
@Test
public void getSubtitleListEnglish() throws Exception {
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(361256, "eng");
SubtitleDescriptor sample = list.get(0);
assertTrue(sample.getName().startsWith("Wonderfalls"));
assertEquals("English", sample.getLanguageName());
// check size
assertTrue(list.size() > 20);
}
@Test
public void getSubtitleListAllLanguages() throws Exception {
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(361256);
OpenSubtitlesSubtitleDescriptor sample = list.get(75);
assertEquals("\"Wonderfalls\" Wound-up Penguin", sample.getProperty(Property.MovieName));
assertEquals("German", sample.getProperty(Property.LanguageName));
assertEquals("imdbid", sample.getProperty(Property.MatchedBy));
// check size
assertTrue(list.size() > 70);
}
@Test
public void getSubtitleListMovieHash() throws Exception {
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(singleton(Query.forHash("2bba5c34b007153b", 717565952, "eng")));
OpenSubtitlesSubtitleDescriptor sample = list.get(0);
assertEquals("firefly.s01e01.serenity.pilot.dvdrip.xvid.srt", sample.getProperty(Property.SubFileName));
assertEquals("English", sample.getProperty(Property.LanguageName));
assertEquals("moviehash", sample.getProperty(Property.MatchedBy));
}
@Test
public void tryUploadSubtitles() throws Exception {
SubFile subtitle = new SubFile();
@ -102,103 +93,94 @@ public class OpenSubtitlesXmlRpcTest {
subtitle.setMovieFileName("firefly.s01e01.serenity.pilot.dvdrip.xvid.avi");
subtitle.setMovieHash("2bba5c34b007153b");
subtitle.setMovieByteSize(717565952);
TryUploadResponse response = xmlrpc.tryUploadSubtitles(subtitle);
assertFalse(response.isUploadRequired());
assertEquals("4513264", response.getSubtitleData().get(0).get(Property.IDSubtitle.toString()));
assertEquals("eng", response.getSubtitleData().get(0).get(Property.SubLanguageID.toString()));
}
@Test
public void checkSubHash() throws Exception {
Map<String, Integer> subHashMap = xmlrpc.checkSubHash(singleton("e12715f466ee73c86694b7ab9f311285"));
assertEquals("247060", subHashMap.values().iterator().next().toString());
assertTrue(1 == subHashMap.size());
}
@Test
public void checkSubHashInvalid() throws Exception {
Map<String, Integer> subHashMap = xmlrpc.checkSubHash(singleton("0123456789abcdef0123456789abcdef"));
assertEquals("0", subHashMap.values().iterator().next().toString());
assertTrue(1 == subHashMap.size());
}
@Test
public void checkMovieHash() throws Exception {
Map<String, Movie> results = xmlrpc.checkMovieHash(singleton("d7aa0275cace4410"), 0);
Movie movie = results.get("d7aa0275cace4410");
assertEquals("Iron Man", movie.getName());
assertEquals(2008, movie.getYear());
assertEquals(371746, movie.getImdbId());
}
@Test
public void checkMovieHashInvalid() throws Exception {
Map<String, Movie> results = xmlrpc.checkMovieHash(singleton("0123456789abcdef"), 0);
// no movie info
assertTrue(results.isEmpty());
}
@Test
public void getIMDBMovieDetails() throws Exception {
Movie movie = xmlrpc.getIMDBMovieDetails(371746);
assertEquals("Iron Man", movie.getName());
assertEquals(2008, movie.getYear());
assertEquals(371746, movie.getImdbId());
}
@Test
public void getIMDBMovieDetailsInvalid() throws Exception {
Movie movie = xmlrpc.getIMDBMovieDetails(371746);
assertEquals("Iron Man", movie.getName());
assertEquals(2008, movie.getYear());
assertEquals(371746, movie.getImdbId());
}
@Test
public void detectLanguage() throws Exception {
String text = "Only those that are prepared to fire should be fired at.";
List<String> languages = xmlrpc.detectLanguage(text.getBytes("UTF-8"));
assertEquals("eng", languages.get(0));
assertTrue(1 == languages.size());
}
@Test
public void fetchSubtitle() throws Exception {
List<OpenSubtitlesSubtitleDescriptor> list = xmlrpc.searchSubtitles(361256, "eng");
// check format
assertEquals("srt", list.get(0).getType());
// fetch subtitle file
ByteBuffer data = list.get(0).fetch();
// check size
assertEquals(48717, data.remaining(), 0);
}
@AfterClass
public static void logout() throws Exception {
// logout manually
xmlrpc.logout();
}
}