moparisthebest
/
filebot
mirror of https://github.com/mitb-archive/filebot
1
0
Fork 0
Code Issues Releases Wiki Activity

* fix cache misconfig to stop anidb abuse 

* build our own anidb index file like with thetvdb
This commit is contained in:
Reinhard Pointner 2013-03-17 14:19:11 +00:00
parent 5dbeb12e88
commit 4dc5e26ae9
6 changed files with 82 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
source/ehcache.xml
View File

@ -19,7 +19,7 @@
/>
<!--
Long-lived (2 days) persistent disk cache for web responses
Short-lived (2 days) persistent disk cache for web responses
-->
<cache name="web-datasource"
maxElementsInMemory="200"
@ -32,6 +32,20 @@
memoryStoreEvictionPolicy="LRU"
/>
<!--
Long-lived (2 weeks) persistent disk cache for web responses
-->
<cache name="web-datasource-lv2"
maxElementsInMemory="200"
maxElementsOnDisk="10000"
eternal="false"
timeToIdleSeconds="1209600"
timeToLiveSeconds="1209600"
overflowToDisk="true"
diskPersistent="true"
memoryStoreEvictionPolicy="LRU"
/>
<!--
Very long-lived cache (2 months) anime/series lists, movie index, etc
-->

16
source/net/sourceforge/filebot/WebServices.java
View File

@ -49,7 +49,7 @@ public final class WebServices {
// episode dbs
public static final TVRageClient TVRage = new TVRageClient();
public static final AnidbClient AniDB = new AnidbClient(getApplicationName().toLowerCase(), 3);
public static final AnidbClient AniDB = new AnidbClientWithLocalSearch(getApplicationName().toLowerCase(), 4);
public static final SerienjunkiesClient Serienjunkies = new SerienjunkiesClient(getApplicationProperty("serienjunkies.apikey"));
// extended TheTVDB module with local search
@ -202,6 +202,20 @@ public final class WebServices {
}
public static class AnidbClientWithLocalSearch extends AnidbClient {
public AnidbClientWithLocalSearch(String client, int clientver) {
super(client, clientver);
}
@Override
public List<AnidbSearchResult> getAnimeTitles() throws Exception {
return asList(MediaDetection.releaseInfo.getAnidbIndex());
}
}
/**
* Dummy constructor to prevent instantiation.
*/

40
source/net/sourceforge/filebot/media/ReleaseInfo.java
View File

@ -35,6 +35,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import net.sourceforge.filebot.web.AnidbClient.AnidbSearchResult;
import net.sourceforge.filebot.web.CachedResource;
import net.sourceforge.filebot.web.Movie;
import net.sourceforge.filebot.web.TheTVDBClient.TheTVDBSearchResult;
@ -243,7 +244,12 @@ public class ReleaseInfo {
public TheTVDBSearchResult[] getTheTVDBIndex() throws IOException {
return theTVDBIndexResource.get();
return tvdbIndexResource.get();
}
public AnidbSearchResult[] getAnidbIndex() throws IOException {
return anidbIndexResource.get();
}
@ -275,7 +281,8 @@ public class ReleaseInfo {
protected final CachedResource<Movie[]> movieListResource = new MovieResource(getBundle(getClass().getName()).getString("url.movie-list"));
protected final CachedResource<String[]> seriesListResource = new SeriesListResource(getBundle(getClass().getName()).getString("url.series-list"));
protected final CachedResource<String[]> seriesDirectMappingsResource = new PatternResource(getBundle(getClass().getName()).getString("url.series-mappings"));
protected final CachedResource<TheTVDBSearchResult[]> theTVDBIndexResource = new TheTVDBIndexResource(getBundle(getClass().getName()).getString("url.thetvdb-index"));
protected final CachedResource<TheTVDBSearchResult[]> tvdbIndexResource = new TheTVDBIndexResource(getBundle(getClass().getName()).getString("url.thetvdb-index"));
protected final CachedResource<AnidbSearchResult[]> anidbIndexResource = new AnidbIndexResource(getBundle(getClass().getName()).getString("url.anidb-index"));
protected static class PatternResource extends CachedResource<String[]> {
@ -353,6 +360,35 @@ public class ReleaseInfo {
}
protected static class AnidbIndexResource extends CachedResource<AnidbSearchResult[]> {
public AnidbIndexResource(String resource) {
super(resource, AnidbSearchResult[].class, 7 * 24 * 60 * 60 * 1000); // check for updates once a week
}
@Override
public AnidbSearchResult[] process(ByteBuffer data) throws IOException {
Scanner scanner = new Scanner(new GZIPInputStream(new ByteBufferInputStream(data)), "UTF-8").useDelimiter("\t|\n");
List<AnidbSearchResult> anime = new ArrayList<AnidbSearchResult>();
while (scanner.hasNext() && scanner.hasNextInt()) {
int aid = scanner.nextInt();
String primaryTitle = scanner.next().trim();
String englishTitle = scanner.next().trim();
if (englishTitle.isEmpty() || englishTitle.equals(primaryTitle)) {
anime.add(new AnidbSearchResult(aid, primaryTitle, null));
} else {
anime.add(new AnidbSearchResult(aid, primaryTitle, singletonMap("en", englishTitle)));
}
}
return anime.toArray(new AnidbSearchResult[0]);
}
}
protected static class FolderEntryFilter implements FileFilter {
private final Pattern entryPattern;

17
source/net/sourceforge/filebot/media/ReleaseInfo.properties
View File

@ -5,25 +5,28 @@ pattern.video.source: CAMRip|CAM|PDVD|TS|TELESYNC|PDVD|PPV|PPVRip|Screener|SCR|S
pattern.video.format: DivX|Xvid|AVC|x264|h264|3ivx|mpeg|mpeg4|mp3|aac|ac3|dd20|dd51|2ch|6ch|DTS|WS|HR|7p|720p|18p|1080p|NTSC
# known release group names
url.release-groups: http://www.filebot.net/data/release-groups.txt
url.release-groups: http://filebot.net/data/release-groups.txt
# blacklisted terms that will be ignored
url.query-blacklist: http://www.filebot.net/data/query-blacklist.txt
url.query-blacklist: http://filebot.net/data/query-blacklist.txt
# clutter files that will be ignored
url.exclude-blacklist: http://www.filebot.net/data/exclude-blacklist.txt
url.exclude-blacklist: http://filebot.net/data/exclude-blacklist.txt
# list of patterns directly matching files to series names
url.series-mappings: http://www.filebot.net/data/series-mappings.txt
url.series-mappings: http://filebot.net/data/series-mappings.txt
# list of all movies (id, name, year)
url.movie-list: http://www.filebot.net/data/movies.txt.gz
url.movie-list: http://filebot.net/data/movies.txt.gz
# list of tv show and anime names
url.series-list: http://www.filebot.net/data/series.list.gz
url.series-list: http://filebot.net/data/series.list.gz
# TheTVDB index
url.thetvdb-index: http://www.filebot.net/data/thetvdb.txt.gz
url.thetvdb-index: http://filebot.net/data/thetvdb.txt.gz
# AniDB index
url.anidb-index: http://filebot.net/data/anidb.txt.gz
# disk folder matcher
pattern.diskfolder.entry: BDMV|HVDVD_TS|VIDEO_TS|AUDIO_TS|VCD|movie.nfo

6
source/net/sourceforge/filebot/web/AnidbClient.java
View File

@ -70,7 +70,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
@Override
public ResultCache getCache() {
return new ResultCache(host, Cache.getCache("web-datasource"));
return new ResultCache(host, Cache.getCache("web-datasource-lv2"));
}
@ -87,7 +87,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
@Override
protected Set<String> getFields(AnidbSearchResult anime) {
return set(anime.getPrimaryTitle(), anime.getOfficialTitle("en"), anime.getOfficialTitle(locale.getLanguage()));
return set(anime.getPrimaryTitle(), anime.getOfficialTitle("en"));
}
};
@ -154,7 +154,7 @@ public class AnidbClient extends AbstractEpisodeListProvider {
public synchronized List<AnidbSearchResult> getAnimeTitles() throws Exception {
URL url = new URL("http", host, "/api/animetitles.dat.gz");
URL url = new URL("http", host, "/api/anime-titles.dat.gz");
ResultCache cache = getCache();
@SuppressWarnings("unchecked")

1
website/data/.htaccess
View File

@ -3,3 +3,4 @@ options +indexes
redirect 301 /data/movies.txt.gz http://sourceforge.net/projects/filebot/files/data/movies.txt.gz/download
redirect 301 /data/series.list.gz http://sourceforge.net/projects/filebot/files/data/series.list.gz/download
redirect 301 /data/thetvdb.txt.gz http://sourceforge.net/projects/filebot/files/data/thetvdb.txt.gz/download
redirect 301 /data/anidb.txt.gz http://sourceforge.net/projects/filebot/files/data/anidb.txt.gz/download