moparisthebest
/
filebot
mirror of https://github.com/mitb-archive/filebot
1
0
Fork 0
Code Issues Releases Wiki Activity

Experiment with new CachedResource framework

This commit is contained in:
Reinhard Pointner 2016-03-06 22:21:13 +00:00
parent 500a4972e1
commit bbed902c63
7 changed files with 184 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
source/ehcache.xml
View File

@ -1,28 +1,9 @@
<ehcache xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="ehcache.xsd" updateCheck="false">
<!--
Persistent disk store location
-->
<diskStore path="ehcache.disk.store.dir" />
<!--
Mandatory Default Cache configuration. These settings will be applied to caches
created pragmatically using CacheManager.add(String cacheName)
-->
<diskStore path="java.io.tmpdir" />
<defaultCache
maxElementsInMemory="100"
eternal="false"
timeToIdleSeconds="120"
timeToLiveSeconds="120"
overflowToDisk="false"
diskPersistent="false"
memoryStoreEvictionPolicy="LRU"
/>
<!--
Short-lived (24 hours) persistent disk cache for web responses
-->
<cache name="web-datasource"
maxElementsInMemory="200"
maxElementsInMemory="400"
maxElementsOnDisk="80000"
eternal="false"
timeToIdleSeconds="86400"
@ -32,73 +13,4 @@
memoryStoreEvictionPolicy="LRU"
/>
<!--
Long-lived (2 weeks) persistent disk cache for web responses
-->
<cache name="web-datasource-lv2"
maxElementsInMemory="200"
maxElementsOnDisk="80000"
eternal="false"
timeToIdleSeconds="1209600"
timeToLiveSeconds="1209600"
overflowToDisk="true"
diskPersistent="true"
memoryStoreEvictionPolicy="LRU"
/>
<!--
Long-lived (2 months) persistent disk cache for web responses (that can be updated via If-Modified or If-None-Match)
-->
<cache name="web-datasource-lv3"
maxElementsInMemory="100"
maxElementsOnDisk="200000"
eternal="false"
timeToIdleSeconds="5256000"
timeToLiveSeconds="5256000"
overflowToDisk="true"
diskPersistent="true"
memoryStoreEvictionPolicy="LRU"
/>
<!--
Very long-lived cache (4 months) anime/series lists, movie index, etc
-->
<cache name="web-persistent-datasource"
maxElementsInMemory="100"
maxElementsOnDisk="80000"
eternal="false"
timeToIdleSeconds="10512000"
timeToLiveSeconds="10512000"
overflowToDisk="true"
diskPersistent="true"
memoryStoreEvictionPolicy="LRU"
/>
<!--
Simple memory cache. Time to live is 4 months.
-->
<cache name="persistent-memory"
maxElementsInMemory="100"
maxElementsOnDisk="50000"
eternal="false"
timeToIdleSeconds="10512000"
timeToLiveSeconds="10512000"
overflowToDisk="true"
diskPersistent="true"
memoryStoreEvictionPolicy="LRU"
/>
<!--
Simple memory cache. Time to live is 2 hours.
-->
<cache name="ephemeral-memory"
maxElementsInMemory="50000"
eternal="false"
timeToIdleSeconds="7200"
timeToLiveSeconds="7200"
overflowToDisk="false"
diskPersistent="false"
memoryStoreEvictionPolicy="LRU"
/>
</ehcache>

53
source/net/filebot/Cache.java
View File

@ -1,13 +1,16 @@
package net.filebot;
import static java.nio.charset.StandardCharsets.*;
import static net.filebot.Logging.*;
import java.io.Serializable;
import java.net.URL;
import java.time.Duration;
import java.util.Arrays;
import java.util.concurrent.Callable;
import java.util.function.Predicate;
import net.filebot.web.CachedResource2;
import net.filebot.web.FloodLimit;
import net.sf.ehcache.Element;
public class Cache {
@ -24,51 +27,60 @@ public class Cache {
public Object get(Object key) {
try {
return cache.get(key).getObjectValue();
Element element = cache.get(key);
if (element != null) {
return element.getObjectValue();
}
} catch (Exception e) {
debug.warning(format("Bad cache state: %s => %s", key, e));
e.printStackTrace();
debug.warning(format("Cache get: %s => %s", key, e));
}
return null;
}
public Object computeIf(Object key, Predicate<Element> condition, Callable<?> callable) throws Exception {
public Object computeIf(Object key, Predicate<Element> condition, Compute<?> compute) throws Exception {
// get if present
Element element = null;
try {
Element element = cache.get(key);
element = cache.get(key);
if (element != null && condition.test(element)) {
return element.getObjectValue();
}
} catch (Exception e) {
debug.warning(format("Bad cache state: %s => %s", key, e));
debug.warning(format("Cache get: %s => %s", key, e));
}
// compute if absent
Object value = callable.call();
Object value = compute.apply(element);
try {
cache.put(new Element(key, value));
} catch (Exception e) {
debug.warning(format("Bad cache state: %s => %s", key, e));
debug.warning(format("Cache put: %s => %s", key, e));
}
return value;
}
public Object computeIfAbsent(Object key, Callable<?> callable) throws Exception {
return computeIf(key, Element::isExpired, callable);
public Object computeIfAbsent(Object key, Compute<?> compute) throws Exception {
return computeIf(key, isAbsent(), compute);
}
public Object computeIfStale(Object key, Duration expirationTime, Callable<?> callable) throws Exception {
return computeIf(key, isStale(expirationTime), callable);
public Object computeIfStale(Object key, Duration expirationTime, Compute<?> compute) throws Exception {
return computeIf(key, isStale(expirationTime), compute);
}
private Predicate<Element> isStale(Duration expirationTime) {
return (element) -> element.isExpired() || System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis();
public Predicate<Element> isAbsent() {
return (element) -> element.getObjectValue() == null;
}
public Predicate<Element> isStale(Duration expirationTime) {
return (element) -> System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis();
}
public void put(Object key, Object value) {
try {
cache.put(new Element(key, value));
} catch (Exception e) {
debug.warning(format("Bad cache state: %s => %s", key, e));
debug.warning(format("Cache put: %s => %s", key, e));
}
}
@ -76,10 +88,15 @@ public class Cache {
try {
cache.remove(key);
} catch (Exception e) {
debug.warning(format("Bad cache state: %s => %s", key, e));
debug.warning(format("Cache remove: %s => %s", key, e));
}
}
@FunctionalInterface
public interface Compute<R> {
R apply(Element element) throws Exception;
}
@Deprecated
public <T> T get(Object key, Class<T> type) {
return type.cast(get(key));
@ -114,4 +131,8 @@ public class Cache {
}
}
public CachedResource2<String, String> resource(String url, Duration expirationTime, FloodLimit limit) {
return new CachedResource2<String, String>(url, URL::new, CachedResource2.fetchIfModified(limit), CachedResource2.decode(UTF_8), expirationTime, this);
}
}

2
source/net/filebot/format/MediaBindingBean.java
View File

@ -447,7 +447,7 @@ public class MediaBindingBean {
// calculate checksum from file
Cache cache = Cache.getCache("crc32", CacheType.Ephemeral);
return (String) cache.computeIfAbsent(inferredMediaFile, () -> crc32(inferredMediaFile));
return (String) cache.computeIfAbsent(inferredMediaFile, element -> crc32(inferredMediaFile));
}
@Define("fn")

123
source/net/filebot/web/CachedResource2.java Normal file
View File

@ -0,0 +1,123 @@
package net.filebot.web;
import static net.filebot.Logging.*;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.time.Duration;
import java.util.concurrent.Callable;
import net.filebot.Cache;
public class CachedResource2<K, R> {
public static final int DEFAULT_RETRY_LIMIT = 2;
public static final Duration DEFAULT_RETRY_DELAY = Duration.ofSeconds(2);
protected final K key;
protected final Source<K> source;
protected final Fetch fetch;
protected final Parse<R> parse;
protected final Duration expirationTime;
protected final int retryCountLimit;
protected final long retryWaitTime;
protected final Cache cache;
public CachedResource2(K key, Source<K> source, Fetch fetch, Parse<R> parse, Duration expirationTime, Cache cache) {
this(key, source, fetch, parse, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, cache);
}
public CachedResource2(K key, Source<K> source, Fetch fetch, Parse<R> parse, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) {
this.key = key;
this.source = source;
this.fetch = fetch;
this.parse = parse;
this.expirationTime = expirationTime;
this.retryCountLimit = retryCountLimit;
this.retryWaitTime = retryWaitTime.toMillis();
this.cache = cache;
}
@SuppressWarnings("unchecked")
public synchronized R get() throws Exception {
return (R) cache.computeIfStale(key, expirationTime, element -> {
URL resource = source.source(key);
long lastModified = element == null ? 0 : element.getLatestOfCreationAndUpdateTime();
debug.fine(format("Fetch %s (If-Modified-Since: %tc)", resource, lastModified));
try {
ByteBuffer data = retry(() -> fetch.fetch(resource, lastModified), retryCountLimit, lastModified);
// 304 Not Modified
if (data == null && element != null && element.getObjectValue() != null) {
return element.getObjectValue();
}
return parse.parse(data);
} catch (IOException e) {
debug.fine(format("Fetch failed => %s", e));
// use previously cached data if possible
if (element == null || element.getObjectValue() == null) {
throw e;
}
return element.getObjectKey();
}
});
}
protected <T> T retry(Callable<T> callable, int retryCount, long retryWaitTime) throws Exception {
try {
return callable.call();
} catch (FileNotFoundException e) {
// resource does not exist, do not retry
throw e;
} catch (IOException e) {
// retry or rethrow exception
if (retryCount > 0) {
throw e;
}
Thread.sleep(retryWaitTime);
return retry(callable, retryCount - 1, retryWaitTime * 2);
}
}
@FunctionalInterface
public interface Source<K> {
URL source(K key) throws Exception;
}
@FunctionalInterface
public interface Fetch {
ByteBuffer fetch(URL url, long lastModified) throws Exception;
}
@FunctionalInterface
public interface Parse<R> {
R parse(ByteBuffer bytes) throws Exception;
}
public static Parse<String> decode(Charset charset) {
return (bb) -> charset.decode(bb).toString();
}
public static Fetch fetchIfModified(FloodLimit limit) {
return (url, lastModified) -> {
try {
limit.acquirePermit();
return WebRequest.fetchIfModified(url, lastModified);
} catch (FileNotFoundException e) {
return ByteBuffer.allocate(0);
}
};
}
}

64
source/net/filebot/web/OMDbClient.java
View File

@ -2,16 +2,13 @@ package net.filebot.web;
import static java.util.Collections.*;
import static java.util.stream.Collectors.*;
import static net.filebot.Logging.*;
import static net.filebot.util.JsonUtilities.*;
import static net.filebot.util.StringUtilities.*;
import static net.filebot.web.WebRequest.*;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.time.Duration;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
@ -27,27 +24,22 @@ import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.swing.Icon;
import net.filebot.Cache;
import net.filebot.CacheType;
import net.filebot.ResourceManager;
import net.filebot.web.TMDbClient.MovieInfo;
import net.filebot.web.TMDbClient.MovieInfo.MovieProperty;
import net.filebot.web.TMDbClient.Person;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
public class OMDbClient implements MovieIdentificationService {
private static final FloodLimit REQUEST_LIMIT = new FloodLimit(20, 10, TimeUnit.SECONDS);
private final String protocol = "http";
private final String host = "www.omdbapi.com";
@Override
public String getName() {
return "OMDb";
@ -70,7 +62,7 @@ public class OMDbClient implements MovieIdentificationService {
}
@Override
public List<Movie> searchMovie(String query, Locale locale) throws IOException {
public List<Movie> searchMovie(String query, Locale locale) throws Exception {
// query by name with year filter if possible
Matcher nameYear = Pattern.compile("(.+)\\b(19\\d{2}|20\\d{2})$").matcher(query);
if (nameYear.matches()) {
@ -80,14 +72,14 @@ public class OMDbClient implements MovieIdentificationService {
}
}
public List<Movie> searchMovie(String movieName, int movieYear) throws IOException {
public List<Movie> searchMovie(String movieName, int movieYear) throws Exception {
Map<String, Object> param = new LinkedHashMap<String, Object>(2);
param.put("s", movieName);
if (movieYear > 0) {
param.put("y", movieYear);
}
Map<?, ?> response = request(param, REQUEST_LIMIT);
Map<?, ?> response = request(param);
List<Movie> result = new ArrayList<Movie>();
for (Object it : getArray(response, "Search")) {
@ -141,40 +133,16 @@ public class OMDbClient implements MovieIdentificationService {
throw new UnsupportedOperationException();
}
public Map<?, ?> request(Map<String, Object> parameters, final FloodLimit limit) throws IOException {
URL url = new URL(protocol, host, "/?" + encodeParameters(parameters, true));
public Map<?, ?> request(Map<String, Object> parameters) throws Exception {
String url = "http://www.omdbapi.com/?" + encodeParameters(parameters, true);
CachedResource<String> json = new CachedResource<String>(url.toString(), String.class, CachedResource.ONE_WEEK) {
Cache cache = Cache.getCache(getName(), CacheType.Weekly);
String json = cache.resource(url, Duration.ofDays(7), REQUEST_LIMIT).get();
@Override
public String process(ByteBuffer data) throws Exception {
return Charset.forName("UTF-8").decode(data).toString();
}
@Override
protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
try {
if (limit != null) {
limit.acquirePermit();
}
return super.fetchData(url, lastModified);
} catch (FileNotFoundException e) {
return ByteBuffer.allocate(0);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
@Override
protected Cache getCache() {
return CacheManager.getInstance().getCache("web-datasource-lv2");
}
};
return asMap(readJson(json.get()));
return asMap(readJson(json));
}
public Map<String, String> getMovieInfo(Integer i, String t, String y, boolean tomatoes) throws IOException {
public Map<String, String> getMovieInfo(Integer i, String t, String y, boolean tomatoes) throws Exception {
// e.g. http://www.imdbapi.com/?i=tt0379786&r=xml&tomatoes=true
Map<String, Object> param = new LinkedHashMap<String, Object>(2);
if (i != null) {
@ -188,10 +156,10 @@ public class OMDbClient implements MovieIdentificationService {
}
param.put("tomatoes", String.valueOf(tomatoes));
return getInfoMap(request(param, REQUEST_LIMIT));
return getInfoMap(request(param));
}
public MovieInfo getMovieInfo(Movie movie) throws IOException {
public MovieInfo getMovieInfo(Movie movie) throws Exception {
Map<String, String> data = movie.getImdbId() > 0 ? getMovieInfo(movie.getImdbId(), null, null, false) : getMovieInfo(null, movie.getName(), String.valueOf(movie.getYear()), false);
// sanity check
@ -243,7 +211,7 @@ public class OMDbClient implements MovieIdentificationService {
}
}
} catch (DateTimeParseException e) {
Logger.getLogger(OMDbClient.class.getName()).log(Level.WARNING, String.format("Bad date: %s: %s", value, e.getMessage()));
debug.warning(format("Bad date: %s =~ %s => %s", value, format, e));
}
}
return null;

10
test/net/filebot/web/AcoustIDClientTest.java
View File

@ -4,10 +4,6 @@ import static org.junit.Assert.*;
import java.io.IOException;
import net.sf.ehcache.CacheManager;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
public class AcoustIDClientTest {
@ -38,10 +34,4 @@ public class AcoustIDClientTest {
assertEquals("聽媽媽的話", info.getTitle());
}
@BeforeClass
@AfterClass
public static void clearCache() {
CacheManager.getInstance().clearAll();
}
}

8
test/net/filebot/web/OMDbClientTest.java
View File

@ -24,12 +24,12 @@ public class OMDbClientTest {
@Test
public void searchMovie2() throws Exception {
List<Movie> results = client.searchMovie("The Illusionist", null);
List<Movie> results = client.searchMovie("The Terminator", null);
Movie movie = results.get(0);
assertEquals("The Illusionist", movie.getName());
assertEquals(2006, movie.getYear());
assertEquals(443543, movie.getImdbId(), 0);
assertEquals("The Terminator", movie.getName());
assertEquals(1984, movie.getYear());
assertEquals(88247, movie.getImdbId());
}
@Test