From bbed902c63e44b9d371f236efe1276c55ccbad80 Mon Sep 17 00:00:00 2001 From: Reinhard Pointner Date: Sun, 6 Mar 2016 22:21:13 +0000 Subject: [PATCH] Experiment with new CachedResource framework --- source/ehcache.xml | 94 +------------ source/net/filebot/Cache.java | 53 +++++--- .../net/filebot/format/MediaBindingBean.java | 2 +- source/net/filebot/web/CachedResource2.java | 123 ++++++++++++++++++ source/net/filebot/web/OMDbClient.java | 64 +++------ test/net/filebot/web/AcoustIDClientTest.java | 10 -- test/net/filebot/web/OMDbClientTest.java | 8 +- 7 files changed, 184 insertions(+), 170 deletions(-) create mode 100644 source/net/filebot/web/CachedResource2.java diff --git a/source/ehcache.xml b/source/ehcache.xml index e77d5595..6779c8bd 100644 --- a/source/ehcache.xml +++ b/source/ehcache.xml @@ -1,28 +1,9 @@ - - - + + - - - - - - - - - - - - - - - - - - diff --git a/source/net/filebot/Cache.java b/source/net/filebot/Cache.java index ce68e380..065c810d 100644 --- a/source/net/filebot/Cache.java +++ b/source/net/filebot/Cache.java @@ -1,13 +1,16 @@ package net.filebot; +import static java.nio.charset.StandardCharsets.*; import static net.filebot.Logging.*; import java.io.Serializable; +import java.net.URL; import java.time.Duration; import java.util.Arrays; -import java.util.concurrent.Callable; import java.util.function.Predicate; +import net.filebot.web.CachedResource2; +import net.filebot.web.FloodLimit; import net.sf.ehcache.Element; public class Cache { @@ -24,51 +27,60 @@ public class Cache { public Object get(Object key) { try { - return cache.get(key).getObjectValue(); + Element element = cache.get(key); + if (element != null) { + return element.getObjectValue(); + } } catch (Exception e) { - debug.warning(format("Bad cache state: %s => %s", key, e)); + e.printStackTrace(); + debug.warning(format("Cache get: %s => %s", key, e)); } return null; } - public Object computeIf(Object key, Predicate condition, Callable callable) throws Exception { + public Object computeIf(Object key, Predicate condition, Compute compute) throws Exception { // get if present + Element element = null; try { - Element element = cache.get(key); + element = cache.get(key); if (element != null && condition.test(element)) { return element.getObjectValue(); } } catch (Exception e) { - debug.warning(format("Bad cache state: %s => %s", key, e)); + debug.warning(format("Cache get: %s => %s", key, e)); } // compute if absent - Object value = callable.call(); + Object value = compute.apply(element); try { cache.put(new Element(key, value)); } catch (Exception e) { - debug.warning(format("Bad cache state: %s => %s", key, e)); + debug.warning(format("Cache put: %s => %s", key, e)); } return value; } - public Object computeIfAbsent(Object key, Callable callable) throws Exception { - return computeIf(key, Element::isExpired, callable); + public Object computeIfAbsent(Object key, Compute compute) throws Exception { + return computeIf(key, isAbsent(), compute); } - public Object computeIfStale(Object key, Duration expirationTime, Callable callable) throws Exception { - return computeIf(key, isStale(expirationTime), callable); + public Object computeIfStale(Object key, Duration expirationTime, Compute compute) throws Exception { + return computeIf(key, isStale(expirationTime), compute); } - private Predicate isStale(Duration expirationTime) { - return (element) -> element.isExpired() || System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis(); + public Predicate isAbsent() { + return (element) -> element.getObjectValue() == null; + } + + public Predicate isStale(Duration expirationTime) { + return (element) -> System.currentTimeMillis() - element.getLatestOfCreationAndUpdateTime() < expirationTime.toMillis(); } public void put(Object key, Object value) { try { cache.put(new Element(key, value)); } catch (Exception e) { - debug.warning(format("Bad cache state: %s => %s", key, e)); + debug.warning(format("Cache put: %s => %s", key, e)); } } @@ -76,10 +88,15 @@ public class Cache { try { cache.remove(key); } catch (Exception e) { - debug.warning(format("Bad cache state: %s => %s", key, e)); + debug.warning(format("Cache remove: %s => %s", key, e)); } } + @FunctionalInterface + public interface Compute { + R apply(Element element) throws Exception; + } + @Deprecated public T get(Object key, Class type) { return type.cast(get(key)); @@ -114,4 +131,8 @@ public class Cache { } } + public CachedResource2 resource(String url, Duration expirationTime, FloodLimit limit) { + return new CachedResource2(url, URL::new, CachedResource2.fetchIfModified(limit), CachedResource2.decode(UTF_8), expirationTime, this); + } + } diff --git a/source/net/filebot/format/MediaBindingBean.java b/source/net/filebot/format/MediaBindingBean.java index 8ff44a30..da38f5fa 100644 --- a/source/net/filebot/format/MediaBindingBean.java +++ b/source/net/filebot/format/MediaBindingBean.java @@ -447,7 +447,7 @@ public class MediaBindingBean { // calculate checksum from file Cache cache = Cache.getCache("crc32", CacheType.Ephemeral); - return (String) cache.computeIfAbsent(inferredMediaFile, () -> crc32(inferredMediaFile)); + return (String) cache.computeIfAbsent(inferredMediaFile, element -> crc32(inferredMediaFile)); } @Define("fn") diff --git a/source/net/filebot/web/CachedResource2.java b/source/net/filebot/web/CachedResource2.java new file mode 100644 index 00000000..874340ce --- /dev/null +++ b/source/net/filebot/web/CachedResource2.java @@ -0,0 +1,123 @@ +package net.filebot.web; + +import static net.filebot.Logging.*; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.time.Duration; +import java.util.concurrent.Callable; + +import net.filebot.Cache; + +public class CachedResource2 { + + public static final int DEFAULT_RETRY_LIMIT = 2; + public static final Duration DEFAULT_RETRY_DELAY = Duration.ofSeconds(2); + + protected final K key; + + protected final Source source; + protected final Fetch fetch; + protected final Parse parse; + + protected final Duration expirationTime; + + protected final int retryCountLimit; + protected final long retryWaitTime; + + protected final Cache cache; + + public CachedResource2(K key, Source source, Fetch fetch, Parse parse, Duration expirationTime, Cache cache) { + this(key, source, fetch, parse, DEFAULT_RETRY_LIMIT, DEFAULT_RETRY_DELAY, expirationTime, cache); + } + + public CachedResource2(K key, Source source, Fetch fetch, Parse parse, int retryCountLimit, Duration retryWaitTime, Duration expirationTime, Cache cache) { + this.key = key; + this.source = source; + this.fetch = fetch; + this.parse = parse; + this.expirationTime = expirationTime; + this.retryCountLimit = retryCountLimit; + this.retryWaitTime = retryWaitTime.toMillis(); + this.cache = cache; + } + + @SuppressWarnings("unchecked") + public synchronized R get() throws Exception { + return (R) cache.computeIfStale(key, expirationTime, element -> { + URL resource = source.source(key); + long lastModified = element == null ? 0 : element.getLatestOfCreationAndUpdateTime(); + + debug.fine(format("Fetch %s (If-Modified-Since: %tc)", resource, lastModified)); + + try { + ByteBuffer data = retry(() -> fetch.fetch(resource, lastModified), retryCountLimit, lastModified); + + // 304 Not Modified + if (data == null && element != null && element.getObjectValue() != null) { + return element.getObjectValue(); + } + + return parse.parse(data); + } catch (IOException e) { + debug.fine(format("Fetch failed => %s", e)); + + // use previously cached data if possible + if (element == null || element.getObjectValue() == null) { + throw e; + } + return element.getObjectKey(); + } + }); + } + + protected T retry(Callable callable, int retryCount, long retryWaitTime) throws Exception { + try { + return callable.call(); + } catch (FileNotFoundException e) { + // resource does not exist, do not retry + throw e; + } catch (IOException e) { + // retry or rethrow exception + if (retryCount > 0) { + throw e; + } + Thread.sleep(retryWaitTime); + return retry(callable, retryCount - 1, retryWaitTime * 2); + } + } + + @FunctionalInterface + public interface Source { + URL source(K key) throws Exception; + } + + @FunctionalInterface + public interface Fetch { + ByteBuffer fetch(URL url, long lastModified) throws Exception; + } + + @FunctionalInterface + public interface Parse { + R parse(ByteBuffer bytes) throws Exception; + } + + public static Parse decode(Charset charset) { + return (bb) -> charset.decode(bb).toString(); + } + + public static Fetch fetchIfModified(FloodLimit limit) { + return (url, lastModified) -> { + try { + limit.acquirePermit(); + return WebRequest.fetchIfModified(url, lastModified); + } catch (FileNotFoundException e) { + return ByteBuffer.allocate(0); + } + }; + } + +} diff --git a/source/net/filebot/web/OMDbClient.java b/source/net/filebot/web/OMDbClient.java index bf08a748..ced8575d 100644 --- a/source/net/filebot/web/OMDbClient.java +++ b/source/net/filebot/web/OMDbClient.java @@ -2,16 +2,13 @@ package net.filebot.web; import static java.util.Collections.*; import static java.util.stream.Collectors.*; +import static net.filebot.Logging.*; import static net.filebot.util.JsonUtilities.*; import static net.filebot.util.StringUtilities.*; import static net.filebot.web.WebRequest.*; import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URL; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; +import java.time.Duration; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoField; @@ -27,27 +24,22 @@ import java.util.Map.Entry; import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.swing.Icon; +import net.filebot.Cache; +import net.filebot.CacheType; import net.filebot.ResourceManager; import net.filebot.web.TMDbClient.MovieInfo; import net.filebot.web.TMDbClient.MovieInfo.MovieProperty; import net.filebot.web.TMDbClient.Person; -import net.sf.ehcache.Cache; -import net.sf.ehcache.CacheManager; public class OMDbClient implements MovieIdentificationService { private static final FloodLimit REQUEST_LIMIT = new FloodLimit(20, 10, TimeUnit.SECONDS); - private final String protocol = "http"; - private final String host = "www.omdbapi.com"; - @Override public String getName() { return "OMDb"; @@ -70,7 +62,7 @@ public class OMDbClient implements MovieIdentificationService { } @Override - public List searchMovie(String query, Locale locale) throws IOException { + public List searchMovie(String query, Locale locale) throws Exception { // query by name with year filter if possible Matcher nameYear = Pattern.compile("(.+)\\b(19\\d{2}|20\\d{2})$").matcher(query); if (nameYear.matches()) { @@ -80,14 +72,14 @@ public class OMDbClient implements MovieIdentificationService { } } - public List searchMovie(String movieName, int movieYear) throws IOException { + public List searchMovie(String movieName, int movieYear) throws Exception { Map param = new LinkedHashMap(2); param.put("s", movieName); if (movieYear > 0) { param.put("y", movieYear); } - Map response = request(param, REQUEST_LIMIT); + Map response = request(param); List result = new ArrayList(); for (Object it : getArray(response, "Search")) { @@ -141,40 +133,16 @@ public class OMDbClient implements MovieIdentificationService { throw new UnsupportedOperationException(); } - public Map request(Map parameters, final FloodLimit limit) throws IOException { - URL url = new URL(protocol, host, "/?" + encodeParameters(parameters, true)); + public Map request(Map parameters) throws Exception { + String url = "http://www.omdbapi.com/?" + encodeParameters(parameters, true); - CachedResource json = new CachedResource(url.toString(), String.class, CachedResource.ONE_WEEK) { + Cache cache = Cache.getCache(getName(), CacheType.Weekly); + String json = cache.resource(url, Duration.ofDays(7), REQUEST_LIMIT).get(); - @Override - public String process(ByteBuffer data) throws Exception { - return Charset.forName("UTF-8").decode(data).toString(); - } - - @Override - protected ByteBuffer fetchData(URL url, long lastModified) throws IOException { - try { - if (limit != null) { - limit.acquirePermit(); - } - return super.fetchData(url, lastModified); - } catch (FileNotFoundException e) { - return ByteBuffer.allocate(0); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - protected Cache getCache() { - return CacheManager.getInstance().getCache("web-datasource-lv2"); - } - }; - - return asMap(readJson(json.get())); + return asMap(readJson(json)); } - public Map getMovieInfo(Integer i, String t, String y, boolean tomatoes) throws IOException { + public Map getMovieInfo(Integer i, String t, String y, boolean tomatoes) throws Exception { // e.g. http://www.imdbapi.com/?i=tt0379786&r=xml&tomatoes=true Map param = new LinkedHashMap(2); if (i != null) { @@ -188,10 +156,10 @@ public class OMDbClient implements MovieIdentificationService { } param.put("tomatoes", String.valueOf(tomatoes)); - return getInfoMap(request(param, REQUEST_LIMIT)); + return getInfoMap(request(param)); } - public MovieInfo getMovieInfo(Movie movie) throws IOException { + public MovieInfo getMovieInfo(Movie movie) throws Exception { Map data = movie.getImdbId() > 0 ? getMovieInfo(movie.getImdbId(), null, null, false) : getMovieInfo(null, movie.getName(), String.valueOf(movie.getYear()), false); // sanity check @@ -243,7 +211,7 @@ public class OMDbClient implements MovieIdentificationService { } } } catch (DateTimeParseException e) { - Logger.getLogger(OMDbClient.class.getName()).log(Level.WARNING, String.format("Bad date: %s: %s", value, e.getMessage())); + debug.warning(format("Bad date: %s =~ %s => %s", value, format, e)); } } return null; diff --git a/test/net/filebot/web/AcoustIDClientTest.java b/test/net/filebot/web/AcoustIDClientTest.java index 38309d89..fa183243 100644 --- a/test/net/filebot/web/AcoustIDClientTest.java +++ b/test/net/filebot/web/AcoustIDClientTest.java @@ -4,10 +4,6 @@ import static org.junit.Assert.*; import java.io.IOException; -import net.sf.ehcache.CacheManager; - -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.Test; public class AcoustIDClientTest { @@ -38,10 +34,4 @@ public class AcoustIDClientTest { assertEquals("聽媽媽的話", info.getTitle()); } - @BeforeClass - @AfterClass - public static void clearCache() { - CacheManager.getInstance().clearAll(); - } - } diff --git a/test/net/filebot/web/OMDbClientTest.java b/test/net/filebot/web/OMDbClientTest.java index dbf6bc0d..7ae8d5af 100644 --- a/test/net/filebot/web/OMDbClientTest.java +++ b/test/net/filebot/web/OMDbClientTest.java @@ -24,12 +24,12 @@ public class OMDbClientTest { @Test public void searchMovie2() throws Exception { - List results = client.searchMovie("The Illusionist", null); + List results = client.searchMovie("The Terminator", null); Movie movie = results.get(0); - assertEquals("The Illusionist", movie.getName()); - assertEquals(2006, movie.getYear()); - assertEquals(443543, movie.getImdbId(), 0); + assertEquals("The Terminator", movie.getName()); + assertEquals(1984, movie.getYear()); + assertEquals(88247, movie.getImdbId()); } @Test