* fully support ETag caching mechanism in TheMovieDB client

source/ehcache.xml

@@ -50,7 +50,7 @@
 	Very long-lived cache (4 months) anime/series lists, movie index, etc
 	-->
 	<cache name="web-persistent-datasource"
-		maxElementsInMemory="50"
+		maxElementsInMemory="200"
 		maxElementsOnDisk="50000"
 		eternal="false"
 		timeToIdleSeconds="10512000"

source/net/sourceforge/filebot/web/CachedXmlResource.java

@@ -17,6 +17,10 @@ public class CachedXmlResource extends AbstractCachedResource<String, String> {
 		super(resource, String.class, ONE_WEEK, 2, 1000);
 	}
 
+	public CachedXmlResource(String resource, long expirationTime, int retryCountLimit, long retryWaitTime) {
+		super(resource, String.class, expirationTime, retryCountLimit, retryWaitTime);
+	}
+
 	@Override
 	protected Cache getCache() {
 		return CacheManager.getInstance().getCache("web-persistent-datasource");

source/net/sourceforge/filebot/web/ETagCachedResource.java

@@ -0,0 +1,51 @@
+package net.sourceforge.filebot.web;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Element;
+
+public abstract class ETagCachedResource<T extends Serializable> extends CachedResource<T> {
+
+	public ETagCachedResource(String resource, Class<T> type) {
+		super(resource, type, ONE_WEEK, 2, 1000);
+	}
+
+	public ETagCachedResource(String resource, Class<T> type, long expirationTime, int retryCountLimit, long retryWaitTime) {
+		super(resource, type, expirationTime, retryCountLimit, retryWaitTime);
+	}
+
+	@Override
+	protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
+		String etagKey = "ETag" + ":" + url.toString();
+		Element etag = getCache().get(etagKey);
+
+		Map<String, String> requestParameters = new HashMap<String, String>();
+		if (etag != null && etag.getObjectValue() != null) {
+			requestParameters.put("If-None-Match", etag.getObjectValue().toString());
+		}
+
+		// If-Modified-Since must not be set if If-None-Match is set
+		Map<String, List<String>> responseHeaders = new HashMap<String, List<String>>();
+		ByteBuffer data = WebRequest.fetch(url, requestParameters.size() > 0 ? -1 : lastModified, requestParameters, responseHeaders);
+
+		if (data != null && responseHeaders.containsKey("ETag")) {
+			getCache().put(new Element(etagKey, responseHeaders.get("ETag").get(0)));
+		}
+
+		return data;
+	}
+
+	@Override
+	protected Cache getCache() {
+		return CacheManager.getInstance().getCache("web-persistent-datasource");
+	}
+
+}

source/net/sourceforge/filebot/web/FloodLimit.java

@@ -1,36 +1,37 @@
-
 package net.sourceforge.filebot.web;
 
-
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
-
 public class FloodLimit {
-	
+
 	private final Semaphore permits;
 	private final ScheduledThreadPoolExecutor timer = new ScheduledThreadPoolExecutor(1);
-	
+
 	private final long releaseDelay;
 	private final TimeUnit timeUnit;
-	
+
-	
 	public FloodLimit(int permitLimit, long releaseDelay, TimeUnit timeUnit) {
 		this.permits = new Semaphore(permitLimit, true);
 		this.releaseDelay = releaseDelay;
 		this.timeUnit = timeUnit;
 	}
-	
+
-	
+	public ScheduledFuture<?> acquirePermit() throws InterruptedException {
-	public void acquirePermit() throws InterruptedException {
 		permits.acquire();
-		timer.schedule(new ReleasePermit(), releaseDelay, timeUnit);
+		return timer.schedule(new ReleasePermit(), releaseDelay, timeUnit);
 	}
-	
+
-	
+	public void releaseNow(ScheduledFuture<?> future) {
+		if (future.cancel(false)) {
+			permits.release();
+		}
+	}
+
 	private class ReleasePermit implements Runnable {
-		
+
 		@Override
 		public void run() {
 			permits.release();

source/net/sourceforge/filebot/web/TMDbClient.java

@@ -21,14 +21,13 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Scanner;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import javax.swing.Icon;
 
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
 import net.sourceforge.filebot.ResourceManager;
 import net.sourceforge.filebot.web.TMDbClient.MovieInfo.MovieProperty;
 import net.sourceforge.filebot.web.TMDbClient.Person.PersonProperty;
@@ -250,7 +249,7 @@ public class TMDbClient implements MovieIdentificationService {
 
 		URL url = new URL("http", host, "/" + version + "/" + resource + "?" + encodeParameters(data, true));
 
-		CachedResource<String> json = new CachedResource<String>(url.toString(), String.class) {
+		CachedResource<String> json = new ETagCachedResource<String>(url.toString(), String.class) {
 
 			@Override
 			public String process(ByteBuffer data) throws Exception {
@@ -260,21 +259,22 @@ public class TMDbClient implements MovieIdentificationService {
 			@Override
 			protected ByteBuffer fetchData(URL url, long lastModified) throws IOException {
 				try {
+					ScheduledFuture<?> permit = null;
 					if (limit != null) {
-						limit.acquirePermit();
+						permit = limit.acquirePermit();
 					}
-					return super.fetchData(url, lastModified);
+
+					ByteBuffer data = super.fetchData(url, lastModified);
+					if (data == null && limit != null && permit != null) {
+						limit.releaseNow(permit);
+					}
+					return data;
 				} catch (FileNotFoundException e) {
 					return ByteBuffer.allocate(0);
 				} catch (InterruptedException e) {
 					throw new RuntimeException(e);
 				}
 			}
-
-			@Override
-			protected Cache getCache() {
-				return CacheManager.getInstance().getCache("web-datasource");
-			}
 		};
 
 		JSONObject object = (JSONObject) JSONValue.parse(json.get());

source/net/sourceforge/filebot/web/WebRequest.java

@@ -16,6 +16,7 @@ import java.nio.charset.Charset;
 import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
 import java.security.cert.X509Certificate;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.logging.Level;
@@ -110,14 +111,14 @@ public final class WebRequest {
 	}
 
 	public static ByteBuffer fetch(URL resource) throws IOException {
-		return fetch(resource, 0, null);
+		return fetch(resource, 0, null, null);
 	}
 
 	public static ByteBuffer fetchIfModified(URL resource, long ifModifiedSince) throws IOException {
-		return fetch(resource, ifModifiedSince, null);
+		return fetch(resource, ifModifiedSince, null, null);
 	}
 
-	public static ByteBuffer fetch(URL url, long ifModifiedSince, Map<String, String> requestParameters) throws IOException {
+	public static ByteBuffer fetch(URL url, long ifModifiedSince, Map<String, String> requestParameters, Map<String, List<String>> responseParameters) throws IOException {
 		URLConnection connection = url.openConnection();
 		if (ifModifiedSince > 0) {
 			connection.setIfModifiedSince(ifModifiedSince);
@@ -146,6 +147,11 @@ public final class WebRequest {
 			in = new InflaterInputStream(in, new Inflater(true));
 		}
 
+		// store response headers
+		if (responseParameters != null) {
+			responseParameters.putAll(connection.getHeaderFields());
+		}
+
 		ByteBufferOutputStream buffer = new ByteBufferOutputStream(contentLength >= 0 ? contentLength : 4 * 1024);
 		try {
 			// read all