+ support IMDb as movie db

source/net/sourceforge/filebot/WebServices.java

@@ -46,7 +46,7 @@ public final class WebServices {
 	
 	
 	public static MovieIdentificationService[] getMovieIdentificationServices() {
-		return new MovieIdentificationService[] { OpenSubtitles, TMDb };
+		return new MovieIdentificationService[] { OpenSubtitles, IMDb, TMDb };
 	}
 	
 	

source/net/sourceforge/filebot/web/IMDbClient.java

@@ -5,6 +5,7 @@ package net.sourceforge.filebot.web;
 import static net.sourceforge.filebot.web.WebRequest.*;
 import static net.sourceforge.tuned.XPathUtilities.*;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -27,7 +28,7 @@ import net.sf.ehcache.CacheManager;
 import net.sourceforge.filebot.ResourceManager;
 
 
-public class IMDbClient extends AbstractEpisodeListProvider {
+public class IMDbClient extends AbstractEpisodeListProvider implements MovieIdentificationService {
 	
 	private final String host = "www.imdb.com";
 	
@@ -52,11 +53,9 @@ public class IMDbClient extends AbstractEpisodeListProvider {
 	
 	@Override
 	public List<SearchResult> fetchSearchResult(String query, Locale locale) throws IOException, SAXException {
-		URL searchUrl = new URL("http", host, "/find?s=tt&q=" + encode(query));
+		Document dom = parsePage(new URL("http", host, "/find?s=tt&q=" + encode(query)));
-		Document dom = getHtmlDocument(openConnection(searchUrl));
 		
 		List<Node> nodes = selectNodes("//TABLE//A[following-sibling::SMALL[contains(.,'series')]]", dom);
-		
 		List<SearchResult> results = new ArrayList<SearchResult>(nodes.size());
 		
 		for (Node node : nodes) {
@@ -69,14 +68,9 @@ public class IMDbClient extends AbstractEpisodeListProvider {
 		
 		// we might have been redirected to the movie page
 		if (results.isEmpty()) {
-			try {
+			Movie movie = scrapeMovie(dom);
-				String name = normalizeName(selectString("//H1/text()", dom));
+			if (movie != null) {
-				String year = new Scanner(selectString("//H1//SPAN", dom)).useDelimiter("\\D+").next();
+				results.add(movie);
-				String url = selectString("//LINK[@rel='canonical']/@href", dom);
-				
-				results.add(new Movie(name, Integer.parseInt(year), getImdbId(url)));
-			} catch (Exception e) {
-				// ignore, we probably got redirected to an error page
 			}
 		}
 		
@@ -87,7 +81,7 @@ public class IMDbClient extends AbstractEpisodeListProvider {
 	@Override
 	public List<Episode> fetchEpisodeList(SearchResult searchResult, Locale locale) throws IOException, SAXException {
 		Movie movie = (Movie) searchResult;
-		Document dom = getHtmlDocument(openConnection(getEpisodeListLink(searchResult).toURL()));
+		Document dom = parsePage(getEpisodeListLink(searchResult).toURL());
 		
 		String seriesName = normalizeName(selectString("//H1/A", dom));
 		Date year = new Date(movie.getYear(), 0, 0);
@@ -112,16 +106,6 @@ public class IMDbClient extends AbstractEpisodeListProvider {
 	}
 	
 	
-	protected URLConnection openConnection(URL url) throws IOException {
-		URLConnection connection = url.openConnection();
-		
-		// IMDb refuses default user agent (Java/1.6.0_12)
-		connection.addRequestProperty("User-Agent", "Scraper");
-		
-		return connection;
-	}
-	
-
 	protected String normalizeName(String name) {
 		// remove quotation marks
 		return name.replaceAll("\"", "");
@@ -154,4 +138,72 @@ public class IMDbClient extends AbstractEpisodeListProvider {
 			throw new RuntimeException(e);
 		}
 	}
+	
+	
+	@Override
+	public List<Movie> searchMovie(String query, Locale locale) throws Exception {
+		Document dom = parsePage(new URL("http", host, "/find?s=tt&q=" + encode(query)));
+		
+		// select movie links followed by year in parenthesis
+		List<Node> nodes = selectNodes("//TABLE//A[string-length(substring-after(substring-before(following::text(),')'),'(')) = 4 and count(following-sibling::SMALL) = 0]", dom);
+		List<Movie> results = new ArrayList<Movie>(nodes.size());
+		
+		for (Node node : nodes) {
+			String name = node.getTextContent().trim();
+			String year = node.getNextSibling().getTextContent().trim().replaceAll("[\\p{Punct}\\p{Space}]+", ""); // remove non-number characters
+			String href = getAttribute("href", node);
+			
+			try {
+				results.add(new Movie(name, Integer.parseInt(year), getImdbId(href)));
+			} catch (NumberFormatException e) {
+				// ignore illegal movies (TV Shows, Videos, Video Games, etc)
+			}
+		}
+		
+		// we might have been redirected to the movie page
+		if (results.isEmpty()) {
+			Movie movie = scrapeMovie(dom);
+			if (movie != null) {
+				results.add(movie);
+			}
+		}
+		
+		return results;
+	}
+	
+	
+	protected Movie scrapeMovie(Document dom) {
+		try {
+			String name = normalizeName(selectString("//H1/text()", dom));
+			String year = new Scanner(selectString("//H1//SPAN", dom)).useDelimiter("\\D+").next();
+			String url = selectString("//LINK[@rel='canonical']/@href", dom);
+			return new Movie(name, Integer.parseInt(year), getImdbId(url));
+		} catch (Exception e) {
+			// ignore, we probably got redirected to an error page
+			return null;
+		}
+	}
+	
+	
+	@Override
+	public Movie getMovieDescriptor(int imdbid, Locale locale) throws Exception {
+		return scrapeMovie(parsePage(new URL("http", host, String.format("/title/tt%07d/", imdbid))));
+	}
+	
+	
+	protected Document parsePage(URL url) throws IOException, SAXException {
+		URLConnection connection = url.openConnection();
+		
+		// IMDb refuses default user agent (Java/1.6.0_12)
+		connection.addRequestProperty("User-Agent", "Mozilla");
+		
+		return getHtmlDocument(connection);
+	}
+	
+	
+	@Override
+	public Movie[] getMovieDescriptors(File[] movieFiles, Locale locale) throws Exception {
+		return new Movie[movieFiles.length]; // UNSUPPORTED OPERATION => EMPTY RESULT
+	}
+	
 }

source/net/sourceforge/filebot/web/WebRequest.java

@@ -98,7 +98,11 @@ public final class WebRequest {
 	
 	public static Document getDocument(InputSource source) throws IOException, SAXException {
 		try {
-			return DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(source);
+			DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+			factory.setValidating(false);
+			factory.setFeature("http://xml.org/sax/features/namespaces", false);
+			factory.setFeature("http://xml.org/sax/features/validation", false);
+			return factory.newDocumentBuilder().parse(source);
 		} catch (ParserConfigurationException e) {
 			// will never happen
 			throw new RuntimeException(e);

test/net/sourceforge/filebot/web/IMDbClientTest.java

@@ -104,6 +104,42 @@ public class IMDbClientTest {
 	}
 	
 	
+	@Test
+	public void searchMovie() throws Exception {
+		List<Movie> results = imdb.searchMovie("Avatar", null);
+		
+		assertEquals(26, results.size());
+		Movie movie = (Movie) results.get(0);
+		
+		assertEquals("Avatar", movie.getName());
+		assertEquals(2009, movie.getYear());
+		assertEquals(499549, movie.getImdbId(), 0);
+	}
+	
+	
+	@Test
+	public void searchMovieRedirect() throws Exception {
+		List<Movie> results = imdb.searchMovie("battle angel alita", null);
+		
+		assertEquals(1, results.size());
+		Movie movie = (Movie) results.get(0);
+		
+		assertEquals("Battle Angel", movie.getName());
+		assertEquals(1993, movie.getYear());
+		assertEquals(107061, movie.getImdbId(), 0);
+	}
+	
+	
+	@Test
+	public void getMovieDescriptor() throws Exception {
+		Movie movie = imdb.getMovieDescriptor(499549, null);
+		
+		assertEquals("Avatar", movie.getName());
+		assertEquals(2009, movie.getYear());
+		assertEquals(499549, movie.getImdbId(), 0);
+	}
+	
+	
 	@Test
 	public void getEpisodeListLink() throws Exception {
 		assertEquals("http://www.imdb.com/title/tt0407362/episodes", imdb.getEpisodeListLink(new Movie("Battlestar Galactica", 2004, 407362)).toString());

website/data/query-blacklist.txt

@@ -12,6 +12,7 @@ PROPER
 READNFO
 REPACK
 RETAIL
+HDRip
 sample[s]?$
 ShareReactor
 ShareZONE