* generate review.json and easily collect good reviews in a separate csv file

BuildData.groovy

@@ -0,0 +1,168 @@
+
+// ------------------------------------------------------------------------- //
+
+
+def sortRegexList(path) {
+	def set = new TreeSet(String.CASE_INSENSITIVE_ORDER)
+	new File(path).eachLine('UTF-8'){
+		// check if regex compiles
+		set += java.util.regex.Pattern.compile(it.trim()).pattern()
+	}
+	
+	def out = set.join('\n').saveAs(path)
+	println "$out\n$out.text\n"
+}
+
+
+// sort and check shared regex collections
+sortRegexList("website/data/release-groups.txt")
+sortRegexList("website/data/query-blacklist.txt")
+sortRegexList("website/data/exclude-blacklist.txt")
+sortRegexList("website/data/series-mappings.txt")
+
+
+// ------------------------------------------------------------------------- //
+
+
+def reviews = []
+new File('reviews.csv').eachLine('UTF-8'){ def s = it.split(';', 3); reviews << [user: s[0], date: s[1], text: s[2].replaceAll(/^["]|["]$/, '').replaceAll(/["]{2}/, '"')] }
+reviews = reviews.sort{ it.date }
+
+def json = new groovy.json.JsonBuilder()
+json.call(reviews as List)
+json.toPrettyString().saveAs('website/reviews.json')
+println "Reviews: " + reviews.size()
+
+
+// ------------------------------------------------------------------------- //
+
+
+def series_out  = new File("website/data/series.list.gz")
+def movies_out  = new File("website/data/movies.txt.gz")
+def thetvdb_out = new File("website/data/thetvdb.txt.gz")
+def anidb_out   = new File("website/data/anidb.txt.gz")
+
+def gz(file, lines) {
+	file.withOutputStream{ out ->
+		new java.util.zip.GZIPOutputStream(out).withWriter('UTF-8'){ writer ->
+			lines.each{ writer.append(it).append('\n') }
+		}
+	}
+}
+
+
+// ------------------------------------------------------------------------- //
+
+// BUILD movies.txt.gz
+def omdb = new TreeSet({ a, b -> a[0].compareTo(b[0]) } as Comparator)
+new File('omdb.txt').eachLine('Windows-1252'){
+	def line = it.split(/\t/)
+	
+	if (line.length > 11 && line[0] ==~ /\d+/) {
+		def imdbid = line[1].substring(2).toInteger()
+		def name = line[2]
+		def year = line[3].toInteger()
+		def runtime = line[5]
+		def rating = tryQuietly{ line[11].toFloat() } ?: 0
+		def votes = tryQuietly{ line[12].replaceAll(/\D/, '').toInteger() } ?: 0
+		
+		if ((year >= 1970 && (runtime =~ /h/ || votes >= 200) && rating >= 1 && votes >= 50) || (votes >= 2000)) {
+			line = line*.replaceAll(/\s+/, ' ')*.trim()
+			omdb << [imdbid, name, year]
+		}
+	}
+}
+omdb = omdb.findAll{ it[0] <= 9999999 && it[1] =~ /^[A-Z0-9]/ && it[1] =~ /[\p{Alpha}]{3}/ && it[1].length() >= 4}.collect{ [it[0].pad(7), it[1], it[2]] }
+
+// save movie data
+def movies = omdb.findAll{ it.size() >= 3 && !it[1].startsWith('"') }
+def movieSorter = new TreeMap(String.CASE_INSENSITIVE_ORDER)
+movies.each{ movieSorter.put([it[1], it[2], it[0]].join('\t'), it) }
+movies = movieSorter.values().collect{ it.join('\t') }
+
+gz(movies_out, movies)
+println "Movie Count: " + movies.size()
+
+// sanity check
+if (movies.size() < 50000) { throw new Exception('Movie index sanity failed') }
+
+// ------------------------------------------------------------------------- //
+
+// BUILD thetvdb-index.gz
+def tvdb = new HashMap()
+def tvdb_txt = new File('tvdb.txt')
+new File('tvdb.txt').eachLine{
+	def line = it.split('\t', 5).toList()
+	tvdb.put(line[0] as Integer, [line[0] as Integer, line[1], line[2], line[3], line[4] as Integer])
+}
+
+def tvdb_updates = new File('updates_all.xml').text.xml.'**'.Series.findResults{ s -> tryQuietly{ [id:s.id.text() as Integer, time:s.time.text() as Integer] } }
+tvdb_updates.each{ update ->
+	if (tvdb[update.id] == null || update.time > tvdb[update.id][4]) {
+		try {
+			retry(2, 500) {
+				def xml = new URL("http://thetvdb.com/api/BA864DEE427E384A/series/${update.id}/en.xml").fetch().text.xml
+				def imdbid = xml.'**'.IMDB_ID.text()
+				def tvdb_name = xml.'**'.SeriesName.text()
+				def imdb_name = _guarded{
+					if (imdbid =~ /tt(\d+)/) {
+						def dom = IMDb.parsePage(IMDb.getMoviePageLink(imdbid.match(/tt(\d+)/) as int).toURL())
+						return net.sourceforge.tuned.XPathUtilities.selectString("//META[@property='og:title']/@content", dom)
+					}
+				}
+				def data = [update.id, imdbid ?: '', tvdb_name ?: '', imdb_name ?: '', update.time]
+				tvdb.put(update.id, data)
+				println "Update $update => $data"
+			}
+		}
+		catch(Throwable e) {
+			def data = [update.id, '', '', '', update.time]
+			tvdb.put(update.id, data)
+			println "Update $update => $data"
+		}
+	}
+}
+
+tvdb.values().findResults{ it.join('\t') }.join('\n').saveAs(tvdb_txt)
+
+
+def thetvdb_index = []
+tvdb.values().each{
+	def n1 = it[2].trim()
+	def n2 = it[3].replaceAll(/^(?i)(The|A)\s/, '').replaceAll(/\s&\s/, ' and ').replaceAll(/\([^\)]*\)$/, '').trim()
+	if (similarity(n1,n2) < 1) {
+		thetvdb_index << [it[0], n1]
+		thetvdb_index << [it[0], n2]
+	} else {
+		thetvdb_index << [it[0], n1]
+	}
+}
+
+
+thetvdb_index = thetvdb_index.findResults{ [it[0] as Integer, it[1].replaceAll(/\s+/, ' ').trim()] }.findAll{ !(it[1] =~ /(?i:duplicate)/ || it[1] =~ /\d{6,}/ || it[1].startsWith('*') || it[1].endsWith('*') || it[1].length() <= 3) }
+thetvdb_index = thetvdb_index.sort(new Comparator() {
+	int compare(a, b) { a[0] <=> b[0] }
+})
+
+// join and sort
+def thetvdb_txt = thetvdb_index.findResults{ [it[0].pad(6), it[1].trim()].join('\t') }
+gz(thetvdb_out, thetvdb_txt)
+println "TheTVDB Index: " + thetvdb_txt.size()
+
+// sanity check
+if (thetvdb_txt.size() < 30000) { throw new Exception('TheTVDB index sanity failed') }
+
+
+
+// BUILD anidb-index.gz
+def anidb = new net.sourceforge.filebot.web.AnidbClient(null, 0).getAnimeTitles()
+
+def anidb_index = anidb.findResults{ [it.getAnimeId(), it.getPrimaryTitle(), it.getEnglishTitle()] }
+
+// join and sort
+def anidb_txt = anidb_index.findResults{ [it[0].pad(5), it[1] ?: '', it[2] ?: ''].join('\t').replaceAll(/['`´‘’ʻ]+/, /'/) }.sort().unique()
+gz(anidb_out, anidb_txt)
+println "AniDB Index: " + anidb_txt.size()
+
+// sanity check
+if (anidb_txt.size() < 5000) { throw new Exception('AniDB index sanity failed') }

reviews.csv

@@ -0,0 +1,15 @@
+Amit Singhal;2010-05-23;one of the best tv-ep renamers out there! kudos! to the author, rednoah
+lakis koulourakis;2011-02-18;best app on series renaming i could find
+mark;2012-03-25;"one of the most liked ""must have"" programs i use. much thanks to author!"
+Jackson;2012-03-26;Stable and works.
+Milos Kaurin;2012-04-14;One of the best (if not the best) multiplatform episode renamer/subtitle downloader
+Gillardino Boccacio;2012-05-12;indispensable tool for people with a large library of videos. Through this program, I was able to restore order in films and serials. Thank you!
+Ed Doran;2012-08-25;A must use tool. Really simplifies watching my favorite movies
+Kate Smith;2012-11-05;awesome user-interface!
+OpenID User;2012-11-05;Extremely quick and efficient. Great for renaming anime, too - I was very impressed at Filebot's ability to consistently rename anime using AniDB. A must for keeping a large collection clean!
+akya;2013-02-21;Extremely efficient.. especially for renaming single episodes.
+hfurius;2013-02-24;easy to use and easy to work. thanks a lot!
+Jan;2013-03-21;Kudos :) Great time saver keeping my collection organized. Stable, useful, easy to use. Donation has been made. Keep up the good work
+Bobby Scholtens;2013-05-27;"Pretty awesome piece of software. Very fast and easy to use. As it states: ""it just works""."
+williamdanis;2013-06-18;Filebot is excellent! Thanks.
+haywardb;2013-08-08;Wow, thank you so much for this program. I had a large collection of tv shows which I was not looking forward to sorting out. Dumped the files into FileBot and within minutes, all done. Even finds correct name within filenames with a load of junk in them.

website/index.html

@@ -197,10 +197,12 @@
 							dataType: 'json',
 							success: function(data) {
 								var review = data[Math.floor(Math.random() * data.length)]
-								$('#review .user').text(review.user)
+								if (review.user.length > 0 && review.date.length > 0 && review.text.length > 0) {
-								$('#review .date').text(review.date)
+									$('#review .user').text(review.user)
-								$('#review .text').text(review.text)
+									$('#review .date').text(review.date)
-								if (review.text.length > 0) { $('#review').show() }
+									$('#review .text').text(review.text)
+									$('#review').show()
+								}
 							}
 						});
 					</script>

website/reviews.json

@@ -1,18 +1,38 @@
 [
     {
-        "user": "Jan",
+        "user": "Amit Singhal",
-        "date": "2013-03-21",
+        "date": "2010-05-23",
-        "text": "Kudos :) Great time saver keeping my collection organized. Stable, useful, easy to use. Donation has been made. Keep up the good work"
+        "text": "one of the best tv-ep renamers out there! kudos! to the author, rednoah"
     },
     {
-        "user": "hfurius",
+        "user": "lakis koulourakis",
-        "date": "2013-02-24",
+        "date": "2011-02-18",
-        "text": "easy to use and easy to work. thanks a lot!"
+        "text": "best app on series renaming i could find"
     },
     {
-        "user": "akya",
+        "user": "mark",
-        "date": "2013-02-21",
+        "date": "2012-03-25",
-        "text": "Extremely efficient.. especially for renaming single episodes."
+        "text": "one of the most liked \"must have\" programs i use. much thanks to author!"
+    },
+    {
+        "user": "Jackson",
+        "date": "2012-03-26",
+        "text": "Stable and works."
+    },
+    {
+        "user": "Milos Kaurin",
+        "date": "2012-04-14",
+        "text": "One of the best (if not the best) multiplatform episode renamer/subtitle downloader"
+    },
+    {
+        "user": "Gillardino Boccacio",
+        "date": "2012-05-12",
+        "text": "indispensable tool for people with a large library of videos. Through this program, I was able to restore order in films and serials. Thank you!"
+    },
+    {
+        "user": "Ed Doran",
+        "date": "2012-08-25",
+        "text": "A must use tool. Really simplifies watching my favorite movies"
     },
     {
         "user": "Kate Smith",
@@ -25,39 +45,19 @@
         "text": "Extremely quick and efficient. Great for renaming anime, too - I was very impressed at Filebot's ability to consistently rename anime using AniDB. A must for keeping a large collection clean!"
     },
     {
-        "user": "Ed Doran",
+        "user": "akya",
-        "date": "2012-08-25",
+        "date": "2013-02-21",
-        "text": "A must use tool. Really simplifies watching my favorite movies"
+        "text": "Extremely efficient.. especially for renaming single episodes."
     },
     {
-        "user": "Gillardino Boccacio",
+        "user": "hfurius",
-        "date": "2012-05-12",
+        "date": "2013-02-24",
-        "text": "indispensable tool for people with a large library of videos. Through this program, I was able to restore order in films and serials. Thank you!"
+        "text": "easy to use and easy to work. thanks a lot!"
     },
     {
-        "user": "Jackson",
+        "user": "Jan",
-        "date": "2012-03-26",
+        "date": "2013-03-21",
-        "text": "Stable and works."
+        "text": "Kudos :) Great time saver keeping my collection organized. Stable, useful, easy to use. Donation has been made. Keep up the good work"
-    },
-    {
-        "user": "mark",
-        "date": "2012-03-25",
-        "text": "one of the most liked \"must have\" programs i use. much thanks to author.!"
-    },
-    {
-        "user": "lakis koulourakis",
-        "date": "2011-02-18",
-        "text": "best app on series renaming i could find"
-    },
-    {
-        "user": "Amit Singhal",
-        "date": "2010-05-23",
-        "text": "one of the best tv-ep renamers out there! kudos! to the author, rednoah"
-    },
-    {
-        "user": "Milos Kaurin",
-        "date": "2012-04-14",
-        "text": "One of the best (if not the best) multiplatform episode renamer/subtitle downloader"
     },
     {
         "user": "Bobby Scholtens",
@@ -65,8 +65,13 @@
         "text": "Pretty awesome piece of software. Very fast and easy to use. As it states: \"it just works\"."
     },
     {
-        "user": "pez",
+        "user": "williamdanis",
-        "date": "2013-06-14",
+        "date": "2013-06-18",
-        "text": "Awesome stuff! I used to spend tons of time manually renaming all my tv shows, but this does all my work near instantly. Absolutly worth the money."
+        "text": "Filebot is excellent! Thanks."
+    },
+    {
+        "user": "haywardb",
+        "date": "2013-08-08",
+        "text": "Wow, thank you so much for this program. I had a large collection of tv shows which I was not looking forward to sorting out. Dumped the files into FileBot and within minutes, all done. Even finds correct name within filenames with a load of junk in them."
     }
 ]