From c21e544e089a4eeb6a0cee84951117296c17554f Mon Sep 17 00:00:00 2001
From: Reinhard Pointner <rednoah@users.noreply.github.com>
Date: Mon, 11 May 2015 10:35:45 +0000
Subject: [PATCH] * added build-data scripts

---
 .../BuildData.groovy                          | 27 +++++++++-------
 build-data/build-data-import.sh               | 31 +++++++++++++++++++
 2 files changed, 46 insertions(+), 12 deletions(-)
 rename BuildData.groovy => build-data/BuildData.groovy (93%)
 mode change 100644 => 100755
 create mode 100755 build-data/build-data-import.sh

diff --git a/BuildData.groovy b/build-data/BuildData.groovy
old mode 100644
new mode 100755
similarity index 93%
rename from BuildData.groovy
rename to build-data/BuildData.groovy
index 8c758234..0012ee4b
--- a/BuildData.groovy
+++ b/build-data/BuildData.groovy
@@ -1,7 +1,11 @@
+#!/usr/bin/env filebot -script
+
 import org.tukaani.xz.*
 
 /* ------------------------------------------------------------------------- */
 
+def dir_website = "../website"
+def dir_data    = "${dir_website}/data"
 
 def sortRegexList(path) {
 	def set = new TreeSet(String.CASE_INSENSITIVE_ORDER)
@@ -13,13 +17,12 @@ def sortRegexList(path) {
 	println "${out}\n${out.text}\n"
 }
 
-
 // sort and check shared regex collections
-sortRegexList("website/data/release-groups.txt")
-sortRegexList("website/data/query-blacklist.txt")
-sortRegexList("website/data/exclude-blacklist.txt")
-sortRegexList("website/data/series-mappings.txt")
-sortRegexList("website/data/add-series-alias.txt")
+sortRegexList("${dir_data}/release-groups.txt")
+sortRegexList("${dir_data}/query-blacklist.txt")
+sortRegexList("${dir_data}/exclude-blacklist.txt")
+sortRegexList("${dir_data}/series-mappings.txt")
+sortRegexList("${dir_data}/add-series-alias.txt")
 
 
 /* ------------------------------------------------------------------------- */
@@ -34,17 +37,17 @@ reviews = reviews.sort{ it.date }
 
 def json = new groovy.json.JsonBuilder()
 json.call(reviews as List)
-json.toPrettyString().saveAs('website/reviews.json')
+json.toPrettyString().saveAs("${dir_website}/reviews.json")
 println "Reviews: " + reviews.size()
 
 
 /* ------------------------------------------------------------------------- */
 
 
-def moviedb_out = new File("website/data/moviedb.txt")
-def thetvdb_out = new File("website/data/thetvdb.txt")
-def anidb_out   = new File("website/data/anidb.txt")
-def osdb_out   = new File("website/data/osdb.txt")
+def moviedb_out = new File("${dir_data}/moviedb.txt")
+def thetvdb_out = new File("${dir_data}/thetvdb.txt")
+def anidb_out   = new File("${dir_data}/anidb.txt")
+def osdb_out    = new File("${dir_data}/osdb.txt")
 
 
 def pack(file, lines) {
@@ -304,7 +307,7 @@ tvdb.values().each{ r ->
 }
 
 // additional custom mappings
-new File('website/data/add-series-alias.txt').splitEachLine(/\t+/, 'UTF-8') { row ->
+new File("${dir_data}/add-series-alias.txt").splitEachLine(/\t+/, 'UTF-8') { row ->
 	def se = thetvdb_index.find{ row[0] == it[1] && !it.contains(row[1]) }
 	if (se == null) die("Unabled to find series '${row[0]}': '${row[1]}'")
 	thetvdb_index << [se[0], row[1]]
diff --git a/build-data/build-data-import.sh b/build-data/build-data-import.sh
new file mode 100755
index 00000000..8d64681d
--- /dev/null
+++ b/build-data/build-data-import.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+for DUMP in "omdb.zip" "tvdb.zip" "anidb.gz" "osdb.gz"
+do
+	if [ ! -f "$DUMP" ]; then
+		touch -a -m -t 197001010000.00 "$DUMP"
+	fi
+done
+
+echo 'Fetch OMDB dump'
+if test "`find omdb.zip -mtime +20`"; then
+    curl -L -o omdb.zip -z omdb.zip "http://beforethecode.com/projects/omdb/download.aspx?e=reinhard.pointner%40gmail.com&tsv=movies"
+    unzip -o omdb.zip
+fi
+
+echo 'Fetch TVDB dump'
+if test "`find tvdb.zip -mtime +5`"; then
+    curl -L -o tvdb.zip -z tvdb.zip "http://thetvdb.com/api/58B4AA94C59AD656/updates/updates_all.zip"
+    unzip -o tvdb.zip
+fi
+
+echo 'Fetch AniDB dump'
+if test "`find anidb.gz -mtime +5`"; then
+    curl -L -o anidb.gz -z anidb.gz "http://anidb.net/api/anime-titles.dat.gz"
+fi
+
+echo 'Fetch OSDB dump'
+if test "`find osdb.gz -mtime +20`"; then
+    curl -L -o osdb.gz -z osdb.gz "http://www.opensubtitles.org/addons/export_movie.php" -sH 'Accept-encoding: gzip'
+    gunzip -c osdb.gz > osdb.txt
+fi