Move data file maintenance to GitHub

2016-02-24 19:27:31 +00:00 · 2016-02-24 19:27:31 +00:00 · ebcd2f9677
parent 610ae228d0
commit ebcd2f9677
6 changed files with 17 additions and 3441 deletions
--- a/build-data/BuildData.groovy
+++ b/build-data/BuildData.groovy
@ -1,29 +1,34 @@
 #!/usr/bin/env filebot -script

+
 import org.tukaani.xz.*

+
 /* ------------------------------------------------------------------------- */

+
 def dir_root    = ".."
 def dir_website = "${dir_root}/website"
 def dir_data    = "${dir_website}/data"

-def sortRegexList(path) {
+// sort and check shared regex collections
+['add-series-alias.txt', 
+ 'exclude-blacklist.txt', 
+ 'query-blacklist.txt', 
+ 'release-groups.txt', 
+ 'series-mappings.txt'
+].each{
+	def input = new URL("https://raw.githubusercontent.com/filebot/data/master/${it}")
+	def output = new File("${dir_data}/${it}")
+
 	def set = new TreeSet(String.CASE_INSENSITIVE_ORDER)
-	new File(path).eachLine('UTF-8'){
-		// check if regex compiles
+	input.getText('UTF-8').split(/\R/).each{
 		set += java.util.regex.Pattern.compile(it.trim()).pattern()
 	}
-	def out = set.join('\n').saveAs(path)
-	println "${out}\n${out.text}\n"
-}

-// sort and check shared regex collections
-sortRegexList("${dir_data}/release-groups.txt")
-sortRegexList("${dir_data}/query-blacklist.txt")
-sortRegexList("${dir_data}/exclude-blacklist.txt")
-sortRegexList("${dir_data}/series-mappings.txt")
-sortRegexList("${dir_data}/add-series-alias.txt")
+	set.join('\n').saveAs(output)
+	println "${output}\n${output.text}\n"
+}


 /* ------------------------------------------------------------------------- */
--- a/website/data/add-series-alias.txt
+++ b/website/data/add-series-alias.txt
@ -1,31 +0,0 @@
-@midnight	At Midnight
-Battlestar Galactica (2003)	BSG
-Bordertown (2016)	Bordertown (US)
-Cosmos: A Spacetime Odyssey	Cosmos A Space Time Odyssey
-CSI: Crime Scene Investigation	CSI
-Forever (2014)	Forever (US)
-House of Cards	HOC
-How I Met Your Mother	HIMYM
-Law & Order: Special Victims Unit	Law & Order SVU
-Law & Order: Special Victims Unit	Law and Order SVU
-M*A*S*H	M.A.S.H.
-M*A*S*H	MASH
-Naruto Shippuden	Naruto Shippuuden
-NCIS: Los Angeles	NCIS LA
-NCIS: Los Angeles	NCIS LosAngeles
-NCIS: New Orleans	Navy CIS New Orleans
-NCIS: New Orleans	NCIS NO
-Outsiders	Outsiders (2016)
-Proof (2015)	Proof (US)
-Resurrection	Resurrection (US)
-Revolution	Revolution (2012)
-Scream	Scream The TV Series
-Sex&Drugs&Rock&Roll	Sex and Drugs and Rock and Roll
-Stargate SG-1	Stargate
-Supergirl	Super Girl
-The Big Bang Theory	TBBT
-The Bridge (2013)	The Bridge (US)
-The Code (2014)	The Code (AU)
-The Late Late Show with Craig Ferguson	Craig Ferguson
-The Walking Dead	TWD
-World Series of Poker	WSOP
--- a/website/data/exclude-blacklist.txt
+++ b/website/data/exclude-blacklist.txt
@ -1,13 +0,0 @@
-!(sample|trailer)
-(?<=\b|_)(NCED|NCOP|(OP|ED)\p{Digit}\p{Alpha})(?=\b|_)
-(sample|trailer)-
-(sample|trailer)[.]
-(sample|trailer)$
-s[.]
-sample
-\((sample|trailer)\)
-\[(sample|trailer)\]
-^(sample|trailer)
-^Deleted.and.Extended.Scenes$
-^Deleted.Scenes$
-^Extras$
--- a/website/data/query-blacklist.txt
+++ b/website/data/query-blacklist.txt
@ -1,482 +0,0 @@
-(?-i:WEB|SPECIAL|MULTi|XXX)
-(?<=^)\w{1,2}(?=$)
-(pro7|rtl|rtl2|srtl|rtlnitro|vox|kabel1|sat1|viva|orf1|orf2|atv|arte|zdf|zdfneo|ard|ardeinsfestival|stv|tele5|sf1|sf2|hr3|bay3|puls4|mdr|wdr|rbb|sixx|sw3)\D\d*
-(Season|Series)\D?\d+
-(The|A|MPG)$
-.+(movies|tv.shows)$
-1-3-3-8.com
-10.bit
-18p
-2DDL
-3D
-7p
-@KIDZ
-[1-3](CD|CDRip)
-[1-9].?of.?[1-9]
-\d{2,4}.\b\d{2}.\b\d{2}.\b\d{2}.\b\d{2}
-\d{3,4}[pi]
-\d{8}.\d{4}
-^(downloads|amc).*$
-^(tvp|tvs)
-^.*(downloads)$
-^[_]+$
-^[lp]
-^\d{1,2}[.]
-^\d{8,64}$
-^__.+$
-^Action$
-^Adventure$
-^Adventures$
-^and$
-^Animation$
-^Animations$
-^Anime$
-^Animes$
-^Applications$
-^bad$
-^Best$
-^Big$
-^British.TV$
-^BTN$
-^Cartoon$
-^Cinema$
-^Classic$
-^clean$
-^cleaned$
-^Comedy$
-^Complete$
-^Complete.Torrents$
-^Completed$
-^Completos$
-^Cover$
-^Crime$
-^Das$
-^data$
-^deluge$
-^Den$
-^Der$
-^Descargado$
-^Descargando$
-^Descargas$
-^Desktop$
-^dev$
-^Die$
-^Disco\d+$
-^disk\d+$
-^Documentaries$
-^Documentary$
-^Documents$
-^Done$
-^Download$
-^Downloading$
-^Downloads$
-^Drama$
-^Dropbox$
-^DVD$
-^emule$
-^Entertainment$
-^Episode$
-^Episodes$
-^Erotic$
-^Erotica$
-^Exotic$
-^extract$
-^extracted$
-^Family$
-^Fantasy$
-^Fertig$
-^file$
-^files$
-^Film$
-^Filme$
-^Filme.Neu$
-^Films$
-^Finished$
-^folder$
-^Game.Show$
-^Games$
-^good$
-^HD[0-9]+$
-^HD[A-Z].DATA$
-^hdd[0-9]+$
-^History$
-^home$
-^Horror$
-^I+$
-^ignore$
-^ignore.folder$
-^In.Progress$
-^inbox$
-^Incoming$
-^Incomplete$
-^jukebox$
-^Kids$
-^Kids.Movies$
-^Kids.Shows$
-^Library$
-^Love$
-^master$
-^MD[0-9].DATA$
-^media$
-^mnt$
-^More$
-^MotoGP$
-^Motor$
-^Motorsport$
-^Motorsports$
-^moved$
-^Movie$
-^movie[0-9]+$
-^Movies$
-^Multimedia$
-^Music$
-^Musique$
-^mynas$
-^Mystery$
-^NAS$
-^net$
-^new$
-^New.Movies$
-^New.Shows$
-^New.TV.Shows$
-^News$
-^of$
-^old$
-^Opnamen$
-^Organize$
-^other$
-^Part$
-^PELICULAS$
-^Películas$
-^Pictures$
-^Piece$
-^plex.film$
-^Plexserv$
-^private$
-^Processed$
-^Processing$
-^public$
-^pyload$
-^Qmultimedia$
-^queue$
-^Radio$
-^raid$
-^Reality$
-^Reality.Shows$
-^Recordings$
-^rename$
-^Review$
-^rip$
-^Romance$
-^rtorrent$
-^Saga$
-^Satire$
-^school$
-^Sci.Fi$
-^Science.Fiction$
-^scratch$
-^scratch.area$
-^Season$
-^Secret$
-^seed$
-^Seeding$
-^Seeds$
-^Serier$
-^Series$
-^SERIJE$
-^sex$
-^share$
-^shared$
-^Shorts$
-^show$
-^Soaps$
-^sort$
-^sorted$
-^SortedMedia$
-^Specials$
-^Sports$
-^staging$
-^Storage$
-^Stuff$
-^sync$
-^System$
-^tank$
-^tao$
-^TBR$
-^Telechargements$
-^temp$
-^temporary$
-^test$
-^testdata$
-^The.Best$
-^The.TV.Series$
-^Thriller$
-^tmp$
-^to$
-^ToProcess$
-^Torrent$
-^Torrents$
-^Tracker$
-^transfer$
-^transmission$
-^TV$
-^TV.Series$
-^TV.Show$
-^TV.Shows$
-^unsorted$
-^user$
-^utorrent$
-^vari$
-^VCD$
-^Video$
-^VIDEO_TS$
-^Videos$
-^Volume\D?\d+$
-^Volumes$
-^War$
-^watch$
-^WD$
-^Working$
-^www$
-^XXX+$
-A.PROCESAR
-A.Release.Lounge
-AAC\d*
-ABC
-Abenteuerfilm
-AC3D
-Actiondrama
-Actionfilm
-Actionkomödie
-Actionthriller
-Addic7ed.com
-aGGr0
-Anaglyph
-Animations
-Animationsfilm
-Anime
-Animes
-Arte
-AUDIO_TS
-Axx
-BDMV
-Best.of.\d{4}
-Best.Quality
-bt.downloads
-btarena.org
-By.Cool.Release
-C.orig
-Castellano
-CBC
-CD\D?\d+
-Channel\D?\d+
-CLASSIC
-CN
-CODIFICAR
-CODY
-ConvertedTV
-Creditless
-Custom.DKsubs
-CVCD
-DC
-DD\d[.,]\d
-DDL
-ddlsource
-ddlsource.com
-DDLValley
-DDLValley.(eu|net)
-ded
-Demonoid
-DKsubs
-DL
-docu
-Dokumentarfilm
-DownRev
-dreambox.cut
-DSKI
-Dual.Audio
-dubbed
-DVD\d+
-DVDsize
-DVDXvID
-DVSKY
-ECE
-ENG
-English
-ENSub
-Ext
-EXTENDED
-ExtraScene
-ExtraTorrent
-Fantasy
-Fantasyfilm
-Fantasykomödie
-FileBot
-Final.Cut
-FIXED
-Fra
-FRE
-French
-funxd
-GER
-German
-Hard.Subbed
-HAVE.TO.BE.CUT
-HBO
-HBO.Documentaries
-HD
-HDR
-HDRip
-HebSubs
-Hi10P
-HiDef
-Hindi
-History.Channel
-HOME.SERVER
-Horrorfilm
-Horrorkomödie
-Horrorthriller
-HQ
-HR
-HSBS
-HVDVD_TS
-iFH
-info
-Ingles.Subs
-iNT
-iNTERNAL
-iPod
-IPT
-ISO
-iTA
-iTALIA
-Italian
-jigaxx
-Katastrophenfilm
-KIDZCORNER
-Kino.und.TV
-kodi
-Komödie
-KOR
-KORSUB
-Kriegsfilm
-Krimikomödie
-Kriminalfilm
-LAB
-Liebeskomödie
-LIMITED
-LMAO
-lol
-Los.Sustitutos
-LowDef
-M.HD
-MegaPack
-MkvCage
-mkvonly
-MoTv
-Movie.Pack
-MovieObject
-MultiSub
-MVGroup.org
-Mysterythriller
-National.Geographic
-new.unsorted
-NewMedia
-NeZu
-NFO
-NG
-NHK
-NL
-NL.Subs
-NLT
-NTSC
-o2.pl
-onlinetvrecorder
-P2P
-PAL
-PBS
-PHoQUE
-plex
-Politdrama
-Pre.?DVD
-PROPER
-ProSieben.MAXX
-PS3
-PSP
-QEBS\d*
-READNFO
-REAL.PROPER
-RecordedTV
-REMASTER
-REMASTERED
-RENOMBRAR
-REPACK
-ReRip
-RESYNC
-RETAIL
-RiffTrax
-RiPSalot
-Romantikkomödie
-sample
-samples
-Screenshot
-sd
-sed
-ShareGo
-ShareReactor
-ShareZONE
-ShortKut
-Snapshots
-sortmovie
-SPA
-Spanish
-Stephen.Kings
-STRATOS
-STV
-Sub
-SUBBED
-Subs
-Subtit
-Subtitle
-swe.?sub
-Swedish
-SYNC
-SYNCED
-SYNCFIX
-t\d{2,4}
-TC
-Thrillerkomödie
-to.be.sorted
-ToSort
-TPB
-Tragikomödie
-TransmissionDL
-TRUEDEF
-TRUEFRENCH
-TS
-TSXVID
-tt\d{7}
-TV.Aufnahmen
-TV.Opnamen
-TV.Thriller
-TVOON.DE
-tvrecorder
-TwoDDL
-TwoDDL.eu
-UE
-UsaBit.com
-utorrent.downloads
-VCD
-VIDEO_TS
-vo
-Volume.TB
-Western
-WS
-www.[\p{Alnum}\p{Punct}]+?.(com|net|tk|ro|cd|me|de|pe)
-x0r
-xbmc
-XORBiTANT
-xRipp
-XtremeDoN
-YIFY
-Zune
--- a/website/data/release-groups.txt
+++ b/website/data/release-groups.txt
--- a/website/data/series-mappings.txt
+++ b/website/data/series-mappings.txt
@ -1,27 +0,0 @@
-11.22.63	11/22/63
-^episodes.(s\d{2}e\d{2}|\dx\d{2})	Episodes
-at.midnight	@midnight
-BSG	Battlestar Galactica
-Craig.Ferguson	The Late Late Show with Craig Ferguson
-CSI(?!.{0,3}(NY|LA|Miami|New.Orleans|Academy|Cyber))	CSI: Crime Scene Investigation
-epl	English Premier League
-greysana	Grey's Anatomy
-HIMYM	How I Met Your Mother
-Hml8p	Homeland
-hoc	House of Cards
-law.and.order.svu	Law & Order: Special Victims Unit
-MBFAGW	My Big Fat American Gypsy Wedding
-Navy.CIS.New.Orleans	NCIS: New Orleans
-NCIS.LA	NCIS: Los Angeles
-NCIS.NO	NCIS: New Orleans
-poid	Person of Interest
-SOS	Shahs of Sunset
-SuiG7p	Suits
-TBBT	The Big Bang Theory
-tv..got	Game of Thrones
-tv..poi	Person of Interest
-tv..shield	Marvel's Agents of S.H.I.E.L.D.
-tv..soa	Sons of Anarchy
-TWD	The Walking Dead
-UndG7p	Underemployed
-wsop	World Series of Poker