From 963eacfe0537a7abddf30441c754c67ca1e40965 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sun, 11 Oct 2015 09:28:11 -0500 Subject: [PATCH 01/76] commenting out mahakala list Due to so many users having issues with this list blocking legitimate domains such as microsoft.com, apple.com, xkcd.com and more, I am turning it off by default. While this drastically reduces the amount of domains blocked, ad-blocking performance still seems to function quite well. Long-time users can simply uncomment the list, but this will make it so new users have a more pleasant experience. If you already have the mahakala list, you will need to remove it first with this command: sudo rm /etc/pihole/list.2.adblock.mahakala.is.domains There are a few other lists I am looking at including, but this will provide an immediate fix. --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 1f5e3fb..12c8b0f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -8,7 +8,7 @@ piholeIP=$(hostname -I) # Ad-list sources--one per line in single quotes sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' -'http://adblock.mahakala.is/' +#'http://adblock.mahakala.is/' 'http://hosts-file.net/.%5Cad_servers.txt' 'http://www.malwaredomainlist.com/hostslist/hosts.txt' 'http://pgl.yoyo.org/adservers/serverlist.php?' From bf397db73e254d72b45cd6abf2bccbb42701005f Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Thu, 15 Oct 2015 18:58:12 -0500 Subject: [PATCH 02/76] adding crontab file I plan to use this to be deployed with the automatic installer. Just pushing the file for now so I can test download it. --- advanced/pihole.cron | 1 + 1 file changed, 1 insertion(+) create mode 100644 advanced/pihole.cron diff --git a/advanced/pihole.cron b/advanced/pihole.cron new file mode 100644 index 0000000..a707607 --- /dev/null +++ b/advanced/pihole.cron @@ -0,0 +1 @@ +@weekly sudo /usr/local/bin/gravity.sh From d5ffec64f25847197c082e37f18e78a194923523 Mon Sep 17 00:00:00 2001 From: Sven Giermann Date: Thu, 22 Oct 2015 15:17:44 +0200 Subject: [PATCH 03/76] add whitespace after $piholeIP In my case there had been no whitespace in piholeIP (I configured it statically because 'hostname -I' returns multiple IP addresses, which ends in errornous hosts file), so it's more safe to add one between IP and hostname. Further, I removed the obsolete latentBlacklist variable. --- gravity.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 12c8b0f..d8c8180 100755 --- a/gravity.sh +++ b/gravity.sh @@ -27,7 +27,6 @@ eventHorizon=pihole.3.eventHorizon.txt accretionDisc=pihole.4.accretionDisc.txt eyeOfTheNeedle=pihole.5.wormhole.txt blacklist=$piholeDir/blacklist.txt -latentBlacklist=$origin/latentBlacklist.txt whitelist=$piholeDir/whitelist.txt latentWhitelist=$origin/latentWhitelist.txt @@ -136,7 +135,7 @@ function gravity_advanced() echo "** $numberOf unique domains trapped in the event horizon." # Format domain list as "192.168.x.x domain.com" echo "** Formatting domains into a HOSTS file..." - cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"'" $0}' > $origin/$accretionDisc + cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $origin/$accretionDisc # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq) From ced96905e9883561d9ad50b0ed665a6a327fb922 Mon Sep 17 00:00:00 2001 From: Emmanuel CHARETTE Date: Sat, 24 Oct 2015 21:55:31 +0200 Subject: [PATCH 04/76] Update basic-install.sh change `shutdown -r now` to reboot for ssh users --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 80bb754..9b5ed24 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -79,4 +79,4 @@ echo "Entering the event horizon..." sudo /usr/local/bin/gravity.sh echo "Restarting..." -sudo shutdown -r now +sudo reboot From deb48c124fca132cbd81bd3b17eacd3f51c4ec44 Mon Sep 17 00:00:00 2001 From: Nathan Jones Date: Sun, 25 Oct 2015 16:53:20 -0700 Subject: [PATCH 05/76] Improve whitelisting script Match domain exactly instead of partially. Gather list of domains to minimize number of times that hosts file must be enumerated. Only add domain to whitelist if it isn't already present. --- advanced/Scripts/whitelist.sh | 47 +++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/whitelist.sh b/advanced/Scripts/whitelist.sh index aed6bd0..d610d0a 100755 --- a/advanced/Scripts/whitelist.sh +++ b/advanced/Scripts/whitelist.sh @@ -1,13 +1,44 @@ #!/bin/bash + +if [ $# = 0 ]; then + echo "Immediately whitelists one or more domains." + echo "Usage: whitelist.sh domain1 [domain2 ...]" +fi + +combopattern="" + # For each argument passed to this script for var in "$@" do - echo "Whitelisting $var..." - # Use sed to search for the domain in /etc/pihole/gravity.list and remove it using an in-place edit - sed -i "/$var/d" /etc/pihole/gravity.list - # Also add the domain to the whitelist.txt in /etc/pihole - echo "$var" >> /etc/pihole/whitelist.txt + echo "Whitelisting $var..." + + # Construct basic pattern to match domain name. + basicpattern=$(echo $var | awk -F '[# \t]' 'NF>0&&$1!="" {print ""$1""}' | sed 's/\./\\./g') + + if [ "$basicpattern" != "" ]; then + # Add to the combination pattern that will be used below + if [ "$combopattern" != "" ]; then combopattern="$combopattern|"; fi + combopattern="$combopattern$basicpattern" + + # Also add the domain to the whitelist but only if it's not already present + grep -E -q "^$basicpattern$" /etc/pihole/whitelist.txt \ + || echo "$var" >> /etc/pihole/whitelist.txt + fi done -echo "** $# domain(s) whitelisted." -# Force dnsmasq to reload /etc/pihole/gravity.list -kill -HUP $(pidof dnsmasq) \ No newline at end of file + +# Now report on and remove matched domains +if [ "$combopattern" != "" ]; then + echo "Modifying hosts file..." + + # Construct pattern to match entry in hosts file. + # This consists of one or more IP addresses followed by the domain name. + pattern=$(echo $combopattern | awk -F '[# \t]' '{printf "%s", "^(([0-9]+\.){3}[0-9]+ +)+("$1")$"}') + + # Output what will be removed and then actually remove + sed -r -n 's/'"$pattern"'/ Removed: \3/p' /etc/pihole/gravity.list + sed -r -i '/'"$pattern"'/d' /etc/pihole/gravity.list + + echo "** $# domain(s) whitelisted." + # Force dnsmasq to reload /etc/pihole/gravity.list + kill -HUP $(pidof dnsmasq) +fi From 6d0c67af9abc6732702a86b677bb10c8f55d0fa0 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 27 Oct 2015 16:16:22 -0500 Subject: [PATCH 06/76] Install and enable PHP Needed to view the admin web interface --- automated install/basic-install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9b5ed24..358cf8e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -51,7 +51,7 @@ sudo apt-get -y install dnsmasq sudo update-rc.d dnsmasq enable echo "Installing a Web server" -sudo apt-get -y install lighttpd +sudo apt-get -y install lighttpd php5-common php5-cgi php5 sudo chown www-data:www-data /var/www sudo chmod 775 /var/www sudo usermod -a -G www-data pi @@ -66,6 +66,7 @@ sudo mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig sudo mv /var/www/index.lighttpd.html /var/www/index.lighttpd.orig sudo curl -o /etc/dnsmasq.conf "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dnsmasq.conf" sudo curl -o /etc/lighttpd/lighttpd.conf "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/lighttpd.conf" +sudo lighty-enable-mod fastcgi fastcgi-php sudo mkdir /var/www/pihole sudo curl -o /var/www/pihole/index.html "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html" From bc3ece778d6d834ba1b2829d8b672b0d710ca2ac Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 27 Oct 2015 16:19:01 -0500 Subject: [PATCH 07/76] /var/www/html instead of /var/www lighttpd in Jessie uses /var/www/html for the root folder, so in order to only edit one lightpd.conf file, the installer will create the correct directory even if not being run on Jessie. --- automated install/basic-install.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 358cf8e..0249be7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -52,8 +52,9 @@ sudo update-rc.d dnsmasq enable echo "Installing a Web server" sudo apt-get -y install lighttpd php5-common php5-cgi php5 -sudo chown www-data:www-data /var/www -sudo chmod 775 /var/www +sudo mkdir /var/www/html +sudo chown www-data:www-data /var/www/html +sudo chmod 775 /var/www/html sudo usermod -a -G www-data pi echo "Stopping services to modify them..." @@ -67,8 +68,8 @@ sudo mv /var/www/index.lighttpd.html /var/www/index.lighttpd.orig sudo curl -o /etc/dnsmasq.conf "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dnsmasq.conf" sudo curl -o /etc/lighttpd/lighttpd.conf "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/lighttpd.conf" sudo lighty-enable-mod fastcgi fastcgi-php -sudo mkdir /var/www/pihole -sudo curl -o /var/www/pihole/index.html "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html" +sudo mkdir /var/www/html/pihole +sudo curl -o /var/www/html/pihole/index.html "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html" echo "Locating the Pi-hole..." sudo curl -o /usr/local/bin/gravity.sh "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh" From cfc469723634293d9b7154dd5005cca23a6c3bc4 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 27 Oct 2015 16:21:20 -0500 Subject: [PATCH 08/76] Download and install the Web interface dnsmasq also needs to be able to read the log file to properly display the statistics --- automated install/basic-install.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 0249be7..9f4892a 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -71,6 +71,15 @@ sudo lighty-enable-mod fastcgi fastcgi-php sudo mkdir /var/www/html/pihole sudo curl -o /var/www/html/pihole/index.html "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html" +echo "Installing the Web interface..." +sudo wget https://github.com/jacobsalmela/AdminLTE/archive/master.zip -O /var/www/master.zip +sudo unzip /var/www/master.zip -d /var/www/html/ +sudo mv /var/www/AdminLTE-master /var/www/html/admin +sudo rm /var/www/master.zip 2>/dev/null +sudo touch /var/log/pihole.log +sudo chmod 644 /var/log/pihole.log +sudo chown dnsmasq:root /var/log/pihole.log + echo "Locating the Pi-hole..." sudo curl -o /usr/local/bin/gravity.sh "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh" sudo curl -o /usr/local/bin/chronometer.sh "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh" From 7a0b92d2c1c38043274d0b945e22df82626e5765 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 27 Oct 2015 17:03:48 -0500 Subject: [PATCH 09/76] missed a few file paths --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9f4892a..7c48fdb 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -64,7 +64,7 @@ sudo service lighttpd stop echo "Backing up original config files and downloading Pi-hole ones..." sudo mv /etc/dnsmasq.conf /etc/dnsmasq.conf.orig sudo mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig -sudo mv /var/www/index.lighttpd.html /var/www/index.lighttpd.orig +sudo mv /var/www/html/index.lighttpd.html /var/www/html/index.lighttpd.orig sudo curl -o /etc/dnsmasq.conf "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dnsmasq.conf" sudo curl -o /etc/lighttpd/lighttpd.conf "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/lighttpd.conf" sudo lighty-enable-mod fastcgi fastcgi-php @@ -74,7 +74,7 @@ sudo curl -o /var/www/html/pihole/index.html "https://raw.githubusercontent.com/ echo "Installing the Web interface..." sudo wget https://github.com/jacobsalmela/AdminLTE/archive/master.zip -O /var/www/master.zip sudo unzip /var/www/master.zip -d /var/www/html/ -sudo mv /var/www/AdminLTE-master /var/www/html/admin +sudo mv /var/www/html/AdminLTE-master /var/www/html/admin sudo rm /var/www/master.zip 2>/dev/null sudo touch /var/log/pihole.log sudo chmod 644 /var/log/pihole.log From 58e4fbd0fba3c032bc2fa40b2f7397fc1df82b0f Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 27 Oct 2015 17:59:01 -0500 Subject: [PATCH 10/76] New, easier-to-remember installation command Also a picture of the Web interface and how to access it since it is now included in the automated installer. --- README.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index be0d436..2c8a756 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,15 @@ +# Automated Install + +1. Install Raspbian +2. Set a **static** IP address +3. Run the command below + +```curl -L install.pi-hole.net | bash``` + +Once installed, **configure any device to use the Raspberry Pi as its DNS server and the ads will be blocked**. You can also configure your router's DHCP options to assign the Pi as clients DNS server so they do not need to do it manually. + +[![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif "AdminLTE Presentation")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY "Donate") + # Raspberry Pi Ad Blocker **A black hole for ads, hence Pi-hole** @@ -8,20 +20,8 @@ The Pi-hole is a DNS/Web server that will **block ads for any device on your net ## Coverage Featured on [MakeUseOf](http://www.makeuseof.com/tag/adblock-everywhere-raspberry-pi-hole-way/) and [Lifehacker](http://lifehacker.com/turn-a-raspberry-pi-into-an-ad-blocker-with-a-single-co-1686093533)! -## Automated Install - -1. Install Raspbian -2. Set a **static** IP address -3. Run the command below - -```curl -s "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/automated%20install/basic-install.sh" | bash``` - -Once installed, **configure any device to use the Raspberry Pi as its DNS server and the ads will be blocked**. You can also configure your router's DHCP options to assign the Pi as clients DNS server so they do not need to do it manually. - A more detailed explanation of the installation can be found [here](http://jacobsalmela.com/block-millions-ads-network-wide-with-a-raspberry-pi-hole-2-0). -[![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif "AdminLTE Presentation")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY "Donate") - ## Gravity The [gravity.sh](https://github.com/jacobsalmela/pi-hole/blob/master/gravity.sh) does most of the magic. The script pulls in ad domains from many sources and compiles them into a single list of [over 1.6 million entries](http://jacobsalmela.com/block-millions-ads-network-wide-with-a-raspberry-pi-hole-2-0). @@ -29,7 +29,11 @@ The [gravity.sh](https://github.com/jacobsalmela/pi-hole/blob/master/gravity.sh) You can add a `whitelist.txt` or `blacklist.txt` in `/etc/pihole/` and the script will apply those files automatically. ## Web Interface -I am also working on a [Web interface](https://github.com/jacobsalmela/AdminLTE#pi-hole-admin-dashboard) so you can view stats and change settings. +The [Web interface](https://github.com/jacobsalmela/AdminLTE#pi-hole-admin-dashboard) will be installed automatically so you can view stats and change settings. You can find it at: + +`http://192.168.1.x/admin/index.php` + +![Web](http://i.imgur.com/m114SCn.png) ## Custom Config File If you want to use your own variables for the gravity script (i.e. storing the files in a different location) and don't want to have to change them every time there is an update to the script, create a file called `/etc/pihole/pihole.conf`. In it, you should add your own variables in a similar fashion as shown below: From d9aa19d2cdfd9254904f1017ae8b954431cfd159 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 27 Oct 2015 18:22:54 -0500 Subject: [PATCH 11/76] Link to Security Now! mention --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 2c8a756..ca6b528 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,17 @@ Once installed, **configure any device to use the Raspberry Pi as its DNS server The Pi-hole is a DNS/Web server that will **block ads for any device on your network**. ## Coverage + +### Security Now! Podcast +Pi-hole is mentioned at 100 minutes and 26 seconds (the link brings you right there) +[![Pi-hole on Security Now!](http://img.youtube.com/vi/p7-osq_y8i8/0.jpg)](http://www.youtube.com/watch?v=p7-osq_y8i8&t=100m26s) + +### Tech Blogs + Featured on [MakeUseOf](http://www.makeuseof.com/tag/adblock-everywhere-raspberry-pi-hole-way/) and [Lifehacker](http://lifehacker.com/turn-a-raspberry-pi-into-an-ad-blocker-with-a-single-co-1686093533)! +## Technical Details + A more detailed explanation of the installation can be found [here](http://jacobsalmela.com/block-millions-ads-network-wide-with-a-raspberry-pi-hole-2-0). ## Gravity From 3290dbbe486460dd582fc4f5d7695008fdbd9719 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 31 Oct 2015 17:09:21 -0500 Subject: [PATCH 12/76] link to Sky-Hole fork Pi-hole in the cloud with Sky-Hole. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ca6b528..09b8852 100644 --- a/README.md +++ b/README.md @@ -60,4 +60,7 @@ A technical and detailed description can be found [here](http://jacobsalmela.com ## Other Operating Systems This script will work for other UNIX-like systems with some slight **modifications**. As long as you can install `dnsmasq` and a Webserver, it should work OK. The automated install only works for a clean install of Raspiban right now since that is how the project originated. +### Examples Of The Pi-hole On Other Operating Systems +- [Sky-Hole](http://dlaa.me/blog/post/skyhole) + [![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif "AdminLTE Presentation")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY "Donate") From 1b51b9ade358bcf6995a83d7a40e5ddb172d36a7 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Thu, 5 Nov 2015 14:31:38 -0600 Subject: [PATCH 13/76] for use as a cron job The log will be flushed daily and stats will be stored in a DB for displaying on the Web interface. --- advanced/Scripts/piholeLogFlush.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 advanced/Scripts/piholeLogFlush.sh diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh new file mode 100755 index 0000000..9647fe2 --- /dev/null +++ b/advanced/Scripts/piholeLogFlush.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Flushes /var/log/pihole.log +truncate -s 0 /var/log/pihole.log From 23eece944308e75ca340b709d259cb15aabc28ad Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 5 Nov 2015 18:11:34 -0800 Subject: [PATCH 14/76] Upstream Branch pseudo-rebase --- gravity.sh | 100 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/gravity.sh b/gravity.sh index f5bf6e8..8e8da8e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,6 +1,6 @@ #!/bin/bash # http://pi-hole.net -# Compiles a list of ad-serving domains by downloading them from multiple sources +# Compiles a list of ad-serving domains by downloading them from multiple sources # This script should only be run after you have a static IP address set on the Pi piholeIP=$(hostname -I) @@ -8,7 +8,7 @@ piholeIP=$(hostname -I) # Ad-list sources--one per line in single quotes sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' -#'http://adblock.mahakala.is/' +'http://adblock.mahakala.is/' 'http://hosts-file.net/.%5Cad_servers.txt' 'http://www.malwaredomainlist.com/hostslist/hosts.txt' 'http://pgl.yoyo.org/adservers/serverlist.php?' @@ -19,6 +19,9 @@ sources=('https://adaway.org/hosts.txt' adList=/etc/pihole/gravity.list origin=/etc/pihole piholeDir=/etc/pihole +if [[ -f $piholeDir/pihole.conf ]];then + . $piholeDir/pihole.conf +fi justDomainsExtension=domains matter=pihole.0.matter.txt andLight=pihole.1.andLight.txt @@ -27,16 +30,10 @@ eventHorizon=pihole.3.eventHorizon.txt accretionDisc=pihole.4.accretionDisc.txt eyeOfTheNeedle=pihole.5.wormhole.txt blacklist=$piholeDir/blacklist.txt +latentBlacklist=$origin/latentBlacklist.txt whitelist=$piholeDir/whitelist.txt latentWhitelist=$origin/latentWhitelist.txt -# After setting defaults, check if there's local overrides -if [[ -r $piholeDir/pihole.conf ]];then - echo "** Local calibration requested..." - . $piholeDir/pihole.conf -fi - - echo "** Neutrino emissions detected..." # Create the pihole resource directory if it doesn't exist. Future files will be stored here @@ -57,12 +54,8 @@ function createSwapFile() sudo dphys-swapfile setup sudo dphys-swapfile swapon } - - -if [[ -n "$noSwap" ]]; then - # if $noSwap is set, don't do anything - : -elif [[ -f /etc/dphys-swapfile ]];then + +if [[ -f /etc/dphys-swapfile ]];then swapSize=$(cat /etc/dphys-swapfile | grep -m1 CONF_SWAPSIZE | cut -d'=' -f2) if [[ $swapSize != 500 ]];then mv /etc/dphys-swapfile /etc/dphys-swapfile.orig @@ -82,31 +75,55 @@ do url=${sources[$i]} # Get just the domain from the URL domain=$(echo "$url" | cut -d'/' -f3) - + # Save the file as list.#.domain saveLocation=$origin/list.$i.$domain.$justDomainsExtension - echo -n "Getting $domain list... " - # Use a case statement to download lists that need special cURL commands to complete properly + agent="Mozilla/10.0" + + echo -n "Getting $domain list... " + + # Use a case statement to download lists that need special cURL commands + # to complete properly and reset the user agent when required case "$domain" in - "adblock.mahakala.is") data=$(curl -s -A 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' -e http://forum.xda-developers.com/ -z $saveLocation $url);; + "adblock.mahakala.is") + agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' + cmd="curl -e http://forum.xda-developers.com/" + ;; + + "pgl.yoyo.org") + cmd="curl -d mimetype=plaintext -d hostformat=hosts" + ;; - "pgl.yoyo.org") data=$(curl -s -d mimetype=plaintext -d hostformat=hosts -z $saveLocation $url);; - - *) data=$(curl -s -z $saveLocation -A "Mozilla/10.0" $url);; + # Default is a simple curl request + *) cmd="curl" esac - if [[ -n "$data" ]];then + # tmp file, so we don't have to store the (long!) lists in RAM + tmpfile=`mktemp` + timeCheck="" + if [ -r $saveLocation ]; then + timeCheck="-z $saveLocation" + fi + CMD="$cmd -s $timeCheck -A '$agent' $url > $tmpfile" + echo "running [$CMD]" + $cmd -s $timeCheck -A "$agent" $url > $tmpfile + + + if [[ -s "$tmpfile" ]];then # Remove comments and print only the domain name # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious # This helps with that and makes it easier to read # It also helps with debugging so each stage of the script can be researched more in depth - echo "$data" | awk 'NF {if ($1 !~ "#") { if (NF>1) {print $2} else {print $1}}}' | \ - sed -e 's/^[. \t]*//' -e 's/\.\.\+/./g' -e 's/[. \t]*$//' | grep "\." > $saveLocation + awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $tmpfile | \ + sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $saveLocation echo "Done." else echo "Skipping list because it does not have any new entries." fi + + # cleanup + rm -f $tmpfile done # Find all files with the .domains extension and compile them into one file and remove CRs @@ -114,39 +131,43 @@ echo "** Aggregating list of domains..." find $origin/ -type f -name "*.$justDomainsExtension" -exec cat {} \; | tr -d '\r' > $origin/$matter # Append blacklist entries if they exist -if [[ -f $blacklist ]];then +if [[ -r $blacklist ]];then numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) echo "** Blacklisting $numberOf domain(s)..." cat $blacklist >> $origin/$matter -else - : fi -function gravity_advanced() ########################### - { - numberOf=$(cat $origin/$andLight | sed '/^\s*$/d' | wc -l) - echo "** $numberOf domains being pulled in by gravity..." +function gravity_advanced() { + + numberOf=$(wc -l $origin/$andLight) + echo "** $numberOf domains being pulled in by gravity..." + # Remove carriage returns and preceding whitespace - cat $origin/$andLight | sed $'s/\r$//' | sed '/^\s*$/d' > $origin/$supernova + # not really needed anymore? + cp $origin/$andLight $origin/$supernova + # Sort and remove duplicates - cat $origin/$supernova | sort | uniq > $origin/$eventHorizon - numberOf=$(cat $origin/$eventHorizon | sed '/^\s*$/d' | wc -l) + sort -u $origin/$supernova > $origin/$eventHorizon + numberOf=$(wc -l $origin/$eventHorizon) echo "** $numberOf unique domains trapped in the event horizon." + # Format domain list as "192.168.x.x domain.com" echo "** Formatting domains into a HOSTS file..." - cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $origin/$accretionDisc + awk '{print "'"$piholeIP"'" $1}' $origin/$eventHorizon > $origin/$accretionDisc + # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq) - } - +} + # Whitelist (if applicable) then remove duplicates and format for dnsmasq -if [[ -f $whitelist ]];then +if [[ -r $whitelist ]];then # Remove whitelist entries numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) plural=; [[ "$numberOf" != "1" ]] && plural=s echo "** Whitelisting $numberOf domain${plural}..." + # Append a "$" to the end, prepend a "^" to the beginning, and # replace "." with "\." of each line to turn each entry into a # regexp so it can be parsed out with grep -x @@ -163,6 +184,7 @@ do echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist done +# Remove whitelist entries from deduped list grep -vxf $latentWhitelist $origin/$matter > $origin/$andLight gravity_advanced From 8e21488542c2bbe32d6663243248b6d0f6ef56fe Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 5 Nov 2015 18:22:17 -0800 Subject: [PATCH 15/76] hawson memory reductions --- gravity.sh | 47 +++++++++++------------------------------------ 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/gravity.sh b/gravity.sh index 8e8da8e..8ca34bb 100755 --- a/gravity.sh +++ b/gravity.sh @@ -44,31 +44,6 @@ else sudo mkdir $piholeDir fi -# Add additional swap to prevent the "Error fork: unable to allocate memory" message: https://github.com/jacobsalmela/pi-hole/issues/37 -function createSwapFile() -######################### - { - echo "** Creating more swap space to accomodate large solar masses..." - sudo dphys-swapfile swapoff - sudo curl -s -o /etc/dphys-swapfile https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dphys-swapfile - sudo dphys-swapfile setup - sudo dphys-swapfile swapon - } - -if [[ -f /etc/dphys-swapfile ]];then - swapSize=$(cat /etc/dphys-swapfile | grep -m1 CONF_SWAPSIZE | cut -d'=' -f2) - if [[ $swapSize != 500 ]];then - mv /etc/dphys-swapfile /etc/dphys-swapfile.orig - echo "** Current swap size is $swapSize" - createSwapFile - else - : - fi -else - echo "** No swap file found. Creating one..." - createSwapFile -fi - # Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and blank lines for ((i = 0; i < "${#sources[@]}"; i++)) do @@ -100,22 +75,22 @@ do esac # tmp file, so we don't have to store the (long!) lists in RAM - tmpfile=`mktemp` - timeCheck="" + patternBuffer=`mktemp` + heisenbergCompensator="" if [ -r $saveLocation ]; then - timeCheck="-z $saveLocation" + heisenbergCompensator="-z $saveLocation" fi - CMD="$cmd -s $timeCheck -A '$agent' $url > $tmpfile" - echo "running [$CMD]" - $cmd -s $timeCheck -A "$agent" $url > $tmpfile + CMD="$cmd -s $heisenbergCompensator -A '$agent' $url > $patternBuffer" + echo "** Engaging pattern transference..." + $cmd -s $heisenbergCompensator -A "$agent" $url > $patternBuffer - if [[ -s "$tmpfile" ]];then + if [[ -s "$patternBuffer" ]];then # Remove comments and print only the domain name # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious # This helps with that and makes it easier to read # It also helps with debugging so each stage of the script can be researched more in depth - awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $tmpfile | \ + awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $patternBuffer | \ sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $saveLocation echo "Done." else @@ -123,7 +98,7 @@ do fi # cleanup - rm -f $tmpfile + rm -f $patternBuffer done # Find all files with the .domains extension and compile them into one file and remove CRs @@ -141,7 +116,7 @@ fi function gravity_advanced() { numberOf=$(wc -l $origin/$andLight) - echo "** $numberOf domains being pulled in by gravity..." + echo "** $numberOf $origin/$andLight domains being pulled in by gravity..." # Remove carriage returns and preceding whitespace # not really needed anymore? @@ -166,7 +141,7 @@ if [[ -r $whitelist ]];then # Remove whitelist entries numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) plural=; [[ "$numberOf" != "1" ]] && plural=s - echo "** Whitelisting $numberOf domain${plural}..." + echo "** Whitelisting $numberOf $whitelist domain${plural}..." # Append a "$" to the end, prepend a "^" to the beginning, and # replace "." with "\." of each line to turn each entry into a From 642c31c361acc6d3e433409322e018909ce51035 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 5 Nov 2015 18:33:05 -0800 Subject: [PATCH 16/76] Theme changes --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 8ca34bb..e8c0acb 100755 --- a/gravity.sh +++ b/gravity.sh @@ -94,7 +94,7 @@ do sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $saveLocation echo "Done." else - echo "Skipping list because it does not have any new entries." + echo "Skipping pattern because transporter logic detected no changes..." fi # cleanup @@ -116,7 +116,7 @@ fi function gravity_advanced() { numberOf=$(wc -l $origin/$andLight) - echo "** $numberOf $origin/$andLight domains being pulled in by gravity..." + echo "** $numberOf domains being pulled in by gravity..." # Remove carriage returns and preceding whitespace # not really needed anymore? From 15b9fb68d7e86dcbc091a204ff862cabca02e312 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Fri, 6 Nov 2015 10:24:12 -0800 Subject: [PATCH 17/76] Remove filename from wc Line 118 and Line 127 changed wc to remove filename from output. --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index e8c0acb..a4400a5 100755 --- a/gravity.sh +++ b/gravity.sh @@ -115,7 +115,7 @@ fi ########################### function gravity_advanced() { - numberOf=$(wc -l $origin/$andLight) + numberOf=$(wc -l < $origin/$andLight) echo "** $numberOf domains being pulled in by gravity..." # Remove carriage returns and preceding whitespace @@ -124,7 +124,7 @@ function gravity_advanced() { # Sort and remove duplicates sort -u $origin/$supernova > $origin/$eventHorizon - numberOf=$(wc -l $origin/$eventHorizon) + numberOf=$(wc -l < $origin/$eventHorizon) echo "** $numberOf unique domains trapped in the event horizon." # Format domain list as "192.168.x.x domain.com" From a06098bb1699b41b446892dba152b4751f807c34 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:03:55 -0600 Subject: [PATCH 18/76] remove filename from whitelist output I also cleared the whitespace (or rather, Atom did). I moved the pinhole.conf to it's current location I commented out mahakala --- gravity.sh | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/gravity.sh b/gravity.sh index a4400a5..33bf226 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,6 +1,6 @@ #!/bin/bash # http://pi-hole.net -# Compiles a list of ad-serving domains by downloading them from multiple sources +# Compiles a list of ad-serving domains by downloading them from multiple sources # This script should only be run after you have a static IP address set on the Pi piholeIP=$(hostname -I) @@ -8,7 +8,7 @@ piholeIP=$(hostname -I) # Ad-list sources--one per line in single quotes sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' -'http://adblock.mahakala.is/' +#'http://adblock.mahakala.is/' 'http://hosts-file.net/.%5Cad_servers.txt' 'http://www.malwaredomainlist.com/hostslist/hosts.txt' 'http://pgl.yoyo.org/adservers/serverlist.php?' @@ -19,9 +19,6 @@ sources=('https://adaway.org/hosts.txt' adList=/etc/pihole/gravity.list origin=/etc/pihole piholeDir=/etc/pihole -if [[ -f $piholeDir/pihole.conf ]];then - . $piholeDir/pihole.conf -fi justDomainsExtension=domains matter=pihole.0.matter.txt andLight=pihole.1.andLight.txt @@ -30,10 +27,14 @@ eventHorizon=pihole.3.eventHorizon.txt accretionDisc=pihole.4.accretionDisc.txt eyeOfTheNeedle=pihole.5.wormhole.txt blacklist=$piholeDir/blacklist.txt -latentBlacklist=$origin/latentBlacklist.txt whitelist=$piholeDir/whitelist.txt latentWhitelist=$origin/latentWhitelist.txt +# After setting defaults, check if there's local overrides +if [[ -r $piholeDir/pihole.conf ]];then + echo "** Local calibration requested..." + . $piholeDir/pihole.conf +fi echo "** Neutrino emissions detected..." # Create the pihole resource directory if it doesn't exist. Future files will be stored here @@ -50,23 +51,23 @@ do url=${sources[$i]} # Get just the domain from the URL domain=$(echo "$url" | cut -d'/' -f3) - + # Save the file as list.#.domain saveLocation=$origin/list.$i.$domain.$justDomainsExtension agent="Mozilla/10.0" - + echo -n "Getting $domain list... " - # Use a case statement to download lists that need special cURL commands + # Use a case statement to download lists that need special cURL commands # to complete properly and reset the user agent when required case "$domain" in - "adblock.mahakala.is") + "adblock.mahakala.is") agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' cmd="curl -e http://forum.xda-developers.com/" ;; - - "pgl.yoyo.org") + + "pgl.yoyo.org") cmd="curl -d mimetype=plaintext -d hostformat=hosts" ;; @@ -77,14 +78,14 @@ do # tmp file, so we don't have to store the (long!) lists in RAM patternBuffer=`mktemp` heisenbergCompensator="" - if [ -r $saveLocation ]; then + if [ -r $saveLocation ]; then heisenbergCompensator="-z $saveLocation" fi CMD="$cmd -s $heisenbergCompensator -A '$agent' $url > $patternBuffer" echo "** Engaging pattern transference..." $cmd -s $heisenbergCompensator -A "$agent" $url > $patternBuffer - + if [[ -s "$patternBuffer" ]];then # Remove comments and print only the domain name # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious @@ -116,11 +117,11 @@ fi function gravity_advanced() { numberOf=$(wc -l < $origin/$andLight) - echo "** $numberOf domains being pulled in by gravity..." + echo "** $numberOf domains being pulled in by gravity..." # Remove carriage returns and preceding whitespace # not really needed anymore? - cp $origin/$andLight $origin/$supernova + cp $origin/$andLight $origin/$supernova # Sort and remove duplicates sort -u $origin/$supernova > $origin/$eventHorizon @@ -135,13 +136,13 @@ function gravity_advanced() { sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq) } - + # Whitelist (if applicable) then remove duplicates and format for dnsmasq if [[ -r $whitelist ]];then # Remove whitelist entries numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) plural=; [[ "$numberOf" != "1" ]] && plural=s - echo "** Whitelisting $numberOf $whitelist domain${plural}..." + echo "** Whitelisting $numberOf domain${plural}..." # Append a "$" to the end, prepend a "^" to the beginning, and # replace "." with "\." of each line to turn each entry into a From 1b65193fe40e9f1efe79765f0f762e3386d873d5 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:05:04 -0600 Subject: [PATCH 19/76] new hostsfile url This seems to be the more appropriate URL: http://hosts-file.net/ad_servers.txt. I compared it to the original URL using opendiff and they are exactly the same. --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 33bf226..2715ddd 100755 --- a/gravity.sh +++ b/gravity.sh @@ -9,7 +9,7 @@ piholeIP=$(hostname -I) sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' #'http://adblock.mahakala.is/' -'http://hosts-file.net/.%5Cad_servers.txt' +'http://hosts-file.net/ad_servers.txt' 'http://www.malwaredomainlist.com/hostslist/hosts.txt' 'http://pgl.yoyo.org/adservers/serverlist.php?' 'http://someonewhocares.org/hosts/hosts' From c297a86211c69557f72bee3241422e3feceb5f67 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:10:17 -0600 Subject: [PATCH 20/76] decluttering output I love the adherence to the theme, but this line makes it difficult to quickly read what is happening. --- gravity.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 2715ddd..b157adb 100755 --- a/gravity.sh +++ b/gravity.sh @@ -82,7 +82,6 @@ do heisenbergCompensator="-z $saveLocation" fi CMD="$cmd -s $heisenbergCompensator -A '$agent' $url > $patternBuffer" - echo "** Engaging pattern transference..." $cmd -s $heisenbergCompensator -A "$agent" $url > $patternBuffer From 932d410b66e0fda80ea78b894c9427121d3d8b4a Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:14:05 -0600 Subject: [PATCH 21/76] anal retentive changes OCD --- gravity.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index b157adb..4c191c3 100755 --- a/gravity.sh +++ b/gravity.sh @@ -76,9 +76,9 @@ do esac # tmp file, so we don't have to store the (long!) lists in RAM - patternBuffer=`mktemp` + patternBuffer=$(mktemp) heisenbergCompensator="" - if [ -r $saveLocation ]; then + if [[ -r $saveLocation ]]; then heisenbergCompensator="-z $saveLocation" fi CMD="$cmd -s $heisenbergCompensator -A '$agent' $url > $patternBuffer" @@ -97,7 +97,7 @@ do echo "Skipping pattern because transporter logic detected no changes..." fi - # cleanup + # Cleanup rm -f $patternBuffer done From e1b8419a07a8a6feea408815c97986ecb44e9877 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:16:14 -0600 Subject: [PATCH 22/76] theme addition Further adherence to the science/star trek theme --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 4c191c3..b1024b0 100755 --- a/gravity.sh +++ b/gravity.sh @@ -75,7 +75,8 @@ do *) cmd="curl" esac - # tmp file, so we don't have to store the (long!) lists in RAM + echo "Narrowing the annular confinment beam..." + # Create a tmp file so we don't have to store the (long!) lists in RAM patternBuffer=$(mktemp) heisenbergCompensator="" if [[ -r $saveLocation ]]; then From a7d1b0b42c9dcc86b8ff274a4185d1ca78d47b50 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:17:14 -0600 Subject: [PATCH 23/76] Revert "theme addition" This reverts commit e1b8419a07a8a6feea408815c97986ecb44e9877. --- gravity.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index b1024b0..4c191c3 100755 --- a/gravity.sh +++ b/gravity.sh @@ -75,8 +75,7 @@ do *) cmd="curl" esac - echo "Narrowing the annular confinment beam..." - # Create a tmp file so we don't have to store the (long!) lists in RAM + # tmp file, so we don't have to store the (long!) lists in RAM patternBuffer=$(mktemp) heisenbergCompensator="" if [[ -r $saveLocation ]]; then From ce0ca3cc61a138a37e1f285eb2f14a5e15ca603a Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 7 Nov 2015 07:58:35 -0600 Subject: [PATCH 24/76] for use with /etc/crontab Adding a system-wide crontab to update the ad source lists, flush the log daily, and (eventually) record the stats for historical purposes. --- advanced/pihole.cron | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/advanced/pihole.cron b/advanced/pihole.cron index a707607..47bc61f 100644 --- a/advanced/pihole.cron +++ b/advanced/pihole.cron @@ -1 +1,26 @@ -@weekly sudo /usr/local/bin/gravity.sh +# /etc/crontab: system-wide crontab +# Unlike any other crontab you don't have to run the `crontab' +# command to install the new version when you edit this file +# and files in /etc/cron.d. These files also have username fields, +# that none of the other crontabs do. + +SHELL=/bin/sh +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +# m h dom mon dow user command +17 * * * * root cd / && run-parts --report /etc/cron.hourly +25 6 * * * root test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.daily ) +47 6 * * 7 root test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.weekly ) +52 6 1 * * root test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.monthly ) + +# Pi-hole: Update the ad sources once a week on Sunday at 01:59 +# Download any updates from the ad lists +59 1 * * 7 root /usr/local/bin/gravity.sh + +# Pi-hole: Parse the log file before it is flushed and save the stats to a database +# This will be used for a historical view of your Pi-hole's performance +#50 11 * * * root /usr/local/bin/dailyLog.sh + +# Pi-hole: Flush the log daily at 11:58 so it doesn't get out of control +# Stats will be viewable in the Web interface thanks to the cron job above +58 11 * * * root /usr/local/bin/piholeLogFlush.sh From 763aa6aa5c6c3fc194fa8c2df6a258c4a7adf41e Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Wed, 11 Nov 2015 00:43:00 -0800 Subject: [PATCH 25/76] Change shebang to environment Find bash in the environment, not hard coded to path --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 4c191c3..9379319 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # http://pi-hole.net # Compiles a list of ad-serving domains by downloading them from multiple sources From 4229a3e5e9835a38c3c19b2e4ad483c03706fd11 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Wed, 28 Oct 2015 17:29:34 -0500 Subject: [PATCH 26/76] whiptail revamp of install script This is a large update that helps users install the Pi-hole a bit easier by using some dialog windows and also letting the user set a static IP (I still need to figure out how to actually set the address in the system, but everything else having to do with the install script works). --- automated install/basic-install.sh | 229 +++++++++++++++++++++-------- 1 file changed, 164 insertions(+), 65 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 7c48fdb..416c421 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1,29 +1,37 @@ #!/bin/bash -# Pi-hole automated install -# Raspberry Pi Ad-blocker +# Pi-hole: A black hole for Internet advertisements +# by Jacob Salmela +# Network-wide ad blocking via your Raspberry Pi # -# Install with this command (from the Pi): +# pi-hole.net/donate # -# curl -s "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/automated%20install/basic-install.sh" | bash +# Install with this command (from your Pi): # -# Or run the commands below in order +# curl -L install.pi-hole.net | bash -clear -echo " _____ _ _ _ " -echo " | __ (_) | | | | " -echo " | |__) | __ | |__ ___ | | ___ " -echo " | ___/ | |__| | '_ \ / _ \| |/ _ \ " -echo " | | | | | | | | (_) | | __/ " -echo " |_| |_| |_| |_|\___/|_|\___| " -echo " " -echo " Raspberry Pi Ad-blocker " -echo " " -echo "Set a static IP before running this!" -echo " " -echo " Press Enter when ready " -echo " " -read +######## VARIABLES ######### +# Get the screen size in case we need a full-screen message and so we can display a dialog that is sized nicely +screenSize=$(stty -a | tr \; \\012 | egrep 'rows|columns' | cut '-d ' -f3) +# Find the rows and columns +rows=$(stty -a | tr \; \\012 | egrep 'rows' | cut -d' ' -f3) +columns=$(stty -a | tr \; \\012 | egrep 'columns' | cut -d' ' -f3) + +# Divide by two so the dialogs take up half of the screen, which looks nice. +r=$(( rows / 2 )) +c=$(( columns / 2 )) + +# Get the current network settings +IPv4addr=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1") +IPv4mask=$(ifconfig | awk -F':' '/inet addr/ && !/127.0.0.1/ {print $4}') +IPv4gw=$(ip route show | awk '/default\ via/ {print $3}') + +# IPv6 support to be added later +IPv6addr=$(ip addr show | awk '/scope\ global/ && /ff:fe/ {print $2}' | cut -d'/' -f1) + +####### FUCNTIONS ########## +backupLegacyPihole() +{ if [[ -f /etc/dnsmasq.d/adList.conf ]];then echo "Original Pi-hole detected. Initiating sub space transport..." sudo mkdir -p /etc/pihole/original/ @@ -36,58 +44,149 @@ if [[ -f /etc/dnsmasq.d/adList.conf ]];then else : fi +} -echo "Updating the Pi..." -sudo apt-get update -sudo apt-get -y upgrade +######## SCRIPT ############ +# Just back up the original Pi-hole right away since it won't take long and it gets it out of the way +backupLegacyPihole -echo "Installing tools..." -sudo apt-get -y install dnsutils -sudo apt-get -y install bc -sudo apt-get -y install toilet +# Display the welcome dialog +whiptail --msgbox --backtitle "Welcome..." --title "Pi-hole automated installer" "This installer will transform your Raspberry Pi into a network-wide ad blocker!" $r $c -echo "Installing DNS..." -sudo apt-get -y install dnsmasq -sudo update-rc.d dnsmasq enable +# Explain the need for a static address +whiptail --msgbox --backtitle "Initating network interface..." --title "Static IP Needed" "The Pi-hole is a SERVER so it needs a STATIC IP ADDRESS to function properly. -echo "Installing a Web server" -sudo apt-get -y install lighttpd php5-common php5-cgi php5 -sudo mkdir /var/www/html -sudo chown www-data:www-data /var/www/html -sudo chmod 775 /var/www/html -sudo usermod -a -G www-data pi +In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." $r $c -echo "Stopping services to modify them..." -sudo service dnsmasq stop -sudo service lighttpd stop +# Ask if the user wannts to use DHCP settings as their static IP +if (whiptail --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address? -echo "Backing up original config files and downloading Pi-hole ones..." -sudo mv /etc/dnsmasq.conf /etc/dnsmasq.conf.orig -sudo mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig -sudo mv /var/www/html/index.lighttpd.html /var/www/html/index.lighttpd.orig -sudo curl -o /etc/dnsmasq.conf "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dnsmasq.conf" -sudo curl -o /etc/lighttpd/lighttpd.conf "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/lighttpd.conf" -sudo lighty-enable-mod fastcgi fastcgi-php -sudo mkdir /var/www/html/pihole -sudo curl -o /var/www/html/pihole/index.html "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html" + IP address: $IPv4addr + Subnet mask: $IPv4mask + Gateway: $IPv4gw" $r $c) then + # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. + whiptail --msgbox --backtitle "IP information..." --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict. But in most cases the router is smart enough to not do that. -echo "Installing the Web interface..." -sudo wget https://github.com/jacobsalmela/AdminLTE/archive/master.zip -O /var/www/master.zip -sudo unzip /var/www/master.zip -d /var/www/html/ -sudo mv /var/www/html/AdminLTE-master /var/www/html/admin -sudo rm /var/www/master.zip 2>/dev/null -sudo touch /var/log/pihole.log -sudo chmod 644 /var/log/pihole.log -sudo chown dnsmasq:root /var/log/pihole.log +If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want. -echo "Locating the Pi-hole..." -sudo curl -o /usr/local/bin/gravity.sh "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh" -sudo curl -o /usr/local/bin/chronometer.sh "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh" -sudo chmod 755 /usr/local/bin/gravity.sh -sudo chmod 755 /usr/local/bin/chronometer.sh +It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." $r $c + # Nothing else to do since the variables are already set above +else + # Otherwise, we need to ask the user to input their desired settings. + # Start by getting the IPv4 address (pre-filling it with info gathered from DHCP) + # Start a loop to let the user enter their information with the chance to go back and edit it if necessary + until [[ $ipSettingsCorrect = True ]] + do + # Ask for the IPv4 address + IPv4addr=$(whiptail --backtitle "Calibrating network interface..." --title "IPv4 address" --inputbox "Enter your desired IPv4 address" $r $c $IPv4addr 3>&1 1>&2 2>&3) + if [[ $? = 0 ]];then + echo "Your static IPv4 address: $IPv4addr" + # Ask for the subnet mask + IPv4mask=$(whiptail --backtitle "Calibrating network interface..." --title "IPv4 netmask" --inputbox "Enter your desired IPv4 subnet mask" $r $c $IPv4mask 3>&1 1>&2 2>&3) + if [[ $? = 0 ]];then + echo "Your static IPv4 netmask: $IPv4mask" + # Ask for the gateway + IPv4gw=$(whiptail --backtitle "Calibrating network interface..." --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" $r $c $IPv4gw 3>&1 1>&2 2>&3) + if [[ $? = 0 ]];then + echo "Your static IPv4 gateway: $IPv4gw" + # Give the user a chance to review their settings before moving on + if (whiptail --title "Static IP Address" --yesno "Are these settings correct? + IP address: $IPv4addr + Subnet mask: $IPv4mask + Gateway: $IPv4gw" $r $c)then + # If the settings are correct, then we need to set the piholeIP + # Saving it to a temporary file us to retrieve it later when we run the gravity.sh script + echo $IPv4addr > /tmp/piholeIP + # After that's done, the loop ends and we move on + ipSettingsCorrect=True -echo "Entering the event horizon..." -sudo /usr/local/bin/gravity.sh + else + # If the settings are wrong, the loop continues + ipSettingsCorrect=False + fi + else + # Cancelling gateway settings window + ipSettingsCorrect=False + echo "User canceled." + exit + fi + else + # Cancelling subnet mask settings window + ipSettingsCorrect=False + echo "User canceled." + exit + fi + else + # Cancelling IPv4 settings window + ipSettingsCorrect=False + echo "User canceled." + exit + fi + done +# End the if statement for DHCP vs. static +fi -echo "Restarting..." -sudo reboot +# These are the commands to actually install the Pi-hole +# Create an associative array so we can display text to the user but run the associated command in the background. +declare -A cmdsAndEchoes=([sudo apt-get update]='Updating...' +[sudo apt-get -y upgrade]='Upgrading...' +[sudo apt-get -y install dnsutils bc toilet]='Installing chronomoter tools...' +[sudo apt-get -y install dnsmasq]='Installing a DNS server...' +[sudo apt-get -y install lighttpd php5-common php5-cgi php5]='Instaling a Web server and PHP...' +[sudo mkdir /var/www/html]='Making an HTML folder...' +[sudo chown www-data:www-data /var/www/html]='Setting permissions for the Web server...' +[sudo chmod 775 /var/www/html]='Setting permissions for the Web server...' +[sudo usermod -a -G www-data pi]='Setting permissions for the Web server...' +[sudo service dnsmasq stop]='Stopping dnsmasq to modify it...' +[sudo service lighttpd stop]='Stopping lighttpd to modify it...' +[sudo mv /etc/dnsmasq.conf /etc/dnsmasq.conf.orig]='Backing up the dnsmasq config file...' +[sudo mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig]='Backing up the lighttpd config file...' +[sudo mv /var/www/html/index.lighttpd.html /var/www/html/index.lighttpd.orig]='Backing up the default Web page...' +[sudo curl -o /etc/dnsmasq.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dnsmasq.conf]='Installing the dnsmasq config file...' +[sudo curl -o /etc/lighttpd/lighttpd.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/lighttpd.conf]='Installing the lighttpd config file...' +[sudo lighty-enable-mod fastcgi fastcgi-php]='Enabling PHP...' +[sudo mkdir /var/www/html/pihole]='Making a directory for the Web interface...' +[sudo curl -o /var/www/html/pihole/index.html https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html]='Installing a blank HTML page to take place of ads...' +[sudo wget https://github.com/jacobsalmela/AdminLTE/archive/master.zip -O /var/www/master.zip]='Downloading the Pi-hole dashboard...' +[sudo unzip /var/www/master.zip -d /var/www/html/]='Unpacking the dashboard...' +[sudo mv /var/www/html/AdminLTE-master /var/www/html/admin]='Renaming the dashboard...' +[sudo rm /var/www/master.zip 2>/dev/null]='Cleaning up the dashboard temp files...' +[sudo touch /var/log/pihole.log]='Creating a log file for the Pi-hole...' +[sudo chmod 644 /var/log/pihole.log]='Making sure the log is readable...' +[sudo chown dnsmasq:root /var/log/pihole.log]='Letting dnsmasq see the log file so stats can be displayed...' +[sudo curl -o /usr/local/bin/gravity.sh "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh"]='Initating sub-space transport...' +[sudo curl -o /usr/local/bin/chronometer.sh "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh"]='Initating sub-space transport...' +[sudo chmod 755 /usr/local/bin/gravity.sh]='Making the scripts executable...' +[sudo chmod 755 /usr/local/bin/chronometer.sh]='Making the scripts executable...' +[sudo /usr/local/bin/gravity.sh]='Entering the event horizion...' +[sudo reboot]='Restarting...') + +# Everything in the parentheses is part of displaying the progress bar +( +# Get total number of commands to be run from the array +n=${#cmdsAndEchoes[*]}; + +# Set counter to increase every time a loop completes +i=0 + +# For each key in the array +for key in "${!cmdsAndEchoes[@]}" +do + +# Calculate the overall progress +percent=$(( 100*(++i)/n )) + +# Update dialog box using the value of each key in the array +# Show the percentage and the echo messages from the array +cat < Date: Thu, 29 Oct 2015 20:10:30 -0500 Subject: [PATCH 27/76] progress dialog working The ellipses cause errors, so they were removed. I was on the right track with the associate array, but it would previously execute the commands out of order. Following directions from the URL in the comments, I made another array that helps keep the array in the order. --- automated install/basic-install.sh | 96 ++++++++++++++++-------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 416c421..56beee8 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -33,7 +33,7 @@ IPv6addr=$(ip addr show | awk '/scope\ global/ && /ff:fe/ {print $2}' | cut -d'/ backupLegacyPihole() { if [[ -f /etc/dnsmasq.d/adList.conf ]];then - echo "Original Pi-hole detected. Initiating sub space transport..." + echo "Original Pi-hole detected. Initiating sub space transport" sudo mkdir -p /etc/pihole/original/ sudo mv /etc/dnsmasq.d/adList.conf /etc/pihole/original/adList.conf.$(date "+%Y-%m-%d") sudo mv /etc/dnsmasq.conf /etc/pihole/original/dnsmasq.conf.$(date "+%Y-%m-%d") @@ -51,10 +51,10 @@ fi backupLegacyPihole # Display the welcome dialog -whiptail --msgbox --backtitle "Welcome..." --title "Pi-hole automated installer" "This installer will transform your Raspberry Pi into a network-wide ad blocker!" $r $c +whiptail --msgbox --backtitle "Welcome" --title "Pi-hole automated installer" "This installer will transform your Raspberry Pi into a network-wide ad blocker!" $r $c # Explain the need for a static address -whiptail --msgbox --backtitle "Initating network interface..." --title "Static IP Needed" "The Pi-hole is a SERVER so it needs a STATIC IP ADDRESS to function properly. +whiptail --msgbox --backtitle "Initating network interface" --title "Static IP Needed" "The Pi-hole is a SERVER so it needs a STATIC IP ADDRESS to function properly. In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." $r $c @@ -65,7 +65,7 @@ if (whiptail --title "Static IP Address" --yesno "Do you want to use your curren Subnet mask: $IPv4mask Gateway: $IPv4gw" $r $c) then # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. - whiptail --msgbox --backtitle "IP information..." --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict. But in most cases the router is smart enough to not do that. + whiptail --msgbox --backtitle "IP information" --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict. But in most cases the router is smart enough to not do that. If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want. @@ -78,15 +78,15 @@ else until [[ $ipSettingsCorrect = True ]] do # Ask for the IPv4 address - IPv4addr=$(whiptail --backtitle "Calibrating network interface..." --title "IPv4 address" --inputbox "Enter your desired IPv4 address" $r $c $IPv4addr 3>&1 1>&2 2>&3) + IPv4addr=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" $r $c $IPv4addr 3>&1 1>&2 2>&3) if [[ $? = 0 ]];then echo "Your static IPv4 address: $IPv4addr" # Ask for the subnet mask - IPv4mask=$(whiptail --backtitle "Calibrating network interface..." --title "IPv4 netmask" --inputbox "Enter your desired IPv4 subnet mask" $r $c $IPv4mask 3>&1 1>&2 2>&3) + IPv4mask=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 netmask" --inputbox "Enter your desired IPv4 subnet mask" $r $c $IPv4mask 3>&1 1>&2 2>&3) if [[ $? = 0 ]];then echo "Your static IPv4 netmask: $IPv4mask" # Ask for the gateway - IPv4gw=$(whiptail --backtitle "Calibrating network interface..." --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" $r $c $IPv4gw 3>&1 1>&2 2>&3) + IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" $r $c $IPv4gw 3>&1 1>&2 2>&3) if [[ $? = 0 ]];then echo "Your static IPv4 gateway: $IPv4gw" # Give the user a chance to review their settings before moving on @@ -127,50 +127,52 @@ else fi # These are the commands to actually install the Pi-hole -# Create an associative array so we can display text to the user but run the associated command in the background. -declare -A cmdsAndEchoes=([sudo apt-get update]='Updating...' -[sudo apt-get -y upgrade]='Upgrading...' -[sudo apt-get -y install dnsutils bc toilet]='Installing chronomoter tools...' -[sudo apt-get -y install dnsmasq]='Installing a DNS server...' -[sudo apt-get -y install lighttpd php5-common php5-cgi php5]='Instaling a Web server and PHP...' -[sudo mkdir /var/www/html]='Making an HTML folder...' -[sudo chown www-data:www-data /var/www/html]='Setting permissions for the Web server...' -[sudo chmod 775 /var/www/html]='Setting permissions for the Web server...' -[sudo usermod -a -G www-data pi]='Setting permissions for the Web server...' -[sudo service dnsmasq stop]='Stopping dnsmasq to modify it...' -[sudo service lighttpd stop]='Stopping lighttpd to modify it...' -[sudo mv /etc/dnsmasq.conf /etc/dnsmasq.conf.orig]='Backing up the dnsmasq config file...' -[sudo mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig]='Backing up the lighttpd config file...' -[sudo mv /var/www/html/index.lighttpd.html /var/www/html/index.lighttpd.orig]='Backing up the default Web page...' -[sudo curl -o /etc/dnsmasq.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dnsmasq.conf]='Installing the dnsmasq config file...' -[sudo curl -o /etc/lighttpd/lighttpd.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/lighttpd.conf]='Installing the lighttpd config file...' -[sudo lighty-enable-mod fastcgi fastcgi-php]='Enabling PHP...' -[sudo mkdir /var/www/html/pihole]='Making a directory for the Web interface...' -[sudo curl -o /var/www/html/pihole/index.html https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html]='Installing a blank HTML page to take place of ads...' -[sudo wget https://github.com/jacobsalmela/AdminLTE/archive/master.zip -O /var/www/master.zip]='Downloading the Pi-hole dashboard...' -[sudo unzip /var/www/master.zip -d /var/www/html/]='Unpacking the dashboard...' -[sudo mv /var/www/html/AdminLTE-master /var/www/html/admin]='Renaming the dashboard...' -[sudo rm /var/www/master.zip 2>/dev/null]='Cleaning up the dashboard temp files...' -[sudo touch /var/log/pihole.log]='Creating a log file for the Pi-hole...' -[sudo chmod 644 /var/log/pihole.log]='Making sure the log is readable...' -[sudo chown dnsmasq:root /var/log/pihole.log]='Letting dnsmasq see the log file so stats can be displayed...' -[sudo curl -o /usr/local/bin/gravity.sh "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh"]='Initating sub-space transport...' -[sudo curl -o /usr/local/bin/chronometer.sh "https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh"]='Initating sub-space transport...' -[sudo chmod 755 /usr/local/bin/gravity.sh]='Making the scripts executable...' -[sudo chmod 755 /usr/local/bin/chronometer.sh]='Making the scripts executable...' -[sudo /usr/local/bin/gravity.sh]='Entering the event horizion...' -[sudo reboot]='Restarting...') +# This is pretty ugly, but it works to present a nice front-end +# http://stackoverflow.com/questions/29161323/how-to-keep-associative-array-order-in-bash +# Maybe it would be better to just show the command output instead of the progress bar +declare -A commands; declare -a echoes; +commands["Updating"]="sudo apt-get update"; echoes+=( "Updating" ) +commands["Upgrading"]="sudo apt-get -y upgrade"; echoes+=( "Upgrading" ) +commands["Installing chronomoter tools"]="sudo apt-get -y install dnsutils bc toilet"; echoes+=( "Installing chronomoter tools" ) +commands["Installing a DNS server"]="sudo apt-get -y install dnsmasq"; echoes+=( "Installing a DNS server" ) +commands["Instaling a Web server and PHP"]="sudo apt-get -y install lighttpd php5-common php5-cgi php5"; echoes+=( "Instaling a Web server and PHP" ) +commands["Making an HTML folder"]="sudo mkdir /var/www/html"; echoes+=( "Making an HTML folder" ) +commands["chowning the Web server"]="sudo chown www-data:www-data /var/www/html"; echoes+=( "chowning the Web server" ) +commands["chmodding the Web server"]="sudo chmod 775 /var/www/html"; echoes+=( "chmodding the Web server" ) +commands["Giving pi access to the Web server"]="sudo usermod -a -G www-data pi"; echoes+=( "Giving pi access to the Web server" ) +commands["Stopping dnsmasq to modify it"]="sudo service dnsmasq stop"; echoes+=( "Stopping dnsmasq to modify it" ) +commands["Stopping lighttpd to modify it"]="sudo service lighttpd stop"; echoes+=( "Stopping lighttpd to modify it" ) +commands["Backing up the dnsmasq config file"]="sudo mv /etc/dnsmasq.conf /etc/dnsmasq.conf.orig"; echoes+=( "Backing up the dnsmasq config file" ) +commands["Backing up the lighttpd config file"]="sudo mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig"; echoes+=( "Backing up the lighttpd config file" ) +commands["Backing up the default Web page"]="sudo mv /var/www/html/index.lighttpd.html /var/www/html/index.lighttpd.orig"; echoes+=( "Backing up the default Web page" ) +commands["Installing the dnsmasq config file"]="sudo curl -o /etc/dnsmasq.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dnsmasq.conf"; echoes+=( "Installing the dnsmasq config file" ) +commands["Installing the lighttpd config file"]="sudo curl -o /etc/lighttpd/lighttpd.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/lighttpd.conf"; echoes+=( "Installing the lighttpd config file" ) +commands["Enabling PHP"]="sudo lighty-enable-mod fastcgi fastcgi-php"; echoes+=( "Enabling PHP" ) +commands["Making a directory for the Web interface"]="sudo mkdir /var/www/html/pihole"; echoes+=( "Making a directory for the Web interface" ) +commands["Installing a blank HTML page to take place of ads"]="sudo curl -o /var/www/html/pihole/index.html https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html"; echoes+=( "Installing a blank HTML page to take place of ads" ) +commands["Downloading the Pi-hole dashboard"]="sudo wget https://github.com/jacobsalmela/AdminLTE/archive/master.zip -O /var/www/master.zip"; echoes+=( "Downloading the Pi-hole dashboard" ) +commands["Unpacking the dashboard"]="sudo unzip /var/www/master.zip -d /var/www/html/"; echoes+=( "Unpacking the dashboard" ) +commands["Renaming the dashboard"]="sudo mv /var/www/html/AdminLTE-master /var/www/html/admin"; echoes+=( "Renaming the dashboard" ) +commands["Cleaning up the dashboard temp files"]="sudo rm /var/www/master.zip 2>/dev/null"; echoes+=( "Cleaning up the dashboard temp files" ) +commands["Creating a log file for the Pi-hole"]="sudo touch /var/log/pihole.log"; echoes+=( "Creating a log file for the Pi-hole" ) +commands["chmodding the log file"]="sudo chmod 644 /var/log/pihole.log"; echoes+=( "chmodding the log file" ) +commands["chowning the log file so stats can be displayed"]="sudo chown dnsmasq:root /var/log/pihole.log"; echoes+=( "chowning the log file so stats can be displayed" ) +commands["Initating sub-space transport of gravity"]="sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh"; echoes+=( "Initating sub-space transport of gravity" ) +commands["Initating sub-space transport of chronometer"]="sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh"; echoes+=( "Initating sub-space transport of chronometer" ) +commands["chmodding gravity"]="sudo chmod 755 /usr/local/bin/gravity.sh"; echoes+=( "chmodding gravity" ) +commands["chmodding the chronometer"]="sudo chmod 755 /usr/local/bin/chronometer.sh"; echoes+=( "chmodding the chronometer" ) +commands["Entering the event horizion"]="sudo /usr/local/bin/gravity.sh"; echoes+=( "Entering the event horizion" ) +commands["Rebooting"]="sudo reboot"; echoes+=( "Rebooting" ) # Everything in the parentheses is part of displaying the progress bar ( # Get total number of commands to be run from the array -n=${#cmdsAndEchoes[*]}; - +n=${#commands[*]}; # Set counter to increase every time a loop completes i=0 -# For each key in the array -for key in "${!cmdsAndEchoes[@]}" +# For each item in the array +for k in "${!echoes[@]}" do # Calculate the overall progress @@ -181,11 +183,13 @@ percent=$(( 100*(++i)/n )) cat < Date: Sat, 31 Oct 2015 09:11:29 -0500 Subject: [PATCH 28/76] set static ip and suppress more output Created a function to set a static IP address. I also sent STDOUT and STDERR to /dev/null so it doesn't mangle the progress bar. When the device reboots, it still gets mangled a bit (via ssh). --- automated install/basic-install.sh | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 56beee8..ea51306 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -27,7 +27,10 @@ IPv4mask=$(ifconfig | awk -F':' '/inet addr/ && !/127.0.0.1/ {print $4}') IPv4gw=$(ip route show | awk '/default\ via/ {print $3}') # IPv6 support to be added later -IPv6addr=$(ip addr show | awk '/scope\ global/ && /ff:fe/ {print $2}' | cut -d'/' -f1) +#IPv6addr=$(ip addr show | awk '/scope\ global/ && /ff:fe/ {print $2}' | cut -d'/' -f1) + +ethernetDevice="eth0" +dhcpcdFile=/etc/dhcpcd.conf ####### FUCNTIONS ########## backupLegacyPihole() @@ -46,6 +49,15 @@ else fi } +set_static_ip() +{ +# Append these lines to /etc/dhcpcd.conf to enable a static IP +echo "interface $ethernetDevice +static ip_address=$IPv4addr/24 +static routers=$IPv4gw +static domain_name_servers=$IPv4gw" | sudo tee -a $dhcpcdFile >/dev/null +} + ######## SCRIPT ############ # Just back up the original Pi-hole right away since it won't take long and it gets it out of the way backupLegacyPihole @@ -59,7 +71,7 @@ whiptail --msgbox --backtitle "Initating network interface" --title "Static IP N In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." $r $c # Ask if the user wannts to use DHCP settings as their static IP -if (whiptail --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address? +if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address? IP address: $IPv4addr Subnet mask: $IPv4mask @@ -90,7 +102,7 @@ else if [[ $? = 0 ]];then echo "Your static IPv4 gateway: $IPv4gw" # Give the user a chance to review their settings before moving on - if (whiptail --title "Static IP Address" --yesno "Are these settings correct? + if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Are these settings correct? IP address: $IPv4addr Subnet mask: $IPv4mask Gateway: $IPv4gw" $r $c)then @@ -125,6 +137,8 @@ else done # End the if statement for DHCP vs. static fi +# Set the static address +set_static_ip # These are the commands to actually install the Pi-hole # This is pretty ugly, but it works to present a nice front-end @@ -186,11 +200,11 @@ $percent Step $i of $n: ${echoes[$k]} XXX EOF + # Execute the command in the background (hidden from the user, not actually a background process) -#${echoes[$k]} -${commands[${echoes[$k]}]} -sleep 1 +${commands[${echoes[$k]}]} > /dev/null 2>&1 done + # As the loop is progressing, the output is sent to whiptail to be displayed to the user ) | whiptail --title "Opening your Pi-hole..." --gauge "Please wait..." $r $c 0 From 37f255d761be78b9682eb4f28991fb5e198caadd Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 31 Oct 2015 09:15:57 -0500 Subject: [PATCH 29/76] save output to log file instead Since the progress bar hides the output now. Instead of destroying it, send it to a log file instead so users who want to see it still can. --- automated install/basic-install.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ea51306..98935a8 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -10,6 +10,8 @@ # curl -L install.pi-hole.net | bash ######## VARIABLES ######### +instalLogLoc=/etc/pihole/install.log + # Get the screen size in case we need a full-screen message and so we can display a dialog that is sized nicely screenSize=$(stty -a | tr \; \\012 | egrep 'rows|columns' | cut '-d ' -f3) @@ -202,7 +204,7 @@ XXX EOF # Execute the command in the background (hidden from the user, not actually a background process) -${commands[${echoes[$k]}]} > /dev/null 2>&1 +${commands[${echoes[$k]}]} > $instalLogLoc 2>&1 done # As the loop is progressing, the output is sent to whiptail to be displayed to the user From a44591ca6be363d40d6c4c13f38021765ce2ce30 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 31 Oct 2015 09:19:57 -0500 Subject: [PATCH 30/76] fix permission denied for log file Just use a temporary file and then move it over once the install is complete. --- automated install/basic-install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 98935a8..9e36448 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -10,6 +10,7 @@ # curl -L install.pi-hole.net | bash ######## VARIABLES ######### +tmpLog=/tmp/pihole-install.log instalLogLoc=/etc/pihole/install.log # Get the screen size in case we need a full-screen message and so we can display a dialog that is sized nicely @@ -206,6 +207,7 @@ EOF # Execute the command in the background (hidden from the user, not actually a background process) ${commands[${echoes[$k]}]} > $instalLogLoc 2>&1 done +sudo mv $tmpLog $instalLogLoc$instalLogLoc # As the loop is progressing, the output is sent to whiptail to be displayed to the user ) | From f37990351e7f9c5e47072450bcabb1dfca099cae Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 31 Oct 2015 09:21:19 -0500 Subject: [PATCH 31/76] dumb mistake fixed the variable name --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9e36448..f39009b 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -205,7 +205,7 @@ XXX EOF # Execute the command in the background (hidden from the user, not actually a background process) -${commands[${echoes[$k]}]} > $instalLogLoc 2>&1 +${commands[${echoes[$k]}]} > $tmpLog 2>&1 done sudo mv $tmpLog $instalLogLoc$instalLogLoc From 6d989f9160c451e56f64c37728ccf3b18d941f24 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sun, 1 Nov 2015 12:54:19 -0600 Subject: [PATCH 32/76] overwrite html folder If the files already exist, the installer stalls out because it asks if you want to replace the existing files. the -o option forces an overwrite. --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f39009b..0db4c02 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -168,7 +168,7 @@ commands["Enabling PHP"]="sudo lighty-enable-mod fastcgi fastcgi-php"; echoes+=( commands["Making a directory for the Web interface"]="sudo mkdir /var/www/html/pihole"; echoes+=( "Making a directory for the Web interface" ) commands["Installing a blank HTML page to take place of ads"]="sudo curl -o /var/www/html/pihole/index.html https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html"; echoes+=( "Installing a blank HTML page to take place of ads" ) commands["Downloading the Pi-hole dashboard"]="sudo wget https://github.com/jacobsalmela/AdminLTE/archive/master.zip -O /var/www/master.zip"; echoes+=( "Downloading the Pi-hole dashboard" ) -commands["Unpacking the dashboard"]="sudo unzip /var/www/master.zip -d /var/www/html/"; echoes+=( "Unpacking the dashboard" ) +commands["Unpacking the dashboard"]="sudo unzip -o /var/www/master.zip -d /var/www/html/"; echoes+=( "Unpacking the dashboard" ) commands["Renaming the dashboard"]="sudo mv /var/www/html/AdminLTE-master /var/www/html/admin"; echoes+=( "Renaming the dashboard" ) commands["Cleaning up the dashboard temp files"]="sudo rm /var/www/master.zip 2>/dev/null"; echoes+=( "Cleaning up the dashboard temp files" ) commands["Creating a log file for the Pi-hole"]="sudo touch /var/log/pihole.log"; echoes+=( "Creating a log file for the Pi-hole" ) From 1482b9563c594e5b9ee3a6c08264f816fc5644c5 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sun, 1 Nov 2015 13:00:55 -0600 Subject: [PATCH 33/76] does gravity use static IP? The changes to the gravity script check for a temp file containing the static address set during the install. I need to verify it works so I have to change the URL. I will change it back before merging with master. --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 0db4c02..79d6409 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -174,7 +174,7 @@ commands["Cleaning up the dashboard temp files"]="sudo rm /var/www/master.zip 2> commands["Creating a log file for the Pi-hole"]="sudo touch /var/log/pihole.log"; echoes+=( "Creating a log file for the Pi-hole" ) commands["chmodding the log file"]="sudo chmod 644 /var/log/pihole.log"; echoes+=( "chmodding the log file" ) commands["chowning the log file so stats can be displayed"]="sudo chown dnsmasq:root /var/log/pihole.log"; echoes+=( "chowning the log file so stats can be displayed" ) -commands["Initating sub-space transport of gravity"]="sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh"; echoes+=( "Initating sub-space transport of gravity" ) +commands["Initating sub-space transport of gravity"]="sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/installation/gravity.sh"; echoes+=( "Initating sub-space transport of gravity" ) commands["Initating sub-space transport of chronometer"]="sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh"; echoes+=( "Initating sub-space transport of chronometer" ) commands["chmodding gravity"]="sudo chmod 755 /usr/local/bin/gravity.sh"; echoes+=( "chmodding gravity" ) commands["chmodding the chronometer"]="sudo chmod 755 /usr/local/bin/chronometer.sh"; echoes+=( "chmodding the chronometer" ) From fb3fde3d19a1a5d44c1df0b480fb7209e239f2f9 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:45:00 -0600 Subject: [PATCH 34/76] testing with no reboot --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 79d6409..68fe9fc 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -179,7 +179,7 @@ commands["Initating sub-space transport of chronometer"]="sudo curl -o /usr/loca commands["chmodding gravity"]="sudo chmod 755 /usr/local/bin/gravity.sh"; echoes+=( "chmodding gravity" ) commands["chmodding the chronometer"]="sudo chmod 755 /usr/local/bin/chronometer.sh"; echoes+=( "chmodding the chronometer" ) commands["Entering the event horizion"]="sudo /usr/local/bin/gravity.sh"; echoes+=( "Entering the event horizion" ) -commands["Rebooting"]="sudo reboot"; echoes+=( "Rebooting" ) +commands["Rebooting"]="sudo service networking restart"; echoes+=( "Restarting networking..." ) # Everything in the parentheses is part of displaying the progress bar ( From eec0a31216287b1b81172e3287ddfa788f153678 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 7 Nov 2015 09:43:04 -0600 Subject: [PATCH 35/76] trying text output I commented out the whiptail dialog at the end in place of text so you can see what is going on. I also install the crontab. --- automated install/basic-install.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 68fe9fc..9fdd055 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -24,7 +24,6 @@ columns=$(stty -a | tr \; \\012 | egrep 'columns' | cut -d' ' -f3) r=$(( rows / 2 )) c=$(( columns / 2 )) -# Get the current network settings IPv4addr=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1") IPv4mask=$(ifconfig | awk -F':' '/inet addr/ && !/127.0.0.1/ {print $4}') IPv4gw=$(ip route show | awk '/default\ via/ {print $3}') @@ -164,6 +163,8 @@ commands["Backing up the lighttpd config file"]="sudo mv /etc/lighttpd/lighttpd. commands["Backing up the default Web page"]="sudo mv /var/www/html/index.lighttpd.html /var/www/html/index.lighttpd.orig"; echoes+=( "Backing up the default Web page" ) commands["Installing the dnsmasq config file"]="sudo curl -o /etc/dnsmasq.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dnsmasq.conf"; echoes+=( "Installing the dnsmasq config file" ) commands["Installing the lighttpd config file"]="sudo curl -o /etc/lighttpd/lighttpd.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/lighttpd.conf"; echoes+=( "Installing the lighttpd config file" ) +commands["Initating auto-pilot"]="sudo mv /etc/crontab /etc/crontab.orig"; echoes+=( "Initiating auto-pilot" ) +commands["Engaging auto-pilot"]="curl -o /etc/crontab https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/pihole.cron"; echoes+=( "Engaging auto-pilot" ) commands["Enabling PHP"]="sudo lighty-enable-mod fastcgi fastcgi-php"; echoes+=( "Enabling PHP" ) commands["Making a directory for the Web interface"]="sudo mkdir /var/www/html/pihole"; echoes+=( "Making a directory for the Web interface" ) commands["Installing a blank HTML page to take place of ads"]="sudo curl -o /var/www/html/pihole/index.html https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html"; echoes+=( "Installing a blank HTML page to take place of ads" ) @@ -182,7 +183,7 @@ commands["Entering the event horizion"]="sudo /usr/local/bin/gravity.sh"; echoes commands["Rebooting"]="sudo service networking restart"; echoes+=( "Restarting networking..." ) # Everything in the parentheses is part of displaying the progress bar -( +#( # Get total number of commands to be run from the array n=${#commands[*]}; # Set counter to increase every time a loop completes @@ -205,10 +206,10 @@ XXX EOF # Execute the command in the background (hidden from the user, not actually a background process) -${commands[${echoes[$k]}]} > $tmpLog 2>&1 +${commands[${echoes[$k]}]} | tee $tmpLog +clear done sudo mv $tmpLog $instalLogLoc$instalLogLoc # As the loop is progressing, the output is sent to whiptail to be displayed to the user -) | -whiptail --title "Opening your Pi-hole..." --gauge "Please wait..." $r $c 0 +#) |whiptail --title "Opening your Pi-hole..." --gauge "Please wait..." $r $c 0 From 8df376860cdd6ec5a5382aed5181a56cc0311a40 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 7 Nov 2015 12:07:50 -0600 Subject: [PATCH 36/76] simple function to run install commands The arrays were cool, but I think this just works easier. --- automated install/basic-install.sh | 113 +++++++++++------------------ 1 file changed, 43 insertions(+), 70 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9fdd055..938cc31 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -60,6 +60,44 @@ static routers=$IPv4gw static domain_name_servers=$IPv4gw" | sudo tee -a $dhcpcdFile >/dev/null } +installPihole() +{ +sudo apt-get update +sudo apt-get -y upgrade +sudo apt-get -y install dnsutils bc toilet +sudo apt-get -y install dnsmasq +sudo apt-get -y install lighttpd php5-common php5-cgi php5 +sudo mkdir /var/www/html +sudo chown www-data:www-data /var/www/html +sudo chmod 775 /var/www/html +sudo usermod -a -G www-data pi +sudo service dnsmasq stop +sudo service lighttpd stop +sudo mv /etc/dnsmasq.conf /etc/dnsmasq.conf.orig +sudo mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig +sudo mv /var/www/html/index.lighttpd.html /var/www/html/index.lighttpd.orig +sudo curl -o /etc/dnsmasq.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dnsmasq.conf +sudo curl -o /etc/lighttpd/lighttpd.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/lighttpd.conf +sudo mv /etc/crontab /etc/crontab.orig +sudo curl -o /etc/crontab https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/pihole.cron +sudo lighty-enable-mod fastcgi fastcgi-php +sudo mkdir /var/www/html/pihole +sudo curl -o /var/www/html/pihole/index.html https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html +sudo wget https://github.com/jacobsalmela/AdminLTE/archive/master.zip -O /var/www/master.zip +sudo unzip -o /var/www/master.zip -d /var/www/html/ +sudo mv /var/www/html/AdminLTE-master /var/www/html/admin +sudo rm /var/www/master.zip 2>/dev/null +sudo touch /var/log/pihole.log +sudo chmod 644 /var/log/pihole.log +sudo chown dnsmasq:root /var/log/pihole.log +sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/installation/gravity.sh +sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh +sudo chmod 755 /usr/local/bin/gravity.sh +sudo chmod 755 /usr/local/bin/chronometer.sh +sudo /usr/local/bin/gravity.sh +sudo service networking restart +} + ######## SCRIPT ############ # Just back up the original Pi-hole right away since it won't take long and it gets it out of the way backupLegacyPihole @@ -139,77 +177,12 @@ else done # End the if statement for DHCP vs. static fi + # Set the static address set_static_ip -# These are the commands to actually install the Pi-hole -# This is pretty ugly, but it works to present a nice front-end -# http://stackoverflow.com/questions/29161323/how-to-keep-associative-array-order-in-bash -# Maybe it would be better to just show the command output instead of the progress bar -declare -A commands; declare -a echoes; -commands["Updating"]="sudo apt-get update"; echoes+=( "Updating" ) -commands["Upgrading"]="sudo apt-get -y upgrade"; echoes+=( "Upgrading" ) -commands["Installing chronomoter tools"]="sudo apt-get -y install dnsutils bc toilet"; echoes+=( "Installing chronomoter tools" ) -commands["Installing a DNS server"]="sudo apt-get -y install dnsmasq"; echoes+=( "Installing a DNS server" ) -commands["Instaling a Web server and PHP"]="sudo apt-get -y install lighttpd php5-common php5-cgi php5"; echoes+=( "Instaling a Web server and PHP" ) -commands["Making an HTML folder"]="sudo mkdir /var/www/html"; echoes+=( "Making an HTML folder" ) -commands["chowning the Web server"]="sudo chown www-data:www-data /var/www/html"; echoes+=( "chowning the Web server" ) -commands["chmodding the Web server"]="sudo chmod 775 /var/www/html"; echoes+=( "chmodding the Web server" ) -commands["Giving pi access to the Web server"]="sudo usermod -a -G www-data pi"; echoes+=( "Giving pi access to the Web server" ) -commands["Stopping dnsmasq to modify it"]="sudo service dnsmasq stop"; echoes+=( "Stopping dnsmasq to modify it" ) -commands["Stopping lighttpd to modify it"]="sudo service lighttpd stop"; echoes+=( "Stopping lighttpd to modify it" ) -commands["Backing up the dnsmasq config file"]="sudo mv /etc/dnsmasq.conf /etc/dnsmasq.conf.orig"; echoes+=( "Backing up the dnsmasq config file" ) -commands["Backing up the lighttpd config file"]="sudo mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig"; echoes+=( "Backing up the lighttpd config file" ) -commands["Backing up the default Web page"]="sudo mv /var/www/html/index.lighttpd.html /var/www/html/index.lighttpd.orig"; echoes+=( "Backing up the default Web page" ) -commands["Installing the dnsmasq config file"]="sudo curl -o /etc/dnsmasq.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dnsmasq.conf"; echoes+=( "Installing the dnsmasq config file" ) -commands["Installing the lighttpd config file"]="sudo curl -o /etc/lighttpd/lighttpd.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/lighttpd.conf"; echoes+=( "Installing the lighttpd config file" ) -commands["Initating auto-pilot"]="sudo mv /etc/crontab /etc/crontab.orig"; echoes+=( "Initiating auto-pilot" ) -commands["Engaging auto-pilot"]="curl -o /etc/crontab https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/pihole.cron"; echoes+=( "Engaging auto-pilot" ) -commands["Enabling PHP"]="sudo lighty-enable-mod fastcgi fastcgi-php"; echoes+=( "Enabling PHP" ) -commands["Making a directory for the Web interface"]="sudo mkdir /var/www/html/pihole"; echoes+=( "Making a directory for the Web interface" ) -commands["Installing a blank HTML page to take place of ads"]="sudo curl -o /var/www/html/pihole/index.html https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/index.html"; echoes+=( "Installing a blank HTML page to take place of ads" ) -commands["Downloading the Pi-hole dashboard"]="sudo wget https://github.com/jacobsalmela/AdminLTE/archive/master.zip -O /var/www/master.zip"; echoes+=( "Downloading the Pi-hole dashboard" ) -commands["Unpacking the dashboard"]="sudo unzip -o /var/www/master.zip -d /var/www/html/"; echoes+=( "Unpacking the dashboard" ) -commands["Renaming the dashboard"]="sudo mv /var/www/html/AdminLTE-master /var/www/html/admin"; echoes+=( "Renaming the dashboard" ) -commands["Cleaning up the dashboard temp files"]="sudo rm /var/www/master.zip 2>/dev/null"; echoes+=( "Cleaning up the dashboard temp files" ) -commands["Creating a log file for the Pi-hole"]="sudo touch /var/log/pihole.log"; echoes+=( "Creating a log file for the Pi-hole" ) -commands["chmodding the log file"]="sudo chmod 644 /var/log/pihole.log"; echoes+=( "chmodding the log file" ) -commands["chowning the log file so stats can be displayed"]="sudo chown dnsmasq:root /var/log/pihole.log"; echoes+=( "chowning the log file so stats can be displayed" ) -commands["Initating sub-space transport of gravity"]="sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/installation/gravity.sh"; echoes+=( "Initating sub-space transport of gravity" ) -commands["Initating sub-space transport of chronometer"]="sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh"; echoes+=( "Initating sub-space transport of chronometer" ) -commands["chmodding gravity"]="sudo chmod 755 /usr/local/bin/gravity.sh"; echoes+=( "chmodding gravity" ) -commands["chmodding the chronometer"]="sudo chmod 755 /usr/local/bin/chronometer.sh"; echoes+=( "chmodding the chronometer" ) -commands["Entering the event horizion"]="sudo /usr/local/bin/gravity.sh"; echoes+=( "Entering the event horizion" ) -commands["Rebooting"]="sudo service networking restart"; echoes+=( "Restarting networking..." ) +# Install and log everything to a file +installPihole | tee $tmpLog -# Everything in the parentheses is part of displaying the progress bar -#( -# Get total number of commands to be run from the array -n=${#commands[*]}; -# Set counter to increase every time a loop completes -i=0 - -# For each item in the array -for k in "${!echoes[@]}" -do - -# Calculate the overall progress -percent=$(( 100*(++i)/n )) - -# Update dialog box using the value of each key in the array -# Show the percentage and the echo messages from the array -cat < Date: Sat, 7 Nov 2015 12:35:08 -0600 Subject: [PATCH 37/76] Closing dialog Also starting the Pi-hole and restarting networking. --- automated install/basic-install.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 938cc31..34a5d24 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -186,3 +186,12 @@ installPihole | tee $tmpLog # Move the log file into /etc/pihole for storage sudo mv $tmpLog $instalLogLoc + +whiptail --msgbox --backtitle "Make it so." --title "Installation Complete!" "Configure your devices to use the Pi-hole as their DNS server using this IP: $IPv4addr. + +The networking service will restart after you close this dialog. If you are using SSH, you may need to reconnect." $r $c + +# Start the services and restart networking +sudo service dnsmasq start +sudo service lighttpd start +sudo ifdown eth0;sudo ifup eth0 From 46b4000e75882e339f5b80b4f509aaaf32daba4a Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 7 Nov 2015 13:09:23 -0600 Subject: [PATCH 38/76] reboot works better --- automated install/basic-install.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 34a5d24..83361cb 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -189,9 +189,9 @@ sudo mv $tmpLog $instalLogLoc whiptail --msgbox --backtitle "Make it so." --title "Installation Complete!" "Configure your devices to use the Pi-hole as their DNS server using this IP: $IPv4addr. -The networking service will restart after you close this dialog. If you are using SSH, you may need to reconnect." $r $c +Your Pi will restart when you close this dialog. If you are using SSH, reconnect using the IP address above. -# Start the services and restart networking -sudo service dnsmasq start -sudo service lighttpd start -sudo ifdown eth0;sudo ifup eth0 +The install log is in /etc/phole." $r $c + + +sudo reboot From 76c217d9cc4ad1952d0f02f63452c4084a4b9384 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sun, 8 Nov 2015 17:21:02 -0600 Subject: [PATCH 39/76] moving to functions Also download the whitelist script. Ask for donations for IPv6 support. --- automated install/basic-install.sh | 214 ++++++++++++++++++----------- 1 file changed, 137 insertions(+), 77 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 83361cb..73e7f1f 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -29,9 +29,10 @@ IPv4mask=$(ifconfig | awk -F':' '/inet addr/ && !/127.0.0.1/ {print $4}') IPv4gw=$(ip route show | awk '/default\ via/ {print $3}') # IPv6 support to be added later -#IPv6addr=$(ip addr show | awk '/scope\ global/ && /ff:fe/ {print $2}' | cut -d'/' -f1) +#IPv6eui64=$(ip addr show | awk '/scope\ global/ && /ff:fe/ {print $2}' | cut -d'/' -f1) +#IPv6linkLocal=$(ip addr show | awk '/inet/ && /scope\ link/ && /fe80/ {print $2}' | cut -d'/' -f1) -ethernetDevice="eth0" +availableInterfaces=$(ip link show | awk -F' ' '/[0-9]: [a-z]/ {print $2}' | grep -v "lo" | cut -d':' -f1) dhcpcdFile=/etc/dhcpcd.conf ####### FUCNTIONS ########## @@ -51,7 +52,113 @@ else fi } -set_static_ip() +welcomeDialogs() +{ +# Display the welcome dialog +whiptail --msgbox --backtitle "Welcome" --title "Pi-hole automated installer" "This installer will transform your Raspberry Pi into a network-wide ad blocker!" $r $c + +# Explain the need for a static address +whiptail --msgbox --backtitle "Initating network interface" --title "Static IP Needed" "The Pi-hole is a SERVER so it needs a STATIC IP ADDRESS to function properly. + +In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." $r $c +} + +use4andor6() +{ +# Let use select IPv4 and/or IPv6 +cmd=(whiptail --separate-output --checklist "Select Protocols" $r $c 2) +options=(IPv4 "Block ads over IPv4" on + IPv6 "Block ads over IPv4" off) +choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty) +clear +for choice in $choices +do + case $choice in + IPv4) + echo "IPv4 selected." + useIPv4=true + ;; + IPv6) + echo "IPv6 selected." + useIPv6=true + ;; + esac +done +} + + +getStaticIPv4Settings() +{ +# Ask if the user wannts to use DHCP settings as their static IP +if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address? + + IP address: $IPv4addr + Subnet mask: $IPv4mask + Gateway: $IPv4gw" $r $c) then + # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. + whiptail --msgbox --backtitle "IP information" --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict. But in most cases the router is smart enough to not do that. + + If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want. + + It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." $r $c + # Nothing else to do since the variables are already set above +else + # Otherwise, we need to ask the user to input their desired settings. + # Start by getting the IPv4 address (pre-filling it with info gathered from DHCP) + # Start a loop to let the user enter their information with the chance to go back and edit it if necessary + until [[ $ipSettingsCorrect = True ]] + do + # Ask for the IPv4 address + IPv4addr=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" $r $c $IPv4addr 3>&1 1>&2 2>&3) + if [[ $? = 0 ]];then + echo "Your static IPv4 address: $IPv4addr" + # Ask for the subnet mask + IPv4mask=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 netmask" --inputbox "Enter your desired IPv4 subnet mask" $r $c $IPv4mask 3>&1 1>&2 2>&3) + if [[ $? = 0 ]];then + echo "Your static IPv4 netmask: $IPv4mask" + # Ask for the gateway + IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" $r $c $IPv4gw 3>&1 1>&2 2>&3) + if [[ $? = 0 ]];then + echo "Your static IPv4 gateway: $IPv4gw" + # Give the user a chance to review their settings before moving on + if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Are these settings correct? + IP address: $IPv4addr + Subnet mask: $IPv4mask + Gateway: $IPv4gw" $r $c)then + # If the settings are correct, then we need to set the piholeIP + # Saving it to a temporary file us to retrieve it later when we run the gravity.sh script + echo $IPv4addr > /tmp/piholeIP + # After that's done, the loop ends and we move on + ipSettingsCorrect=True + else + # If the settings are wrong, the loop continues + ipSettingsCorrect=False + fi + else + # Cancelling gateway settings window + ipSettingsCorrect=False + echo "User canceled." + exit + fi + else + # Cancelling subnet mask settings window + ipSettingsCorrect=False + echo "User canceled." + exit + fi + else + # Cancelling IPv4 settings window + ipSettingsCorrect=False + echo "User canceled." + exit + fi +done +# End the if statement for DHCP vs. static +fi +} + + +setStaticIPv4() { # Append these lines to /etc/dhcpcd.conf to enable a static IP echo "interface $ethernetDevice @@ -76,6 +183,7 @@ sudo service lighttpd stop sudo mv /etc/dnsmasq.conf /etc/dnsmasq.conf.orig sudo mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig sudo mv /var/www/html/index.lighttpd.html /var/www/html/index.lighttpd.orig +sudo mv /etc/crontab /etc/crontab.orig sudo curl -o /etc/dnsmasq.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dnsmasq.conf sudo curl -o /etc/lighttpd/lighttpd.conf https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/lighttpd.conf sudo mv /etc/crontab /etc/crontab.orig @@ -92,94 +200,46 @@ sudo chmod 644 /var/log/pihole.log sudo chown dnsmasq:root /var/log/pihole.log sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/installation/gravity.sh sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh +sudo curl -o /usr/local/bin/whitelist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/whitelist.sh sudo chmod 755 /usr/local/bin/gravity.sh sudo chmod 755 /usr/local/bin/chronometer.sh +sudo chmod 755 /usr/local/bin/whitelist.sh sudo /usr/local/bin/gravity.sh sudo service networking restart } ######## SCRIPT ############ +# Start the installer +welcomeDialogs + # Just back up the original Pi-hole right away since it won't take long and it gets it out of the way backupLegacyPihole -# Display the welcome dialog -whiptail --msgbox --backtitle "Welcome" --title "Pi-hole automated installer" "This installer will transform your Raspberry Pi into a network-wide ad blocker!" $r $c +# Let the user decide if they want to block ads over IPv4 and/or IPv6 +use4andor6 -# Explain the need for a static address -whiptail --msgbox --backtitle "Initating network interface" --title "Static IP Needed" "The Pi-hole is a SERVER so it needs a STATIC IP ADDRESS to function properly. - -In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." $r $c - -# Ask if the user wannts to use DHCP settings as their static IP -if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address? - - IP address: $IPv4addr - Subnet mask: $IPv4mask - Gateway: $IPv4gw" $r $c) then - # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. - whiptail --msgbox --backtitle "IP information" --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict. But in most cases the router is smart enough to not do that. - -If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want. - -It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." $r $c - # Nothing else to do since the variables are already set above +# Decide is IPv4 will be used +if [[ "$useIPv4" = true ]];then + echo "Using IPv4" + getStaticIPv4Settings + setStaticIPv4 else - # Otherwise, we need to ask the user to input their desired settings. - # Start by getting the IPv4 address (pre-filling it with info gathered from DHCP) - # Start a loop to let the user enter their information with the chance to go back and edit it if necessary - until [[ $ipSettingsCorrect = True ]] - do - # Ask for the IPv4 address - IPv4addr=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" $r $c $IPv4addr 3>&1 1>&2 2>&3) - if [[ $? = 0 ]];then - echo "Your static IPv4 address: $IPv4addr" - # Ask for the subnet mask - IPv4mask=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 netmask" --inputbox "Enter your desired IPv4 subnet mask" $r $c $IPv4mask 3>&1 1>&2 2>&3) - if [[ $? = 0 ]];then - echo "Your static IPv4 netmask: $IPv4mask" - # Ask for the gateway - IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" $r $c $IPv4gw 3>&1 1>&2 2>&3) - if [[ $? = 0 ]];then - echo "Your static IPv4 gateway: $IPv4gw" - # Give the user a chance to review their settings before moving on - if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Are these settings correct? - IP address: $IPv4addr - Subnet mask: $IPv4mask - Gateway: $IPv4gw" $r $c)then - # If the settings are correct, then we need to set the piholeIP - # Saving it to a temporary file us to retrieve it later when we run the gravity.sh script - echo $IPv4addr > /tmp/piholeIP - # After that's done, the loop ends and we move on - ipSettingsCorrect=True - - else - # If the settings are wrong, the loop continues - ipSettingsCorrect=False - fi - else - # Cancelling gateway settings window - ipSettingsCorrect=False - echo "User canceled." - exit - fi - else - # Cancelling subnet mask settings window - ipSettingsCorrect=False - echo "User canceled." - exit - fi - else - # Cancelling IPv4 settings window - ipSettingsCorrect=False - echo "User canceled." - exit - fi - done -# End the if statement for DHCP vs. static + echo "IPv4 will NOT be used." +fi + +# Decide is IPv6 will be used +if [[ "$useIPv6" = true ]];then + whiptail --msgbox --backtitle "Coming soon..." --title "IPv6 not yet supported" "I need your help. Consider donating at: + + http://pi-hole.net/donate" $r $c + echo "Using IPv6" +else + whiptail --msgbox --backtitle "Coming soon..." --title "IPv6 not yet supported" "I need your help. Consider donating at: + + http://pi-hole.net/donate" $r $c + echo "IPv6 will NOT be used." fi -# Set the static address -set_static_ip # Install and log everything to a file installPihole | tee $tmpLog From ad91eb44fbac32f0e1efa41b42e669168293d076 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Mon, 9 Nov 2015 18:33:32 -0600 Subject: [PATCH 40/76] choose an interface also fixing ipv6 logic --- automated install/basic-install.sh | 60 +++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 73e7f1f..d171bf8 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -63,6 +63,40 @@ whiptail --msgbox --backtitle "Initating network interface" --title "Static IP N In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." $r $c } +chooseInterface() +{ +# Turn the available interfaces into an array so it can be used with a whiptail dialog +interfacesArray=() +while read -r line +do +interfacesArray+=("$line" "available" "OFF") +done <<< "$availableInterfaces" + +# Find out how many interfaces are available to choose from +interfaceCount=$(echo "$availableInterfaces" | wc -l) +chooseInterfaceCmd=(whiptail --separate-output --radiolist "Choose An Interface" $r $c $interfaceCount) +chooseInterfaceOptions=$("${chooseInterfaceCmd[@]}" "${interfacesArray[@]}" 2>&1 >/dev/tty) +for desiredInterface in $chooseInterfaceOptions +do + piholeInterface=$desiredInterface + # case $chooseInterfaceOptions in + # eth0) + # echo "Ethernet" + # useIPv4=true + # ;; + # IPv6) + # echo "IPv6 selected." + # useIPv6=true + # ;; + # esac + # chosenInterface="$desiredInterface" + # echo "$desiredInterface" + echo "Chosen: $piholeInterface" +done + + +} + use4andor6() { # Let use select IPv4 and/or IPv6 @@ -79,13 +113,18 @@ do useIPv4=true ;; IPv6) - echo "IPv6 selected." + echo "IPv6 selected." useIPv6=true ;; esac done } +useIPv6dialog() +{ +whiptail --msgbox --backtitle "Coming soon..." --title "IPv6 not yet supported" "I need your help for IPv6. Consider donating at: http://pi-hole.net/donate" $r $c +} + getStaticIPv4Settings() { @@ -215,6 +254,8 @@ welcomeDialogs # Just back up the original Pi-hole right away since it won't take long and it gets it out of the way backupLegacyPihole +chooseInterface + # Let the user decide if they want to block ads over IPv4 and/or IPv6 use4andor6 @@ -224,23 +265,24 @@ if [[ "$useIPv4" = true ]];then getStaticIPv4Settings setStaticIPv4 else + useIPv4=false echo "IPv4 will NOT be used." fi # Decide is IPv6 will be used if [[ "$useIPv6" = true ]];then - whiptail --msgbox --backtitle "Coming soon..." --title "IPv6 not yet supported" "I need your help. Consider donating at: - - http://pi-hole.net/donate" $r $c - echo "Using IPv6" + # If only IPv6 is selected, exit because it is not supported yet + if [[ "$useIPv6" = true ]] && [[ "$useIPv4" = false ]];then + useIPv6dialog + exit + else + useIPv6dialog + fi else - whiptail --msgbox --backtitle "Coming soon..." --title "IPv6 not yet supported" "I need your help. Consider donating at: - - http://pi-hole.net/donate" $r $c + useIPv6=false echo "IPv6 will NOT be used." fi - # Install and log everything to a file installPihole | tee $tmpLog From 0936e394d48f12fb0ee5cf19e33467c9c2697785 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 10 Nov 2015 08:12:24 -0600 Subject: [PATCH 41/76] reboot only if new IP was set --- automated install/basic-install.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d171bf8..2963dfc 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -200,7 +200,7 @@ fi setStaticIPv4() { # Append these lines to /etc/dhcpcd.conf to enable a static IP -echo "interface $ethernetDevice +echo "interface $piholeInterface static ip_address=$IPv4addr/24 static routers=$IPv4gw static domain_name_servers=$IPv4gw" | sudo tee -a $dhcpcdFile >/dev/null @@ -295,5 +295,12 @@ Your Pi will restart when you close this dialog. If you are using SSH, reconnec The install log is in /etc/phole." $r $c - -sudo reboot +# If the current IP address equals the desired address, no change is needed +if [[ $IPv4addr = "$(cat /tmp/piholeIP)" ]];then + # So just start the services + echo "sudo service dnsmasq start" + echo "sudo service lighttpd start" +else + # Restart to apply the new static IP address + echo "sudo reboot" +fi From 814873c6430054211a098c88e0b9b05230d2e38b Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 10 Nov 2015 17:28:05 -0600 Subject: [PATCH 42/76] reboot logic --- automated install/basic-install.sh | 41 +++++++++++------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2963dfc..a5cef6b 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -69,7 +69,7 @@ chooseInterface() interfacesArray=() while read -r line do -interfacesArray+=("$line" "available" "OFF") +interfacesArray+=("$line" "available" "ON") done <<< "$availableInterfaces" # Find out how many interfaces are available to choose from @@ -79,22 +79,8 @@ chooseInterfaceOptions=$("${chooseInterfaceCmd[@]}" "${interfacesArray[@]}" 2>&1 for desiredInterface in $chooseInterfaceOptions do piholeInterface=$desiredInterface - # case $chooseInterfaceOptions in - # eth0) - # echo "Ethernet" - # useIPv4=true - # ;; - # IPv6) - # echo "IPv6 selected." - # useIPv6=true - # ;; - # esac - # chosenInterface="$desiredInterface" - # echo "$desiredInterface" - echo "Chosen: $piholeInterface" + echo "Using interface: $piholeInterface" done - - } use4andor6() @@ -125,7 +111,6 @@ useIPv6dialog() whiptail --msgbox --backtitle "Coming soon..." --title "IPv6 not yet supported" "I need your help for IPv6. Consider donating at: http://pi-hole.net/donate" $r $c } - getStaticIPv4Settings() { # Ask if the user wannts to use DHCP settings as their static IP @@ -142,6 +127,8 @@ if (whiptail --backtitle "Calibrating network interface" --title "Static IP Addr It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." $r $c # Nothing else to do since the variables are already set above else + # Since a custom address will be used, restart at the end of the script to apply the new changes + rebootNeeded=true # Otherwise, we need to ask the user to input their desired settings. # Start by getting the IPv4 address (pre-filling it with info gathered from DHCP) # Start a loop to let the user enter their information with the chance to go back and edit it if necessary @@ -240,6 +227,7 @@ sudo chown dnsmasq:root /var/log/pihole.log sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/installation/gravity.sh sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh sudo curl -o /usr/local/bin/whitelist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/whitelist.sh +sudo curl -o /usr/local/bin/piholeLogFlush.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/piholeLogFlush.sh sudo chmod 755 /usr/local/bin/gravity.sh sudo chmod 755 /usr/local/bin/chronometer.sh sudo chmod 755 /usr/local/bin/whitelist.sh @@ -254,6 +242,7 @@ welcomeDialogs # Just back up the original Pi-hole right away since it won't take long and it gets it out of the way backupLegacyPihole +# Find interfaces and let the user choose one chooseInterface # Let the user decide if they want to block ads over IPv4 and/or IPv6 @@ -280,7 +269,7 @@ if [[ "$useIPv6" = true ]];then fi else useIPv6=false - echo "IPv6 will NOT be used." + echo "IPv6 will NOT be used. Consider a donation at pi-hole.net/donate" fi # Install and log everything to a file @@ -291,16 +280,16 @@ sudo mv $tmpLog $instalLogLoc whiptail --msgbox --backtitle "Make it so." --title "Installation Complete!" "Configure your devices to use the Pi-hole as their DNS server using this IP: $IPv4addr. -Your Pi will restart when you close this dialog. If you are using SSH, reconnect using the IP address above. +If you didn't use DHCP settings as your new static address, the Pi will restart after this dialog. If you are using SSH, you may need to reconnect using the IP address above. The install log is in /etc/phole." $r $c -# If the current IP address equals the desired address, no change is needed -if [[ $IPv4addr = "$(cat /tmp/piholeIP)" ]];then - # So just start the services - echo "sudo service dnsmasq start" - echo "sudo service lighttpd start" -else +# If a custom address was set, restart +if [[ "$rebootNeeded" = true ]];then # Restart to apply the new static IP address - echo "sudo reboot" + sudo reboot +else + # If not, just start the services since the address will stay the same + sudo service dnsmasq start + sudo service lighttpd start fi From dee3df4e1195c1ae45c61240640e14bc04548845 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 10 Nov 2015 17:28:53 -0600 Subject: [PATCH 43/76] download master gravity --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a5cef6b..85f79e5 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -224,7 +224,7 @@ sudo rm /var/www/master.zip 2>/dev/null sudo touch /var/log/pihole.log sudo chmod 644 /var/log/pihole.log sudo chown dnsmasq:root /var/log/pihole.log -sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/installation/gravity.sh +sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh sudo curl -o /usr/local/bin/whitelist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/whitelist.sh sudo curl -o /usr/local/bin/piholeLogFlush.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/piholeLogFlush.sh From 0cb20fc080afdc95dcf6484cd66d2bfeb8c329e3 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 10 Nov 2015 17:34:25 -0600 Subject: [PATCH 44/76] typo --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 85f79e5..2a2b88d 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -282,7 +282,7 @@ whiptail --msgbox --backtitle "Make it so." --title "Installation Complete!" "Co If you didn't use DHCP settings as your new static address, the Pi will restart after this dialog. If you are using SSH, you may need to reconnect using the IP address above. -The install log is in /etc/phole." $r $c +The install log is in /etc/pihole." $r $c # If a custom address was set, restart if [[ "$rebootNeeded" = true ]];then From d0d165ad165e1ba873df73bd97691de7c9bf7afa Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 10 Nov 2015 21:27:53 -0600 Subject: [PATCH 45/76] chmod the log flusher --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2a2b88d..f0a7d13 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -231,8 +231,8 @@ sudo curl -o /usr/local/bin/piholeLogFlush.sh https://raw.githubusercontent.com/ sudo chmod 755 /usr/local/bin/gravity.sh sudo chmod 755 /usr/local/bin/chronometer.sh sudo chmod 755 /usr/local/bin/whitelist.sh +sudo chmod 755 /usr/local/bin/piholeLogFlush.sh sudo /usr/local/bin/gravity.sh -sudo service networking restart } ######## SCRIPT ############ From 689426180a7ed7540bec4e529e4db206903f40ee Mon Sep 17 00:00:00 2001 From: dschaper Date: Sun, 15 Nov 2015 13:59:51 +0000 Subject: [PATCH 46/76] Resolve merge conflicts --- gravity.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 9379319..c6405c7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,11 +1,18 @@ #!/usr/bin/env bash # http://pi-hole.net # Compiles a list of ad-serving domains by downloading them from multiple sources - -# This script should only be run after you have a static IP address set on the Pi -piholeIP=$(hostname -I) +piholeIPfile=/tmp/piholeIP +if [[ -f $piholeIPfile ]];then + # If the file exists, it means it was exported from the installation script and we should use that value instead of detecting it in this script + piholeIP=$(cat $piholeIPfile) + rm $piholeIPfile +else + # Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script + piholeIP=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1") +fi # Ad-list sources--one per line in single quotes +# The mahakala source is commented out due to many users having issues with it blocking legitimate domains. Uncomment at your own risk sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' #'http://adblock.mahakala.is/' @@ -129,8 +136,7 @@ function gravity_advanced() { # Format domain list as "192.168.x.x domain.com" echo "** Formatting domains into a HOSTS file..." - awk '{print "'"$piholeIP"'" $1}' $origin/$eventHorizon > $origin/$accretionDisc - + cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $origin/$accretionDisc # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq) From b005da395f21ffdabd3459867b28621a0d778039 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sun, 15 Nov 2015 20:54:15 -0600 Subject: [PATCH 47/76] Updated install instructions Also a screenshot of the dialog-based installer --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 09b8852..3e02f0d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # Automated Install 1. Install Raspbian -2. Set a **static** IP address -3. Run the command below +2. Run the command below -```curl -L install.pi-hole.net | bash``` +### ```curl -L install.pi-hole.net | bash``` + +![Pi-hole automated installation](http://i.imgur.com/Un7lBlj.png) Once installed, **configure any device to use the Raspberry Pi as its DNS server and the ads will be blocked**. You can also configure your router's DHCP options to assign the Pi as clients DNS server so they do not need to do it manually. From 1578a28363ec2bef64556b46427e04ea21b6f7bf Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Thu, 19 Nov 2015 19:36:34 -0600 Subject: [PATCH 48/76] pihole in the cloud link --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3e02f0d..258168c 100644 --- a/README.md +++ b/README.md @@ -63,5 +63,6 @@ This script will work for other UNIX-like systems with some slight **modificatio ### Examples Of The Pi-hole On Other Operating Systems - [Sky-Hole](http://dlaa.me/blog/post/skyhole) +- [Pi-hole in the Cloud!](http://blog.codybunch.com/2015/07/28/Pi-Hole-in-the-cloud/) [![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif "AdminLTE Presentation")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY "Donate") From d0e9e90ebeaf4638231d78470dd638d38542a0ac Mon Sep 17 00:00:00 2001 From: Cole Turner Date: Fri, 20 Nov 2015 22:53:43 -0500 Subject: [PATCH 49/76] pinholeIP should only use one IP Updated piholeIP because if there are multiple network adapters, the gravity list will populate incorrectly with both IP addresses. Only one address should be used. --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index c6405c7..c2d1ff9 100755 --- a/gravity.sh +++ b/gravity.sh @@ -8,7 +8,7 @@ if [[ -f $piholeIPfile ]];then rm $piholeIPfile else # Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script - piholeIP=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1") + piholeIP=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1" | (head -n1)) fi # Ad-list sources--one per line in single quotes From 4e50a03dfc6169bf691fae25f61e9fc3365272b3 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 21 Nov 2015 13:29:03 -0600 Subject: [PATCH 50/76] uninstall script This should remove the Pi-hole. --- automated install/uninstall.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 automated install/uninstall.sh diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh new file mode 100644 index 0000000..b0dd336 --- /dev/null +++ b/automated install/uninstall.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Completely uninstalls the Pi-hole + +######### SCRIPT ########### +sudo apt-get -y remove --purge dnsutils bc toilet +sudo apt-get -y remove --purge dnsmasq +sudo apt-get -y remove --purge lighttpd php5-common php5-cgi php5 +sudo rm -rf /var/www/html +sudo rm /etc/dnsmasq.conf /etc/dnsmasq.conf.orig +sudo rm /etc/crontab +sudo mv /etc/crontab.orig /etc/crontab +sudo rm /etc/dnsmasq.conf +sudo rm -rf /etc/lighttpd/ +sudo rm /var/log/pihole.log +sudo rm /usr/local/bin/gravity.sh +sudo rm /usr/local/bin/chronometer.sh +sudo rm /usr/local/bin/whitelist.sh +sudo rm /usr/local/bin/piholeLogFlush.sh +sudo rm -rf /etc/pihole/ From f02c37ec07a33c60588823c0420a9e9888d4af7f Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Sun, 22 Nov 2015 23:49:38 -0800 Subject: [PATCH 51/76] Start code refactor, eliminate some repetitive code Begin code refactor prepatation, prevent DRY where possible. --- gravity.sh | 174 ++++++++++++++++++++--------------------------------- 1 file changed, 66 insertions(+), 108 deletions(-) diff --git a/gravity.sh b/gravity.sh index 4c191c3..62a2cf2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,11 +1,18 @@ -#!/bin/bash +#!/usr/bin/env bash # http://pi-hole.net # Compiles a list of ad-serving domains by downloading them from multiple sources - -# This script should only be run after you have a static IP address set on the Pi -piholeIP=$(hostname -I) +piholeIPfile=/tmp/piholeIP +if [[ -f $piholeIPfile ]];then + # If the file exists, it means it was exported from the installation script and we should use that value instead of detecting it in this script + piholeIP=$(cat $piholeIPfile) + rm $piholeIPfile +else + # Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script + piholeIP=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1" | (head -n1)) +fi # Ad-list sources--one per line in single quotes +# The mahakala source is commented out due to many users having issues with it blocking legitimate domains. Uncomment at your own risk sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' #'http://adblock.mahakala.is/' @@ -16,139 +23,90 @@ sources=('https://adaway.org/hosts.txt' 'http://winhelp2002.mvps.org/hosts.txt') # Variables for various stages of downloading and formatting the list -adList=/etc/pihole/gravity.list -origin=/etc/pihole -piholeDir=/etc/pihole -justDomainsExtension=domains -matter=pihole.0.matter.txt -andLight=pihole.1.andLight.txt -supernova=pihole.2.supernova.txt -eventHorizon=pihole.3.eventHorizon.txt -accretionDisc=pihole.4.accretionDisc.txt -eyeOfTheNeedle=pihole.5.wormhole.txt +basename=pihole +piholeDir=/etc/$basename +adList=$piholeDir/gravity.list blacklist=$piholeDir/blacklist.txt whitelist=$piholeDir/whitelist.txt -latentWhitelist=$origin/latentWhitelist.txt +latentWhitelist=$piholeDir/latentWhitelist.txt +justDomainsExtension=domains +matter=$basename.0.matter.txt +andLight=$basename.1.andLight.txt +supernova=$basename.2.supernova.txt +eventHorizon=$basename.3.eventHorizon.txt +accretionDisc=$basename.4.accretionDisc.txt +eyeOfTheNeedle=$basename.5.wormhole.txt # After setting defaults, check if there's local overrides if [[ -r $piholeDir/pihole.conf ]];then echo "** Local calibration requested..." - . $piholeDir/pihole.conf + . $piholeDir/pihole.conf fi echo "** Neutrino emissions detected..." # Create the pihole resource directory if it doesn't exist. Future files will be stored here if [[ -d $piholeDir ]];then - : + : else - echo "** Creating pihole directory..." - sudo mkdir $piholeDir + echo "** Creating pihole directory..." + sudo mkdir $piholeDir fi # Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and blank lines for ((i = 0; i < "${#sources[@]}"; i++)) do - url=${sources[$i]} - # Get just the domain from the URL - domain=$(echo "$url" | cut -d'/' -f3) + url=${sources[$i]} + # Get just the domain from the URL + domain=$(echo "$url" | cut -d'/' -f3) - # Save the file as list.#.domain - saveLocation=$origin/list.$i.$domain.$justDomainsExtension + # Save the file as list.#.domain + saveLocation=$piholeDir/list.$i.$domain.$justDomainsExtension - agent="Mozilla/10.0" + agent="Mozilla/10.0" - echo -n "Getting $domain list... " + echo -n "Getting $domain list... " - # Use a case statement to download lists that need special cURL commands - # to complete properly and reset the user agent when required - case "$domain" in - "adblock.mahakala.is") - agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' - cmd="curl -e http://forum.xda-developers.com/" - ;; + # Use a case statement to download lists that need special cURL commands + # to complete properly and reset the user agent when required + case "$domain" in + "adblock.mahakala.is") + agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' + cmd_ext="-e http://forum.xda-developers.com/" + ;; - "pgl.yoyo.org") - cmd="curl -d mimetype=plaintext -d hostformat=hosts" - ;; + "pgl.yoyo.org") + echo "** $numberOf domains being pulled in by gravity..." - # Default is a simple curl request - *) cmd="curl" - esac + # Remove carriage returns and preceding whitespace + # not really needed anymore? + cp $piholeDir/$andLight $piholeDir/$supernova - # tmp file, so we don't have to store the (long!) lists in RAM - patternBuffer=$(mktemp) - heisenbergCompensator="" - if [[ -r $saveLocation ]]; then - heisenbergCompensator="-z $saveLocation" - fi - CMD="$cmd -s $heisenbergCompensator -A '$agent' $url > $patternBuffer" - $cmd -s $heisenbergCompensator -A "$agent" $url > $patternBuffer + # Sort and remove duplicates + sort -u $piholeDir/$supernova > $piholeDir/$eventHorizon + numberOf=$(wc -l < $piholeDir/$eventHorizon) + echo "** $numberOf unique domains trapped in the event horizon." - - if [[ -s "$patternBuffer" ]];then - # Remove comments and print only the domain name - # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious - # This helps with that and makes it easier to read - # It also helps with debugging so each stage of the script can be researched more in depth - awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $patternBuffer | \ - sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $saveLocation - echo "Done." - else - echo "Skipping pattern because transporter logic detected no changes..." - fi - - # Cleanup - rm -f $patternBuffer -done - -# Find all files with the .domains extension and compile them into one file and remove CRs -echo "** Aggregating list of domains..." -find $origin/ -type f -name "*.$justDomainsExtension" -exec cat {} \; | tr -d '\r' > $origin/$matter - -# Append blacklist entries if they exist -if [[ -r $blacklist ]];then - numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) - echo "** Blacklisting $numberOf domain(s)..." - cat $blacklist >> $origin/$matter -fi - -########################### -function gravity_advanced() { - - numberOf=$(wc -l < $origin/$andLight) - echo "** $numberOf domains being pulled in by gravity..." - - # Remove carriage returns and preceding whitespace - # not really needed anymore? - cp $origin/$andLight $origin/$supernova - - # Sort and remove duplicates - sort -u $origin/$supernova > $origin/$eventHorizon - numberOf=$(wc -l < $origin/$eventHorizon) - echo "** $numberOf unique domains trapped in the event horizon." - - # Format domain list as "192.168.x.x domain.com" - echo "** Formatting domains into a HOSTS file..." - awk '{print "'"$piholeIP"'" $1}' $origin/$eventHorizon > $origin/$accretionDisc - - # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it - sudo cp $origin/$accretionDisc $adList - kill -HUP $(pidof dnsmasq) + # Format domain list as "192.168.x.x domain.com" + echo "** Formatting domains into a HOSTS file..." + cat $piholeDir/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $piholeDir/$accretionDisc + # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it + sudo cp $piholeDir/$accretionDisc $adList + kill -HUP $(pidof dnsmasq) } # Whitelist (if applicable) then remove duplicates and format for dnsmasq if [[ -r $whitelist ]];then - # Remove whitelist entries - numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) - plural=; [[ "$numberOf" != "1" ]] && plural=s - echo "** Whitelisting $numberOf domain${plural}..." + # Remove whitelist entries + numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) + plural=; [[ "$numberOf" != "1" ]] && plural=s + echo "** Whitelisting $numberOf domain${plural}..." - # Append a "$" to the end, prepend a "^" to the beginning, and - # replace "." with "\." of each line to turn each entry into a - # regexp so it can be parsed out with grep -x - awk -F '[# \t]' 'NF>0&&$1!="" {print "^"$1"$"}' $whitelist | sed 's/\./\\./g' > $latentWhitelist + # Append a "$" to the end, prepend a "^" to the beginning, and + # replace "." with "\." of each line to turn each entry into a + # regexp so it can be parsed out with grep -x + awk -F '[# \t]' 'NF>0&&$1!="" {print "^"$1"$"}' $whitelist | sed 's/\./\\./g' > $latentWhitelist else - rm $latentWhitelist + rm $latentWhitelist fi # Prevent our sources from being pulled into the hole @@ -156,10 +114,10 @@ plural=; [[ "${#sources[@]}" != "1" ]] && plural=s echo "** Whitelisting ${#sources[@]} ad list source${plural}..." for url in ${sources[@]} do - echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist + echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist done # Remove whitelist entries from deduped list -grep -vxf $latentWhitelist $origin/$matter > $origin/$andLight +grep -vxf $latentWhitelist $piholeDir/$matter > $piholeDir/$andLight gravity_advanced From 668d14e32d6b1f506d3ff34c41a561aa34855bfc Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 23 Nov 2015 00:36:01 -0800 Subject: [PATCH 52/76] Add functions Start to add functions for code reuse --- gravity.sh | 53 +++++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/gravity.sh b/gravity.sh index 62a2cf2..eb4c150 100755 --- a/gravity.sh +++ b/gravity.sh @@ -46,36 +46,35 @@ echo "** Neutrino emissions detected..." # Create the pihole resource directory if it doesn't exist. Future files will be stored here if [[ -d $piholeDir ]];then - : + # Temporary hack to allow non-root access to pihole directory + # Will update later, needed for existing installs, new installs should + # create this directory as non-root + sudo chmod 777 $piholeDir + find "$piholeDir" -type f -exec sudo chmod 666 {} \; else echo "** Creating pihole directory..." - sudo mkdir $piholeDir + mkdir $piholeDir fi -# Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and blank lines -for ((i = 0; i < "${#sources[@]}"; i++)) -do - url=${sources[$i]} - # Get just the domain from the URL - domain=$(echo "$url" | cut -d'/' -f3) +########################### +function gravity_patterncheck() { - # Save the file as list.#.domain - saveLocation=$piholeDir/list.$i.$domain.$justDomainsExtension + patternBuffer=$1 - agent="Mozilla/10.0" - - echo -n "Getting $domain list... " - - # Use a case statement to download lists that need special cURL commands - # to complete properly and reset the user agent when required - case "$domain" in - "adblock.mahakala.is") - agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' - cmd_ext="-e http://forum.xda-developers.com/" - ;; - - "pgl.yoyo.org") - echo "** $numberOf domains being pulled in by gravity..." + # check if the patternbuffer is a non-zero length file + if [[ -s "$patternBuffer" ]];then + # Remove comments and print only the domain name + # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious + # This helps with that and makes it easier to read + # It also helps with debugging so each stage of the script can be researched more in depth + awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $patternBuffer | \ + sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $saveLocation + echo "Done." + else + # curl didn't download any host files, probably because of the date check + echo "Transporter logic detected no changes, pattern skipped..." + fi +} # Remove carriage returns and preceding whitespace # not really needed anymore? @@ -90,10 +89,12 @@ do echo "** Formatting domains into a HOSTS file..." cat $piholeDir/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $piholeDir/$accretionDisc # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it - sudo cp $piholeDir/$accretionDisc $adList - kill -HUP $(pidof dnsmasq) + cp $piholeDir/$accretionDisc $adList + sudo kill -HUP $(pidof dnsmasq) } +gravity_spinup + # Whitelist (if applicable) then remove duplicates and format for dnsmasq if [[ -r $whitelist ]];then # Remove whitelist entries From a1e609d0059b1db0212494621cad0f824bb52315 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 23 Nov 2015 01:16:00 -0800 Subject: [PATCH 53/76] Ugly hack permissions Temporary hack for /etc/pihole directory and file permissions to pull out sudo requirement. (Irony is that is requires sudo.) This will be fixed later, but for now it's needed for existing installs so they don't break. --- gravity.sh | 58 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/gravity.sh b/gravity.sh index eb4c150..f3b8f82 100755 --- a/gravity.sh +++ b/gravity.sh @@ -57,10 +57,10 @@ else fi ########################### -function gravity_patterncheck() { - +# patternCheck - check to see if curl downloaded any new files, and then process those +# files so they are in host format. +function gravity_patternCheck() { patternBuffer=$1 - # check if the patternbuffer is a non-zero length file if [[ -s "$patternBuffer" ]];then # Remove comments and print only the domain name @@ -76,6 +76,32 @@ function gravity_patterncheck() { fi } + # Use a case statement to download lists that need special cURL commands + # to complete properly and reset the user agent when required + case "$domain" in + "adblock.mahakala.is") + agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' + cmd_ext="-e http://forum.xda-developers.com/" + ;; + + "pgl.yoyo.org") + cmd_ext="-d mimetype=plaintext -d hostformat=hosts" + ;; + + # Default is a simple request + *) cmd_ext="" + esac + gravity_transport $url $cmd_ext $agent +done + # Compress files to singularity and white/blacklist + gravity_Schwarzchild +} + +function gravity_advanced() { + + numberOf=$(wc -l < $piholeDir/$andLight) + echo "** $numberOf domains being pulled in by gravity..." + # Remove carriage returns and preceding whitespace # not really needed anymore? cp $piholeDir/$andLight $piholeDir/$supernova @@ -95,30 +121,4 @@ function gravity_patterncheck() { gravity_spinup -# Whitelist (if applicable) then remove duplicates and format for dnsmasq -if [[ -r $whitelist ]];then - # Remove whitelist entries - numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) - plural=; [[ "$numberOf" != "1" ]] && plural=s - echo "** Whitelisting $numberOf domain${plural}..." - - # Append a "$" to the end, prepend a "^" to the beginning, and - # replace "." with "\." of each line to turn each entry into a - # regexp so it can be parsed out with grep -x - awk -F '[# \t]' 'NF>0&&$1!="" {print "^"$1"$"}' $whitelist | sed 's/\./\\./g' > $latentWhitelist -else - rm $latentWhitelist -fi - -# Prevent our sources from being pulled into the hole -plural=; [[ "${#sources[@]}" != "1" ]] && plural=s -echo "** Whitelisting ${#sources[@]} ad list source${plural}..." -for url in ${sources[@]} -do - echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist -done - -# Remove whitelist entries from deduped list -grep -vxf $latentWhitelist $piholeDir/$matter > $piholeDir/$andLight - gravity_advanced From fe46afd08d2058899f34722e5113b67f8b8a320f Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 23 Nov 2015 01:47:24 -0800 Subject: [PATCH 54/76] More functions More refactoring --- gravity.sh | 73 +++++++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/gravity.sh b/gravity.sh index f3b8f82..9cb017d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -42,6 +42,10 @@ if [[ -r $piholeDir/pihole.conf ]];then echo "** Local calibration requested..." . $piholeDir/pihole.conf fi +########################### +# collapse - begin formation of pihole +function gravity_collapse() { + echo "** Neutrino emissions detected..." # Create the pihole resource directory if it doesn't exist. Future files will be stored here @@ -55,46 +59,42 @@ else echo "** Creating pihole directory..." mkdir $piholeDir fi - -########################### -# patternCheck - check to see if curl downloaded any new files, and then process those -# files so they are in host format. -function gravity_patternCheck() { - patternBuffer=$1 - # check if the patternbuffer is a non-zero length file - if [[ -s "$patternBuffer" ]];then - # Remove comments and print only the domain name - # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious - # This helps with that and makes it easier to read - # It also helps with debugging so each stage of the script can be researched more in depth - awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $patternBuffer | \ - sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $saveLocation - echo "Done." - else - # curl didn't download any host files, probably because of the date check - echo "Transporter logic detected no changes, pattern skipped..." - fi } - # Use a case statement to download lists that need special cURL commands - # to complete properly and reset the user agent when required - case "$domain" in - "adblock.mahakala.is") - agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' - cmd_ext="-e http://forum.xda-developers.com/" - ;; +# spinup - main gravity function +function gravity_spinup() { - "pgl.yoyo.org") - cmd_ext="-d mimetype=plaintext -d hostformat=hosts" - ;; +# Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and blank lines +for ((i = 0; i < "${#sources[@]}"; i++)) +do + url=${sources[$i]} + # Get just the domain from the URL +# Whitelist (if applicable) domains +if [[ -r $whitelist ]];then + # Remove whitelist entries + numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) + plural=; [[ "$numberOf" != "1" ]] && plural=s + echo "** Whitelisting $numberOf domain${plural}..." - # Default is a simple request - *) cmd_ext="" - esac - gravity_transport $url $cmd_ext $agent + # Append a "$" to the end, prepend a "^" to the beginning, and + # replace "." with "\." of each line to turn each entry into a + # regexp so it can be parsed out with grep -x + awk -F '[# \t]' 'NF>0&&$1!="" {print "^"$1"$"}' $whitelist | sed 's/\./\\./g' > $latentWhitelist +else + rm $latentWhitelist +fi + +# Prevent our sources from being pulled into the hole +plural=; [[ "${#sources[@]}" != "1" ]] && plural=s +echo "** Whitelisting ${#sources[@]} ad list source${plural}..." +for url in ${sources[@]} +do + echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist done - # Compress files to singularity and white/blacklist - gravity_Schwarzchild + +# Remove whitelist entries from list +grep -vxf $latentWhitelist $piholeDir/$matter > $piholeDir/$andLight + } function gravity_advanced() { @@ -120,5 +120,6 @@ function gravity_advanced() { } gravity_spinup - +gravity_transport +gravity_Schwartzchild gravity_advanced From af909e0ee67bf5893e476c047c40e78a0085678c Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 23 Nov 2015 02:52:12 -0800 Subject: [PATCH 55/76] Copyright issues Some of the blocklists are copyright and need to be pulled and stored as is. They can be processed further in the script to remove comments. --- gravity.sh | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 9cb017d..f8f91bd 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Pi-hole: A black hole for Internet advertisements +# (c) 2015 by Jacob Salmela GPL 2.0 +# Network-wide ad blocking via your Raspberry Pi # http://pi-hole.net # Compiles a list of ad-serving domains by downloading them from multiple sources piholeIPfile=/tmp/piholeIP @@ -45,7 +48,6 @@ fi ########################### # collapse - begin formation of pihole function gravity_collapse() { - echo "** Neutrino emissions detected..." # Create the pihole resource directory if it doesn't exist. Future files will be stored here @@ -61,6 +63,44 @@ else fi } +# patternCheck - check to see if curl downloaded any new files, and then process those +# files so they are in host format. +function gravity_patternCheck() { + patternBuffer=$1 + # check if the patternbuffer is a non-zero length file + if [[ -s "$patternBuffer" ]];then + # Some of the blocklists are copyright, they need to be downloaded + # and stored as is. They can be processed for content after they + # have been saved. + cp $patternBuffer $saveLocation + echo "Done." + else + # curl didn't download any host files, probably because of the date check + echo "Transporter logic detected no changes, pattern skipped..." + fi +} + +# transport - curl the specified url with any needed command extentions, then patternCheck +function gravity_transport() { + url=$1 + cmd_ext=$2 + agent=$3 + # tmp file, so we don't have to store the (long!) lists in RAM + patternBuffer=$(mktemp) + heisenbergCompensator="" + if [[ -r $saveLocation ]]; then + # if domain has been saved, add file for date check to only download newer + heisenbergCompensator="-z $saveLocation" + fi + # Silently curl url + curl -s $cmd_ext $heisenbergCompensator -A "$agent" $url > $patternBuffer + + gravity_patternCheck $patternBuffer + + # Cleanup + rm -f $patternBuffer + +} # spinup - main gravity function function gravity_spinup() { @@ -69,6 +109,48 @@ for ((i = 0; i < "${#sources[@]}"; i++)) do url=${sources[$i]} # Get just the domain from the URL + domain=$(echo "$url" | cut -d'/' -f3) + + # Save the file as list.#.domain + saveLocation=$piholeDir/list.$i.$domain.$justDomainsExtension + + agent="Mozilla/10.0" + + echo -n "Getting $domain list... " + + # Use a case statement to download lists that need special cURL commands + # to complete properly and reset the user agent when required + case "$domain" in + "adblock.mahakala.is") + agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' + cmd_ext="-e http://forum.xda-developers.com/" + ;; + + "pgl.yoyo.org") + cmd_ext="-d mimetype=plaintext -d hostformat=hosts" + ;; + + # Default is a simple request + *) cmd_ext="" + esac + gravity_transport $url $cmd_ext $agent +done +} + +# Schwarzchild - aggregate domains to one list, and then white/blacklist unified list +function gravity_Schwarzchild() { + +# Find all files with the .domains extension and compile them into one file and remove CRs +echo "** Aggregating list of domains..." +find $piholeDir/ -type f -name "*.$justDomainsExtension" -exec cat {} \; | tr -d '\r' > $piholeDir/$matter + +# Append blacklist entries if they exist +if [[ -r $blacklist ]];then + numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) + echo "** Blacklisting $numberOf domain(s)..." + cat $blacklist >> $piholeDir/$matter +fi + # Whitelist (if applicable) domains if [[ -r $whitelist ]];then # Remove whitelist entries @@ -102,9 +184,12 @@ function gravity_advanced() { numberOf=$(wc -l < $piholeDir/$andLight) echo "** $numberOf domains being pulled in by gravity..." - # Remove carriage returns and preceding whitespace - # not really needed anymore? - cp $piholeDir/$andLight $piholeDir/$supernova + # Remove comments and print only the domain name + # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious + # This helps with that and makes it easier to read + # It also helps with debugging so each stage of the script can be researched more in depth + awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$andLight | \ + sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $piholeDir/$supernova # Sort and remove duplicates sort -u $piholeDir/$supernova > $piholeDir/$eventHorizon @@ -120,6 +205,5 @@ function gravity_advanced() { } gravity_spinup -gravity_transport -gravity_Schwartzchild +gravity_Schwarzchild gravity_advanced From 4f1bb12d36b64c239cb1ed2bb3f117bb64995498 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 23 Nov 2015 03:11:16 -0800 Subject: [PATCH 56/76] More functions Splitting into more functions --- gravity.sh | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/gravity.sh b/gravity.sh index f8f91bd..4a57112 100755 --- a/gravity.sh +++ b/gravity.sh @@ -179,6 +179,19 @@ grep -vxf $latentWhitelist $piholeDir/$matter > $piholeDir/$andLight } +function gravity_unique() { + # Sort and remove duplicates + sort -u $piholeDir/$supernova > $piholeDir/$eventHorizon + numberOf=$(wc -l < $piholeDir/$eventHorizon) + echo "** $numberOf unique domains trapped in the event horizon." +} +function gravity_hostFormat() { + # Format domain list as "192.168.x.x domain.com" + echo "** Formatting domains into a HOSTS file..." + cat $piholeDir/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $piholeDir/$accretionDisc + # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it + cp $piholeDir/$accretionDisc $adList +} function gravity_advanced() { numberOf=$(wc -l < $piholeDir/$andLight) @@ -191,19 +204,11 @@ function gravity_advanced() { awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$andLight | \ sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $piholeDir/$supernova - # Sort and remove duplicates - sort -u $piholeDir/$supernova > $piholeDir/$eventHorizon - numberOf=$(wc -l < $piholeDir/$eventHorizon) - echo "** $numberOf unique domains trapped in the event horizon." - - # Format domain list as "192.168.x.x domain.com" - echo "** Formatting domains into a HOSTS file..." - cat $piholeDir/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $piholeDir/$accretionDisc - # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it - cp $piholeDir/$accretionDisc $adList sudo kill -HUP $(pidof dnsmasq) } gravity_spinup gravity_Schwarzchild +gravity_unique +gravity_hostFormat gravity_advanced From a5d29a1048f1629b7854dc8cefe25675b904b088 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 23 Nov 2015 03:26:10 -0800 Subject: [PATCH 57/76] Fix function ordering Unique is now called after the comments are scrubbed from the unified domain list. --- gravity.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 4a57112..af1e98d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -137,7 +137,7 @@ do done } -# Schwarzchild - aggregate domains to one list, and then white/blacklist unified list +# Schwarzchild - aggregate domains to one list and add blacklisted domains function gravity_Schwarzchild() { # Find all files with the .domains extension and compile them into one file and remove CRs @@ -150,6 +150,9 @@ if [[ -r $blacklist ]];then echo "** Blacklisting $numberOf domain(s)..." cat $blacklist >> $piholeDir/$matter fi +} + +function gravity_pulsar() { # Whitelist (if applicable) domains if [[ -r $whitelist ]];then @@ -176,7 +179,6 @@ done # Remove whitelist entries from list grep -vxf $latentWhitelist $piholeDir/$matter > $piholeDir/$andLight - } function gravity_unique() { @@ -194,9 +196,6 @@ function gravity_hostFormat() { } function gravity_advanced() { - numberOf=$(wc -l < $piholeDir/$andLight) - echo "** $numberOf domains being pulled in by gravity..." - # Remove comments and print only the domain name # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious # This helps with that and makes it easier to read @@ -204,11 +203,16 @@ function gravity_advanced() { awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$andLight | \ sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $piholeDir/$supernova + numberOf=$(wc -l < $piholeDir/$supernova) + echo "** $numberOf domains being pulled in by gravity..." + gravity_unique + sudo kill -HUP $(pidof dnsmasq) } gravity_spinup gravity_Schwarzchild -gravity_unique +gravity_pulsar gravity_hostFormat gravity_advanced + From e2b518568ea1477068a90587c4dedf0d5859ee9b Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 23 Nov 2015 12:39:47 -0800 Subject: [PATCH 58/76] Forgot to start the directory check function --- gravity.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gravity.sh b/gravity.sh index af1e98d..e3337e4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -210,6 +210,7 @@ function gravity_advanced() { sudo kill -HUP $(pidof dnsmasq) } +gravity_collapse gravity_spinup gravity_Schwarzchild gravity_pulsar From 20399b82620a67a2322a4da51feb79e1620062e4 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 23 Nov 2015 13:12:11 -0800 Subject: [PATCH 59/76] Build array of active domains to concatenate Fix #86 Builds an array of active domains, and then loops through that list to pick files to concatenate after truncating Matter. --- gravity.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index e3337e4..ae20d43 100755 --- a/gravity.sh +++ b/gravity.sh @@ -113,6 +113,7 @@ do # Save the file as list.#.domain saveLocation=$piholeDir/list.$i.$domain.$justDomainsExtension + activeDomains[$i]=$saveLocation agent="Mozilla/10.0" @@ -140,9 +141,13 @@ done # Schwarzchild - aggregate domains to one list and add blacklisted domains function gravity_Schwarzchild() { -# Find all files with the .domains extension and compile them into one file and remove CRs +# Find all active domains and compile them into one file and remove CRs echo "** Aggregating list of domains..." -find $piholeDir/ -type f -name "*.$justDomainsExtension" -exec cat {} \; | tr -d '\r' > $piholeDir/$matter +truncate -s 0 $piholeDir/$matter +for i in "${activeDomains[@]}" +do + cat $i |tr -d '\r' >> $piholeDir/$matter +done # Append blacklist entries if they exist if [[ -r $blacklist ]];then From 04f6c478ccd1f3d651dd858d9cb77339217eef2f Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sun, 15 Nov 2015 20:54:15 -0600 Subject: [PATCH 60/76] Updated install instructions Also a screenshot of the dialog-based installer --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 09b8852..3e02f0d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # Automated Install 1. Install Raspbian -2. Set a **static** IP address -3. Run the command below +2. Run the command below -```curl -L install.pi-hole.net | bash``` +### ```curl -L install.pi-hole.net | bash``` + +![Pi-hole automated installation](http://i.imgur.com/Un7lBlj.png) Once installed, **configure any device to use the Raspberry Pi as its DNS server and the ads will be blocked**. You can also configure your router's DHCP options to assign the Pi as clients DNS server so they do not need to do it manually. From ecb514dd0a206176876b6cba59b4b8ae9a1001be Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Thu, 19 Nov 2015 19:36:34 -0600 Subject: [PATCH 61/76] pihole in the cloud link --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3e02f0d..258168c 100644 --- a/README.md +++ b/README.md @@ -63,5 +63,6 @@ This script will work for other UNIX-like systems with some slight **modificatio ### Examples Of The Pi-hole On Other Operating Systems - [Sky-Hole](http://dlaa.me/blog/post/skyhole) +- [Pi-hole in the Cloud!](http://blog.codybunch.com/2015/07/28/Pi-Hole-in-the-cloud/) [![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif "AdminLTE Presentation")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY "Donate") From 03bbef07b5ef08c7005bade5b634d303149c5f58 Mon Sep 17 00:00:00 2001 From: Cole Turner Date: Fri, 20 Nov 2015 22:53:43 -0500 Subject: [PATCH 62/76] pinholeIP should only use one IP Updated piholeIP because if there are multiple network adapters, the gravity list will populate incorrectly with both IP addresses. Only one address should be used. --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index c6405c7..c2d1ff9 100755 --- a/gravity.sh +++ b/gravity.sh @@ -8,7 +8,7 @@ if [[ -f $piholeIPfile ]];then rm $piholeIPfile else # Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script - piholeIP=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1") + piholeIP=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1" | (head -n1)) fi # Ad-list sources--one per line in single quotes From 464003d6741c81a6bc969db90eb4b34bb9932413 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 21 Nov 2015 13:29:03 -0600 Subject: [PATCH 63/76] uninstall script This should remove the Pi-hole. --- automated install/uninstall.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 automated install/uninstall.sh diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh new file mode 100644 index 0000000..b0dd336 --- /dev/null +++ b/automated install/uninstall.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Completely uninstalls the Pi-hole + +######### SCRIPT ########### +sudo apt-get -y remove --purge dnsutils bc toilet +sudo apt-get -y remove --purge dnsmasq +sudo apt-get -y remove --purge lighttpd php5-common php5-cgi php5 +sudo rm -rf /var/www/html +sudo rm /etc/dnsmasq.conf /etc/dnsmasq.conf.orig +sudo rm /etc/crontab +sudo mv /etc/crontab.orig /etc/crontab +sudo rm /etc/dnsmasq.conf +sudo rm -rf /etc/lighttpd/ +sudo rm /var/log/pihole.log +sudo rm /usr/local/bin/gravity.sh +sudo rm /usr/local/bin/chronometer.sh +sudo rm /usr/local/bin/whitelist.sh +sudo rm /usr/local/bin/piholeLogFlush.sh +sudo rm -rf /etc/pihole/ From 29ae66c6db1d8748ba54fc79a9ec12dc56211afb Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Wed, 25 Nov 2015 13:43:59 -0800 Subject: [PATCH 64/76] Check IP info for outgoing interface --- automated install/basic-install.sh | 46 ++++++++++++------------------ 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f0a7d13..2609973 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Pi-hole: A black hole for Internet advertisements # by Jacob Salmela # Network-wide ad blocking via your Raspberry Pi @@ -24,9 +24,11 @@ columns=$(stty -a | tr \; \\012 | egrep 'columns' | cut -d' ' -f3) r=$(( rows / 2 )) c=$(( columns / 2 )) -IPv4addr=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1") -IPv4mask=$(ifconfig | awk -F':' '/inet addr/ && !/127.0.0.1/ {print $4}') -IPv4gw=$(ip route show | awk '/default\ via/ {print $3}') +# Find IP used to route to outside world +IPv4info=$(ip route get 8.8.8.8) +IPv4dev=$(echo $IPv4info| awk '{print $5}') +IPv4addr=$(ip -o -f inet addr show dev $IPv4dev | awk '{print $4}') +IPv4gw=$(echo $IPv4info | awk '{print $3}') # IPv6 support to be added later #IPv6eui64=$(ip addr show | awk '/scope\ global/ && /ff:fe/ {print $2}' | cut -d'/' -f1) @@ -117,7 +119,6 @@ getStaticIPv4Settings() if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address? IP address: $IPv4addr - Subnet mask: $IPv4mask Gateway: $IPv4gw" $r $c) then # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. whiptail --msgbox --backtitle "IP information" --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict. But in most cases the router is smart enough to not do that. @@ -138,22 +139,17 @@ else IPv4addr=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" $r $c $IPv4addr 3>&1 1>&2 2>&3) if [[ $? = 0 ]];then echo "Your static IPv4 address: $IPv4addr" - # Ask for the subnet mask - IPv4mask=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 netmask" --inputbox "Enter your desired IPv4 subnet mask" $r $c $IPv4mask 3>&1 1>&2 2>&3) - if [[ $? = 0 ]];then - echo "Your static IPv4 netmask: $IPv4mask" - # Ask for the gateway + # Ask for the gateway IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" $r $c $IPv4gw 3>&1 1>&2 2>&3) if [[ $? = 0 ]];then echo "Your static IPv4 gateway: $IPv4gw" # Give the user a chance to review their settings before moving on if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Are these settings correct? IP address: $IPv4addr - Subnet mask: $IPv4mask Gateway: $IPv4gw" $r $c)then # If the settings are correct, then we need to set the piholeIP # Saving it to a temporary file us to retrieve it later when we run the gravity.sh script - echo $IPv4addr > /tmp/piholeIP + echo ${IPv4addr%/*} > /tmp/piholeIP # After that's done, the loop ends and we move on ipSettingsCorrect=True else @@ -167,12 +163,6 @@ else exit fi else - # Cancelling subnet mask settings window - ipSettingsCorrect=False - echo "User canceled." - exit - fi - else # Cancelling IPv4 settings window ipSettingsCorrect=False echo "User canceled." @@ -188,7 +178,7 @@ setStaticIPv4() { # Append these lines to /etc/dhcpcd.conf to enable a static IP echo "interface $piholeInterface -static ip_address=$IPv4addr/24 +static ip_address=$IPv4addr static routers=$IPv4gw static domain_name_servers=$IPv4gw" | sudo tee -a $dhcpcdFile >/dev/null } @@ -224,15 +214,15 @@ sudo rm /var/www/master.zip 2>/dev/null sudo touch /var/log/pihole.log sudo chmod 644 /var/log/pihole.log sudo chown dnsmasq:root /var/log/pihole.log -sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh -sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh -sudo curl -o /usr/local/bin/whitelist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/whitelist.sh -sudo curl -o /usr/local/bin/piholeLogFlush.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/piholeLogFlush.sh -sudo chmod 755 /usr/local/bin/gravity.sh -sudo chmod 755 /usr/local/bin/chronometer.sh -sudo chmod 755 /usr/local/bin/whitelist.sh -sudo chmod 755 /usr/local/bin/piholeLogFlush.sh -sudo /usr/local/bin/gravity.sh +sudo curl -o /usr/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh +sudo curl -o /usr/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh +sudo curl -o /usr/bin/whitelist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/whitelist.sh +sudo curl -o /usr/bin/piholeLogFlush.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/piholeLogFlush.sh +sudo chmod 755 /usr/bin/gravity.sh +sudo chmod 755 /usr/bin/chronometer.sh +sudo chmod 755 /usr/bin/whitelist.sh +sudo chmod 755 /usr/bin/piholeLogFlush.sh +sudo /usr/bin/gravity.sh } ######## SCRIPT ############ From 6e13134328c779375b74b2a2f6d00ee39e67f578 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Wed, 25 Nov 2015 13:51:11 -0800 Subject: [PATCH 65/76] Revert FHS change until merged --- automated install/basic-install.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2609973..f2a98fb 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -214,15 +214,15 @@ sudo rm /var/www/master.zip 2>/dev/null sudo touch /var/log/pihole.log sudo chmod 644 /var/log/pihole.log sudo chown dnsmasq:root /var/log/pihole.log -sudo curl -o /usr/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh -sudo curl -o /usr/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh -sudo curl -o /usr/bin/whitelist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/whitelist.sh -sudo curl -o /usr/bin/piholeLogFlush.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/piholeLogFlush.sh -sudo chmod 755 /usr/bin/gravity.sh -sudo chmod 755 /usr/bin/chronometer.sh -sudo chmod 755 /usr/bin/whitelist.sh -sudo chmod 755 /usr/bin/piholeLogFlush.sh -sudo /usr/bin/gravity.sh +sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh +sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh +sudo curl -o /usr/local/bin/whitelist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/whitelist.sh +sudo curl -o /usr/local/bin/piholeLogFlush.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/piholeLogFlush.sh +sudo chmod 755 /usr/local/bin/gravity.sh +sudo chmod 755 /usr/local/bin/chronometer.sh +sudo chmod 755 /usr/local/bin/whitelist.sh +sudo chmod 755 /usr/local/bin/piholeLogFlush.sh +sudo /usr/local/bin/gravity.sh } ######## SCRIPT ############ From 21ee63b7fb77da949eca5aca710afade971a0236 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Wed, 25 Nov 2015 14:08:13 -0800 Subject: [PATCH 66/76] Get IP info from outgoing interface Use IP to get interface information from outgoing interface. Use CIDR notation to make things easier. --- automated install/basic-install.sh | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f0a7d13..f2a98fb 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Pi-hole: A black hole for Internet advertisements # by Jacob Salmela # Network-wide ad blocking via your Raspberry Pi @@ -24,9 +24,11 @@ columns=$(stty -a | tr \; \\012 | egrep 'columns' | cut -d' ' -f3) r=$(( rows / 2 )) c=$(( columns / 2 )) -IPv4addr=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1") -IPv4mask=$(ifconfig | awk -F':' '/inet addr/ && !/127.0.0.1/ {print $4}') -IPv4gw=$(ip route show | awk '/default\ via/ {print $3}') +# Find IP used to route to outside world +IPv4info=$(ip route get 8.8.8.8) +IPv4dev=$(echo $IPv4info| awk '{print $5}') +IPv4addr=$(ip -o -f inet addr show dev $IPv4dev | awk '{print $4}') +IPv4gw=$(echo $IPv4info | awk '{print $3}') # IPv6 support to be added later #IPv6eui64=$(ip addr show | awk '/scope\ global/ && /ff:fe/ {print $2}' | cut -d'/' -f1) @@ -117,7 +119,6 @@ getStaticIPv4Settings() if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address? IP address: $IPv4addr - Subnet mask: $IPv4mask Gateway: $IPv4gw" $r $c) then # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. whiptail --msgbox --backtitle "IP information" --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict. But in most cases the router is smart enough to not do that. @@ -138,22 +139,17 @@ else IPv4addr=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" $r $c $IPv4addr 3>&1 1>&2 2>&3) if [[ $? = 0 ]];then echo "Your static IPv4 address: $IPv4addr" - # Ask for the subnet mask - IPv4mask=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 netmask" --inputbox "Enter your desired IPv4 subnet mask" $r $c $IPv4mask 3>&1 1>&2 2>&3) - if [[ $? = 0 ]];then - echo "Your static IPv4 netmask: $IPv4mask" - # Ask for the gateway + # Ask for the gateway IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" $r $c $IPv4gw 3>&1 1>&2 2>&3) if [[ $? = 0 ]];then echo "Your static IPv4 gateway: $IPv4gw" # Give the user a chance to review their settings before moving on if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Are these settings correct? IP address: $IPv4addr - Subnet mask: $IPv4mask Gateway: $IPv4gw" $r $c)then # If the settings are correct, then we need to set the piholeIP # Saving it to a temporary file us to retrieve it later when we run the gravity.sh script - echo $IPv4addr > /tmp/piholeIP + echo ${IPv4addr%/*} > /tmp/piholeIP # After that's done, the loop ends and we move on ipSettingsCorrect=True else @@ -167,12 +163,6 @@ else exit fi else - # Cancelling subnet mask settings window - ipSettingsCorrect=False - echo "User canceled." - exit - fi - else # Cancelling IPv4 settings window ipSettingsCorrect=False echo "User canceled." @@ -188,7 +178,7 @@ setStaticIPv4() { # Append these lines to /etc/dhcpcd.conf to enable a static IP echo "interface $piholeInterface -static ip_address=$IPv4addr/24 +static ip_address=$IPv4addr static routers=$IPv4gw static domain_name_servers=$IPv4gw" | sudo tee -a $dhcpcdFile >/dev/null } From b011fe28d9ba095f6f6a057dc3a08bc0749f305b Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Wed, 25 Nov 2015 14:34:06 -0800 Subject: [PATCH 67/76] No longer need a reboot --- automated install/basic-install.sh | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f2a98fb..47c402f 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -128,8 +128,6 @@ if (whiptail --backtitle "Calibrating network interface" --title "Static IP Addr It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." $r $c # Nothing else to do since the variables are already set above else - # Since a custom address will be used, restart at the end of the script to apply the new changes - rebootNeeded=true # Otherwise, we need to ask the user to input their desired settings. # Start by getting the IPv4 address (pre-filling it with info gathered from DHCP) # Start a loop to let the user enter their information with the chance to go back and edit it if necessary @@ -180,7 +178,8 @@ setStaticIPv4() echo "interface $piholeInterface static ip_address=$IPv4addr static routers=$IPv4gw -static domain_name_servers=$IPv4gw" | sudo tee -a $dhcpcdFile >/dev/null +static domain_name_servers=$IPv4gw" | sudo tee -a $dhcpcdFile >/dev/null +sudo ip addr replace dev $piholeInterface $IPv4addr } installPihole() @@ -274,12 +273,5 @@ If you didn't use DHCP settings as your new static address, the Pi will restart The install log is in /etc/pihole." $r $c -# If a custom address was set, restart -if [[ "$rebootNeeded" = true ]];then - # Restart to apply the new static IP address - sudo reboot -else - # If not, just start the services since the address will stay the same - sudo service dnsmasq start - sudo service lighttpd start -fi +sudo service dnsmasq start +sudo service lighttpd start From bc0a8e7733925799d0401c40fc1132a30ed516de Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Wed, 25 Nov 2015 18:08:17 -0600 Subject: [PATCH 68/76] final dialog verbiage Updating dialog to reflect changes in #90 --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 47c402f..3c6d14f 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -178,7 +178,7 @@ setStaticIPv4() echo "interface $piholeInterface static ip_address=$IPv4addr static routers=$IPv4gw -static domain_name_servers=$IPv4gw" | sudo tee -a $dhcpcdFile >/dev/null +static domain_name_servers=$IPv4gw" | sudo tee -a $dhcpcdFile >/dev/null sudo ip addr replace dev $piholeInterface $IPv4addr } @@ -269,7 +269,7 @@ sudo mv $tmpLog $instalLogLoc whiptail --msgbox --backtitle "Make it so." --title "Installation Complete!" "Configure your devices to use the Pi-hole as their DNS server using this IP: $IPv4addr. -If you didn't use DHCP settings as your new static address, the Pi will restart after this dialog. If you are using SSH, you may need to reconnect using the IP address above. +If you set a new IP address, it should work fine, but you may want to reboot the Pi at some point. The install log is in /etc/pihole." $r $c From 23856d65e21bb0b960882eecff48cdb932c586af Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Wed, 25 Nov 2015 16:35:36 -0800 Subject: [PATCH 69/76] IP address detection update --- gravity.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index c2d1ff9..4952054 100755 --- a/gravity.sh +++ b/gravity.sh @@ -8,7 +8,9 @@ if [[ -f $piholeIPfile ]];then rm $piholeIPfile else # Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script - piholeIP=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1" | (head -n1)) + IPv4dev=$(ip route get 8.8.8.8 | awk '{print $5}') + piholeIPCIDR=$(ip -o -f inet addr show dev $IPv4dev | awk '{print $4}') + piholeIP=${piholeIPCIDR%/*} fi # Ad-list sources--one per line in single quotes From 29ae57465bbbf06606894278cda84ad8df08f15f Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Wed, 25 Nov 2015 19:51:07 -0800 Subject: [PATCH 70/76] Bug Fix - Cleanup Files --- gravity.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index ae20d43..d70f0dc 100755 --- a/gravity.sh +++ b/gravity.sh @@ -145,7 +145,7 @@ function gravity_Schwarzchild() { echo "** Aggregating list of domains..." truncate -s 0 $piholeDir/$matter for i in "${activeDomains[@]}" -do +do cat $i |tr -d '\r' >> $piholeDir/$matter done @@ -199,6 +199,16 @@ function gravity_hostFormat() { # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it cp $piholeDir/$accretionDisc $adList } +function gravity_blackbody() { + for file in $piholeDir/*.$justDomainsExtension + do + if [[ " ${activeDomains[@]} " =~ " ${file} " ]]; then + : + else + rm -f $file + fi + done +} function gravity_advanced() { # Remove comments and print only the domain name @@ -221,4 +231,4 @@ gravity_Schwarzchild gravity_pulsar gravity_hostFormat gravity_advanced - +gravity_blackbody From c45dc277b6117aae69e1e60cbe2d7e77ab2e4525 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 26 Nov 2015 15:29:13 -0800 Subject: [PATCH 71/76] Tighten themeatics --- gravity.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index d70f0dc..018989c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -73,10 +73,10 @@ function gravity_patternCheck() { # and stored as is. They can be processed for content after they # have been saved. cp $patternBuffer $saveLocation - echo "Done." + echo "List updated, transport successful..." else # curl didn't download any host files, probably because of the date check - echo "Transporter logic detected no changes, pattern skipped..." + echo "No changes detected, transport skipped..." fi } @@ -117,7 +117,7 @@ do agent="Mozilla/10.0" - echo -n "Getting $domain list... " + echo -n " Getting $domain list: " # Use a case statement to download lists that need special cURL commands # to complete properly and reset the user agent when required From d1e475da89154fe2ad83e03391137ca1fae0e99e Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 26 Nov 2015 15:48:52 -0800 Subject: [PATCH 72/76] Move black/white list check to pulsar --- gravity.sh | 144 ++++++++++++++++++++++++++++------------------------- 1 file changed, 77 insertions(+), 67 deletions(-) diff --git a/gravity.sh b/gravity.sh index 018989c..27f5d82 100755 --- a/gravity.sh +++ b/gravity.sh @@ -4,6 +4,7 @@ # Network-wide ad blocking via your Raspberry Pi # http://pi-hole.net # Compiles a list of ad-serving domains by downloading them from multiple sources + piholeIPfile=/tmp/piholeIP if [[ -f $piholeIPfile ]];then # If the file exists, it means it was exported from the installation script and we should use that value instead of detecting it in this script @@ -15,7 +16,8 @@ else fi # Ad-list sources--one per line in single quotes -# The mahakala source is commented out due to many users having issues with it blocking legitimate domains. Uncomment at your own risk +# The mahakala source is commented out due to many users having issues with it blocking legitimate domains. +# Uncomment at your own risk sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' #'http://adblock.mahakala.is/' @@ -45,62 +47,65 @@ if [[ -r $piholeDir/pihole.conf ]];then echo "** Local calibration requested..." . $piholeDir/pihole.conf fi + ########################### # collapse - begin formation of pihole function gravity_collapse() { echo "** Neutrino emissions detected..." -# Create the pihole resource directory if it doesn't exist. Future files will be stored here -if [[ -d $piholeDir ]];then + # Create the pihole resource directory if it doesn't exist. Future files will be stored here + if [[ -d $piholeDir ]];then # Temporary hack to allow non-root access to pihole directory # Will update later, needed for existing installs, new installs should # create this directory as non-root sudo chmod 777 $piholeDir find "$piholeDir" -type f -exec sudo chmod 666 {} \; -else + else echo "** Creating pihole directory..." mkdir $piholeDir -fi + fi } # patternCheck - check to see if curl downloaded any new files, and then process those # files so they are in host format. function gravity_patternCheck() { - patternBuffer=$1 - # check if the patternbuffer is a non-zero length file - if [[ -s "$patternBuffer" ]];then - # Some of the blocklists are copyright, they need to be downloaded - # and stored as is. They can be processed for content after they - # have been saved. - cp $patternBuffer $saveLocation - echo "List updated, transport successful..." - else - # curl didn't download any host files, probably because of the date check - echo "No changes detected, transport skipped..." - fi + patternBuffer=$1 + # check if the patternbuffer is a non-zero length file + if [[ -s "$patternBuffer" ]];then + # Some of the blocklists are copyright, they need to be downloaded + # and stored as is. They can be processed for content after they + # have been saved. + cp $patternBuffer $saveLocation + echo "List updated, transport successful..." + else + # curl didn't download any host files, probably because of the date check + echo "No changes detected, transport skipped..." + fi } # transport - curl the specified url with any needed command extentions, then patternCheck function gravity_transport() { - url=$1 - cmd_ext=$2 - agent=$3 - # tmp file, so we don't have to store the (long!) lists in RAM - patternBuffer=$(mktemp) - heisenbergCompensator="" - if [[ -r $saveLocation ]]; then - # if domain has been saved, add file for date check to only download newer - heisenbergCompensator="-z $saveLocation" - fi - # Silently curl url - curl -s $cmd_ext $heisenbergCompensator -A "$agent" $url > $patternBuffer + url=$1 + cmd_ext=$2 + agent=$3 + + # tmp file, so we don't have to store the (long!) lists in RAM + patternBuffer=$(mktemp) + heisenbergCompensator="" + if [[ -r $saveLocation ]]; then + # if domain has been saved, add file for date check to only download newer + heisenbergCompensator="-z $saveLocation" + fi - gravity_patternCheck $patternBuffer - - # Cleanup - rm -f $patternBuffer + # Silently curl url + curl -s $cmd_ext $heisenbergCompensator -A "$agent" $url > $patternBuffer + # Check for list updates + gravity_patternCheck $patternBuffer + # Cleanup + rm -f $patternBuffer } + # spinup - main gravity function function gravity_spinup() { @@ -141,26 +146,26 @@ done # Schwarzchild - aggregate domains to one list and add blacklisted domains function gravity_Schwarzchild() { -# Find all active domains and compile them into one file and remove CRs -echo "** Aggregating list of domains..." -truncate -s 0 $piholeDir/$matter -for i in "${activeDomains[@]}" -do - cat $i |tr -d '\r' >> $piholeDir/$matter -done - -# Append blacklist entries if they exist -if [[ -r $blacklist ]];then - numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) - echo "** Blacklisting $numberOf domain(s)..." - cat $blacklist >> $piholeDir/$matter -fi + # Find all active domains and compile them into one file and remove CRs + echo "** Aggregating list of domains..." + truncate -s 0 $piholeDir/$matter + for i in "${activeDomains[@]}" + do + cat $i |tr -d '\r' >> $piholeDir/$matter + done } function gravity_pulsar() { + + # Append blacklist entries if they exist + if [[ -r $blacklist ]];then + numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) + echo "** Blacklisting $numberOf domain(s)..." + cat $blacklist >> $piholeDir/$matter + fi -# Whitelist (if applicable) domains -if [[ -r $whitelist ]];then + # Whitelist (if applicable) domains + if [[ -r $whitelist ]];then # Remove whitelist entries numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) plural=; [[ "$numberOf" != "1" ]] && plural=s @@ -170,20 +175,20 @@ if [[ -r $whitelist ]];then # replace "." with "\." of each line to turn each entry into a # regexp so it can be parsed out with grep -x awk -F '[# \t]' 'NF>0&&$1!="" {print "^"$1"$"}' $whitelist | sed 's/\./\\./g' > $latentWhitelist -else + else rm $latentWhitelist -fi + fi -# Prevent our sources from being pulled into the hole -plural=; [[ "${#sources[@]}" != "1" ]] && plural=s -echo "** Whitelisting ${#sources[@]} ad list source${plural}..." -for url in ${sources[@]} -do + # Prevent our sources from being pulled into the hole + plural=; [[ "${#sources[@]}" != "1" ]] && plural=s + echo "** Whitelisting ${#sources[@]} ad list source${plural}..." + for url in ${sources[@]} + do echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist -done + done -# Remove whitelist entries from list -grep -vxf $latentWhitelist $piholeDir/$matter > $piholeDir/$andLight + # Remove whitelist entries from list + grep -vxf $latentWhitelist $piholeDir/$matter > $piholeDir/$andLight } function gravity_unique() { @@ -192,6 +197,7 @@ function gravity_unique() { numberOf=$(wc -l < $piholeDir/$eventHorizon) echo "** $numberOf unique domains trapped in the event horizon." } + function gravity_hostFormat() { # Format domain list as "192.168.x.x domain.com" echo "** Formatting domains into a HOSTS file..." @@ -199,16 +205,20 @@ function gravity_hostFormat() { # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it cp $piholeDir/$accretionDisc $adList } + function gravity_blackbody() { - for file in $piholeDir/*.$justDomainsExtension - do - if [[ " ${activeDomains[@]} " =~ " ${file} " ]]; then - : - else - rm -f $file - fi - done + # Loop through list files + for file in $piholeDir/*.$justDomainsExtension + do + # If list is active then leave it (noop) else rm the list + if [[ " ${activeDomains[@]} " =~ " ${file} " ]]; then + : + else + rm -f $file + fi + done } + function gravity_advanced() { # Remove comments and print only the domain name From 9892d06c0db96ad090061778049e765d05283007 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 26 Nov 2015 15:56:37 -0800 Subject: [PATCH 73/76] Code format cleanups --- gravity.sh | 61 +++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/gravity.sh b/gravity.sh index 27f5d82..f34ce80 100755 --- a/gravity.sh +++ b/gravity.sh @@ -51,7 +51,7 @@ fi ########################### # collapse - begin formation of pihole function gravity_collapse() { -echo "** Neutrino emissions detected..." + echo "** Neutrino emissions detected..." # Create the pihole resource directory if it doesn't exist. Future files will be stored here if [[ -d $piholeDir ]];then @@ -66,8 +66,7 @@ echo "** Neutrino emissions detected..." fi } -# patternCheck - check to see if curl downloaded any new files, and then process those -# files so they are in host format. +# patternCheck - check to see if curl downloaded any new files. function gravity_patternCheck() { patternBuffer=$1 # check if the patternbuffer is a non-zero length file @@ -83,7 +82,7 @@ function gravity_patternCheck() { fi } -# transport - curl the specified url with any needed command extentions, then patternCheck +# transport - curl the specified url with any needed command extentions function gravity_transport() { url=$1 cmd_ext=$2 @@ -109,9 +108,9 @@ function gravity_transport() { # spinup - main gravity function function gravity_spinup() { -# Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and blank lines -for ((i = 0; i < "${#sources[@]}"; i++)) -do + # Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and # blank lines + for ((i = 0; i < "${#sources[@]}"; i++)) + do url=${sources[$i]} # Get just the domain from the URL domain=$(echo "$url" | cut -d'/' -f3) @@ -139,8 +138,8 @@ do # Default is a simple request *) cmd_ext="" esac - gravity_transport $url $cmd_ext $agent -done + gravity_transport $url $cmd_ext $agent + done } # Schwarzchild - aggregate domains to one list and add blacklisted domains @@ -155,6 +154,7 @@ function gravity_Schwarzchild() { done } +# Pulsar - White/blacklist application function gravity_pulsar() { # Append blacklist entries if they exist @@ -192,25 +192,26 @@ function gravity_pulsar() { } function gravity_unique() { - # Sort and remove duplicates - sort -u $piholeDir/$supernova > $piholeDir/$eventHorizon - numberOf=$(wc -l < $piholeDir/$eventHorizon) - echo "** $numberOf unique domains trapped in the event horizon." + # Sort and remove duplicates + sort -u $piholeDir/$supernova > $piholeDir/$eventHorizon + numberOf=$(wc -l < $piholeDir/$eventHorizon) + echo "** $numberOf unique domains trapped in the event horizon." } function gravity_hostFormat() { - # Format domain list as "192.168.x.x domain.com" - echo "** Formatting domains into a HOSTS file..." - cat $piholeDir/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $piholeDir/$accretionDisc - # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it - cp $piholeDir/$accretionDisc $adList + # Format domain list as "192.168.x.x domain.com" + echo "** Formatting domains into a HOSTS file..." + cat $piholeDir/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $piholeDir/$accretionDisc + # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it + cp $piholeDir/$accretionDisc $adList } +# blackbody - remove any remnant files from script processes function gravity_blackbody() { # Loop through list files for file in $piholeDir/*.$justDomainsExtension do - # If list is active then leave it (noop) else rm the list + # If list is in active array then leave it (noop) else rm the list if [[ " ${activeDomains[@]} " =~ " ${file} " ]]; then : else @@ -220,19 +221,19 @@ function gravity_blackbody() { } function gravity_advanced() { + # Remove comments and print only the domain name + # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious + # This helps with that and makes it easier to read + # It also helps with debugging so each stage of the script can be researched more in depth + awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$andLight | \ + sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $piholeDir/$supernova - # Remove comments and print only the domain name - # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious - # This helps with that and makes it easier to read - # It also helps with debugging so each stage of the script can be researched more in depth - awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$andLight | \ - sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $piholeDir/$supernova + numberOf=$(wc -l < $piholeDir/$supernova) + echo "** $numberOf domains being pulled in by gravity..." - numberOf=$(wc -l < $piholeDir/$supernova) - echo "** $numberOf domains being pulled in by gravity..." - gravity_unique - - sudo kill -HUP $(pidof dnsmasq) + gravity_unique + + sudo kill -HUP $(pidof dnsmasq) } gravity_collapse From a2cddda590ad983dc7ae8915646130559eb1e947 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 27 Nov 2015 18:29:44 -0600 Subject: [PATCH 74/76] if whitelist.txt doesn't exist, create it --- advanced/Scripts/whitelist.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/whitelist.sh b/advanced/Scripts/whitelist.sh index d610d0a..97385c9 100755 --- a/advanced/Scripts/whitelist.sh +++ b/advanced/Scripts/whitelist.sh @@ -1,5 +1,9 @@ #!/bin/bash +if [[ ! -f /etc/pihole/whitelist.txt ]];then + touch /etc/pihole/whitelist.txt +fi + if [ $# = 0 ]; then echo "Immediately whitelists one or more domains." echo "Usage: whitelist.sh domain1 [domain2 ...]" @@ -29,7 +33,7 @@ done # Now report on and remove matched domains if [ "$combopattern" != "" ]; then echo "Modifying hosts file..." - + # Construct pattern to match entry in hosts file. # This consists of one or more IP addresses followed by the domain name. pattern=$(echo $combopattern | awk -F '[# \t]' '{printf "%s", "^(([0-9]+\.){3}[0-9]+ +)+("$1")$"}') From 573c1ef94f1baa0da7ec53dd9221ddf7c03957bc Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 27 Nov 2015 18:34:02 -0600 Subject: [PATCH 75/76] double bracket notation --- advanced/Scripts/whitelist.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/whitelist.sh b/advanced/Scripts/whitelist.sh index 97385c9..ea06cc8 100755 --- a/advanced/Scripts/whitelist.sh +++ b/advanced/Scripts/whitelist.sh @@ -4,7 +4,7 @@ if [[ ! -f /etc/pihole/whitelist.txt ]];then touch /etc/pihole/whitelist.txt fi -if [ $# = 0 ]; then +if [[ $# = 0 ]]; then echo "Immediately whitelists one or more domains." echo "Usage: whitelist.sh domain1 [domain2 ...]" fi @@ -19,9 +19,9 @@ do # Construct basic pattern to match domain name. basicpattern=$(echo $var | awk -F '[# \t]' 'NF>0&&$1!="" {print ""$1""}' | sed 's/\./\\./g') - if [ "$basicpattern" != "" ]; then + if [[ "$basicpattern" != "" ]]; then # Add to the combination pattern that will be used below - if [ "$combopattern" != "" ]; then combopattern="$combopattern|"; fi + if [[ "$combopattern" != "" ]]; then combopattern="$combopattern|"; fi combopattern="$combopattern$basicpattern" # Also add the domain to the whitelist but only if it's not already present @@ -31,7 +31,7 @@ do done # Now report on and remove matched domains -if [ "$combopattern" != "" ]; then +if [[ "$combopattern" != "" ]]; then echo "Modifying hosts file..." # Construct pattern to match entry in hosts file. From 831322b8b2d6faa07b3951cd1973dab4d6ebaf62 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 27 Nov 2015 18:38:33 -0600 Subject: [PATCH 76/76] replace file paths with variables --- advanced/Scripts/whitelist.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/whitelist.sh b/advanced/Scripts/whitelist.sh index ea06cc8..cda557c 100755 --- a/advanced/Scripts/whitelist.sh +++ b/advanced/Scripts/whitelist.sh @@ -1,7 +1,8 @@ #!/bin/bash - -if [[ ! -f /etc/pihole/whitelist.txt ]];then - touch /etc/pihole/whitelist.txt +whitelist=/etc/pihole/whitelist.txt +adList=/etc/pihole/gravity.list +if [[ ! -f $whitelist ]];then + touch $whitelist fi if [[ $# = 0 ]]; then @@ -25,8 +26,8 @@ do combopattern="$combopattern$basicpattern" # Also add the domain to the whitelist but only if it's not already present - grep -E -q "^$basicpattern$" /etc/pihole/whitelist.txt \ - || echo "$var" >> /etc/pihole/whitelist.txt + grep -E -q "^$basicpattern$" $whitelist \ + || echo "$var" >> $whitelist fi done @@ -39,8 +40,8 @@ if [[ "$combopattern" != "" ]]; then pattern=$(echo $combopattern | awk -F '[# \t]' '{printf "%s", "^(([0-9]+\.){3}[0-9]+ +)+("$1")$"}') # Output what will be removed and then actually remove - sed -r -n 's/'"$pattern"'/ Removed: \3/p' /etc/pihole/gravity.list - sed -r -i '/'"$pattern"'/d' /etc/pihole/gravity.list + sed -r -n 's/'"$pattern"'/ Removed: \3/p' $adList + sed -r -i '/'"$pattern"'/d' $adList echo "** $# domain(s) whitelisted." # Force dnsmasq to reload /etc/pihole/gravity.list