From 47baa1a6fdbb3b60f4e44a94a52417b5d381b79f Mon Sep 17 00:00:00 2001 From: jacobsalmela Date: Tue, 17 Feb 2015 16:10:28 -0600 Subject: [PATCH] Removes more duplicates that uniq could not find http://jacobsalmela.com/raspberry-pi-ad-blocker-advanced-setup/#comment- 1860675175 Thanks to napgravy for figuring this out. It seems the DOS-style line-endings that prevented uniq from getting rid of them. This reduces the ad domains from ~140,000 to around ~120,000 but it is much more accurate. --- gravity-adv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-adv.sh b/gravity-adv.sh index b024ec3..468d8d0 100755 --- a/gravity-adv.sh +++ b/gravity-adv.sh @@ -27,7 +27,7 @@ curl -A 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' - # Sort the aggregated results and remove any duplicates echo "Removing duplicates and formatting to address=//"$piholeIP -cat /tmp/matter.txt | sort | uniq | sed '/^$/d' | awk -v "IP=$piholeIP" '{sub(/\r$/,""); print "address=/"$0"/"IP}' > /tmp/andLight.txt +cat /tmp/matter.txt | sed $'s/\r$//' | sort | uniq | sed '/^$/d' | awk -v "IP=$piholeIP" '{sub(/\r$/,""); print "address=/"$0"/"IP}' > /tmp/andLight.txt # Count how many domains were added so it can be displayed to the user numberOfAdsBlocked=$(cat /tmp/andLight.txt | wc -l | sed 's/^[ \t]*//')