#!/bin/bash
# # # Description
# Grap Blocklists for use with /etc/hosts
#
# Script=get-hosts-block.sh
# Author=Michael
# Email=use website contact
# Website=http://www.inet-design.com/
# License=GPL
# Script repository=none
# Last Edit Date=Sun 30 Aug 2020
#
# Instructions:
# As root,
# - cp /etc/hosts /{$WorkingDir}/hosts.orig
#
# - Copy computer name line from /etc/hosts
#   and any other addtions to /{$WorkingDir}/hosts.mine.unique
# e.g.
#   127.0.0.1       {computer-name}
#   0.0.0.0         fonts.googleapis.com
#   0.0.0.0         fonts.googleapis.com:443
#   0.0.0.0         tracking.klickthru.com
#   ...
#
# - Fix all pathing/naming...
#
# - Add to root crontab
#
# Notes:
# I started building this for multipe lists, but found
# StevenBlack's already includes Peter Lowe's (yoyo.org).
#
# If more than one list gets used, need to clean/combine them
# and will need to re-work 'hosts' building at end.
#
# Root crontab
# 45 6 * * 1 /home/michael/common/bin/get-hosts-block.sh >>/home/michael/common/bin/log/get-hosts-block.sh.log 2>&1
# 06:45 on Monday, that way if it breaks, it won't have broken much and can be fixed at start of day.
#
# # # # #

# Include Standard Paths & Functions
# IncFile="$(dirname "${BASH_SOURCE[0]}")/includes/stdpaths.sh"
# if [ -f "$IncFile" ] ; then
#   . "$IncFile"
# else
#   echo 'No Standard Paths!  Exiting'
#   exit 1
# fi
# IncFile="$(dirname "${BASH_SOURCE[0]}")/includes/stdfunctions.sh"
# if [ -f "$IncFile" ] ; then
#   . "$IncFile"
# fi
function randomUserAgent {
  RndNum=`echo $((1 + RANDOM % 11))`
  UserAgent[0]='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)'
  UserAgent[1]='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)'
  UserAgent[2]='Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)'
  UserAgent[3]='Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko'
  UserAgent[4]='Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; MDDCJS)'
  UserAgent[5]='Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0;  Trident/5.0)'
  UserAgent[6]='Mozilla/5.0 (iPad; CPU OS 8_4_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12H321 Safari/600.1.4'
  UserAgent[7]='Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1'
  UserAgent[8]='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0'
  UserAgent[9]='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393'
  UserAgent[10]='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'
  UserAgent[11]='Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US)'

  echo ${UserAgent[$RndNum]}
}

renice -n 15 -p $$ >/dev/null 2>/dev/null
# # # #
# Static Setups
# Random user aget
UserAgent=`randomUserAgent`
# # # #

# change to {$WorkingDir}
# WorkingDir="/home/michael/data/trash/hoststest"
WorkingDir="/root/hosts"
if [ ! -d "$WorkingDir" ] ; then
  echo 'Hey, you goofed...'
  echo "md $WorkingDir"
  echo '-and-'
  echo 'place hosts.orig and hosts.mine.unique in it...'
  exit
fi
cd "$WorkingDir"

# backup existing hosts
CurrDateTime=$(date +"%Y%m%d-%H%M%S")
cp --preserve=all "/etc/hosts" "$WorkingDir/hosts.$CurrDateTime"

# # # [AAA-START]
# # Notes
#
# Add as many block lists as desired, one per line.
#
# escape ('\') items that bork bash, eg..
# https://pgl.yoyo.org/adservers/serverlist.php?showintro=0;hostformat=hosts
# https://pgl.yoyo.org/adservers/serverlist.php?showintro=0\;hostformat=hosts
# PS: Don't use above as it's HTML, not sure where 'raw' is...
#
HostLists=(
https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
)
# # # [AAA-END]

DateTime=`date +"%Y%m%d%H%M"`
# echo -n "$DateTime" >> attictemps.csv

i=0
for HostList in "${HostLists[@]}"; do 
  let "i=i+1"
#   echo "hostlist.$i"
#   echo "$HostList"

wget \
--user-agent="$UserAgent" \
--passive-ftp \
--execute robots=off \
--output-document="hostlist.$i" \
--append-output hostlist.log \
"$HostList"

done

cp --preserve=all 'hosts.mine.unique' 'hosts'
cat "hostlist.$i" >> 'hosts'
cp --preserve=all 'hosts' '/etc/hosts'