added a bunch of crap

README.md

@@ -1,13 +1,20 @@
 # Gopherhole
 
-DNS server for blocking ads. Like Pi hole but for hackers.
+Go built DNS server for blocking ads. Like Pi-hole but for hackers.
+
+Works with Pi-hole block lists.
+
+Find more Pi-Hole compatible block lists here: <https://firebog.net/>
 
 - [x] Has support for custom DNS records and block lists.
-- [x] Supports custom upstream DNS servers for recursively resolving domains
+- [x] Supports custom upstream DNS servers for recursively resolving domains.
+- [x] Supports reloading block lists via HTTP API.
+- [ ] Setting to use hosts file as upstream
+- [ ] Generate example config command.
 - [ ] Supports adding block lists and records via HTTP API.
 - [ ] Prometheus metrics support.
 - [ ] HTTP JSON API metrics.
-- [ ] DNS over HTTPS
+- [ ] DNS over HTTPS.
 
 ## How to use
 

blocklists.go

@@ -57,11 +57,13 @@ func (mdb *memoryBlocklistManager) Reload(ctx context.Context) error {
 	close(bc)
 
 	mdb.blocklists = []Blocklist{}
+	count := 0
 	for bl := range bc {
 		mdb.blocklists = append(mdb.blocklists, bl)
+		count += len(bl.Domains)
 	}
 
-	mdb.Printf("successfully loaded '%d' block lists", len(mdb.blocklists))
+	mdb.Printf("successfully loaded %d block lists totaling %d domains", len(mdb.blocklists), count)
 	return nil
 }
 

config.go

@@ -30,12 +30,28 @@ func LoadConfig(path string) (*Configuration, error) {
 
 type Configuration struct {
 	BindAddress net.IP
-	Upstream    []string
-	Blocklists  []string
-	Records     map[string]ConfigRecord
+	UseHosts    bool                    `json:"useHosts"`
+	Upstream    []string                `json:"upstreams"`
+	Blocklists  []string                `json:"blocklists"`
+	Records     map[string]ConfigRecord `json:"records"`
 }
 
 type ConfigRecord struct {
 	Type   string
 	Record string
 }
+
+func NewConfig() *Configuration {
+	return &Configuration{
+		BindAddress: net.IPv4(127, 0, 0, 1),
+		Upstream:    []string{"1.1.1.1"},
+		UseHosts:    true,
+		Blocklists:  []string{"https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt"},
+		Records: map[string]ConfigRecord{
+			"home.internal": ConfigRecord{
+				Type:   "A",
+				Record: "127.0.0.1",
+			},
+		},
+	}
+}

config.json

@@ -1,5 +1,6 @@
 {
-    "upstream": [
+    "useHosts": true,
+    "upstreams": [
         "1.1.1.1",
         "8.8.8.8"
     ],
@@ -13,9 +14,60 @@
             "Record": "internal.veldhousen.ninja"
         }
     },
+    "whitelists": [ ],
     "blocklists": [
         "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
         "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt",
-        "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt"
+        "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt",
+        "https://v.firebog.net/hosts/Easyprivacy.txt",
+        "https://v.firebog.net/hosts/Prigent-Ads.txt",
+        "https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt",
+        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts",
+        "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt",
+        "https://hostfiles.frogeye.fr/firstparty-trackers-hosts.txt",
+        "https://hostfiles.frogeye.fr/multiparty-trackers-hosts.txt",
+        "https://www.github.developerdan.com/hosts/lists/ads-and-tracking-extended.txt",
+        "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt",
+        "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt",
+        "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt",
+        "https://adaway.org/hosts.txt",
+        "https://v.firebog.net/hosts/AdguardDNS.txt",
+        "https://v.firebog.net/hosts/Admiral.txt",
+        "https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt",
+        "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt",
+        "https://v.firebog.net/hosts/Easylist.txt",
+        "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext",
+        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts",
+        "https://raw.githubusercontent.com/bigdargon/hostsVN/master/hosts",
+        "https://raw.githubusercontent.com/jdlingyu/ad-wars/master/hosts",
+        "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt",
+        "https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt",
+        "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt",
+        "https://mirror1.malwaredomains.com/files/justdomains",
+        "https://v.firebog.net/hosts/Prigent-Crypto.txt",
+        "https://v.firebog.net/hosts/Prigent-Malware.txt",
+        "https://mirror.cedia.org.ec/malwaredomains/immortal_domains.txt",
+        "https://www.malwaredomainlist.com/hostslist/hosts.txt",
+        "https://bitbucket.org/ethanr/dns-blacklists/raw/8575c9f96e5b4a1308f2f12394abd86d0927a4a0/bad_lists/Mandiant_APT1_Report_Appendix_D.txt",
+        "https://phishing.army/download/phishing_army_blocklist_extended.txt",
+        "https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt",
+        "https://v.firebog.net/hosts/Shalla-mal.txt",
+        "https://raw.githubusercontent.com/Spam404/lists/master/main-blacklist.txt",
+        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts",
+        "https://urlhaus.abuse.ch/downloads/hostfile/",
+        "https://raw.githubusercontent.com/HorusTeknoloji/TR-PhishingList/master/url-lists.txt",
+        "https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADhosts_without_controversies.txt",
+        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts",
+        "https://v.firebog.net/hosts/static/w3kbl.txt",
+        "https://www.dshield.org/feeds/suspiciousdomains_Low.txt",
+        "https://www.dshield.org/feeds/suspiciousdomains_Medium.txt",
+        "https://www.dshield.org/feeds/suspiciousdomains_High.txt",
+        "https://raw.githubusercontent.com/matomo-org/referrer-spam-blacklist/master/spammers.txt",
+        "https://someonewhocares.org/hosts/zero/hosts",
+        "https://raw.githubusercontent.com/vokins/yhosts/master/hosts",
+        "https://winhelp2002.mvps.org/hosts.txt",
+        "https://hosts.nfz.moe/basic/hosts",
+        "https://raw.githubusercontent.com/RooneyMcNibNug/pihole-stuff/master/SNAFU.txt",
+        "https://ssl.bblck.me/blacklists/hosts-file.txt"
     ]
 }

dns.go

@@ -24,6 +24,16 @@ type dnsHandler struct {
 //NewDNSHandler creates a new DNS server handler
 func NewDNSHandler(blockForwardIP string, cache DNSCacher, blocklist BlocklistManager, r DNSResolver, cfg Configuration) (dns.Handler, error) {
 	l := log.New(os.Stdout, "[DNS Server] ", log.LUTC|log.Lmicroseconds|log.Lshortfile)
+
+	if cfg.UseHosts {
+		hostsConf, _ := dns.ClientConfigFromFile("/etc/resolv.conf")
+		hostsServers := make([]net.IP, len(hostsConf.Servers))
+		for idx, hip := range hostsConf.Servers {
+			hostsServers[idx] = net.ParseIP(hip)
+		}
+		r = append(r, hostsServers...)
+	}
+
 	return &dnsHandler{
 		logger:         l,
 		blockForwardIP: net.ParseIP(blockForwardIP),

http.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"log"
 	"net/http"
 	"os"
@@ -8,18 +9,18 @@ import (
 )
 
 //NewHTTPHandler creates a new http handler
-func NewHTTPHandler(bm BlocklistManager) http.Handler {
-	return newAPIHandler(nil, bm)
+func NewHTTPHandler(cfg Configuration, bm BlocklistManager) http.Handler {
+	return newAPIHandler(nil, cfg, bm)
 }
 
-func newAPIHandler(inner http.Handler, bm BlocklistManager) http.Handler {
+func newAPIHandler(inner http.Handler, cfg Configuration, bm BlocklistManager) http.Handler {
 	l := log.New(os.Stdout, "[HTTP API] ", log.LUTC|log.Lshortfile)
 	apiHandler := http.NewServeMux()
 
 	/*
 		1. update block lists
 	*/
-	apiHandler.HandleFunc("/api/gopherhole/blocklists/reload", func(w http.ResponseWriter, req *http.Request) {
+	apiHandler.HandleFunc("/blocklists/reload", func(w http.ResponseWriter, req *http.Request) {
 		if req.Method != http.MethodPost {
 			http.Error(w, "method not allowed, must be a POST", http.StatusMethodNotAllowed)
 			return
@@ -32,9 +33,25 @@ func newAPIHandler(inner http.Handler, bm BlocklistManager) http.Handler {
 		l.Println("reloaded blocklists successfully")
 	})
 
+	apiHandler.HandleFunc("/config", func(w http.ResponseWriter, req *http.Request) {
+		switch req.Method {
+		case http.MethodGet:
+			w.Header().Set("Content-Type", "application/json")
+			encoder := json.NewEncoder(w)
+			encoder.SetIndent("", "    ")
+
+			if err := encoder.Encode(cfg); err != nil {
+				log.Printf("could not send config: %v", err)
+			}
+		default:
+			http.Error(w, "method not allowed, must be a POST", http.StatusMethodNotAllowed)
+		}
+	})
+
+	h := http.StripPrefix("/api/gopherhole", apiHandler)
 	return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
-		if strings.HasPrefix(req.Host, "/api/gopherhole") {
-			apiHandler.ServeHTTP(res, req)
+		if strings.HasPrefix(req.URL.Path, "/api/gopherhole") {
+			h.ServeHTTP(res, req)
 			return
 		}
 

main.go

@@ -36,7 +36,7 @@ func main() {
 	go func(bm BlocklistManager) {
 		httpAddr := fmt.Sprintf("%s:80", *httpAddress)
 		log.Printf("HTTP server listening @ %s", httpAddr)
-		handler := NewHTTPHandler(bm)
+		handler := NewHTTPHandler(*cfg, bm)
 
 		if err := http.ListenAndServe(httpAddr, handler); err != nil {
 			log.Fatal(err)