initial commit

pull/1/head
Adam Veldhousen 3 years ago
commit ee6b8def4a
Signed by: adam
GPG Key ID: 6DB29003C6DD1E4B

1
.gitignore vendored

@ -0,0 +1 @@
.bin

@ -0,0 +1,19 @@
# Gopherhole
[![GoDoc Reference](https://godoc.org/github.com/adamveld12/gopherhole?status.svg)](http://godoc.org/github.com/adamveld12/gopherhole)
[![GitHub Actions](https://github.com/adamveld12/gopherhole/workflows/Go/badge.svg)](https://github.com/adamveld12/gopherhole/actions?query=workflow%3AGo)
[![Go Report Card](https://goreportcard.com/badge/github.com/adamveld12/gopherhole)](https://goreportcard.com/report/github.com/adamveld12/gopherhole)
Fully customizable DNS server.
- HTTP API to configure on the fly
- Supports Redis/local in memory caching
- Access logging
- Custom rules
Compatible with pihole block lists: https://firebog.net/
## LICENSE
MIT

@ -0,0 +1,131 @@
{
"database": "./db.sqlite",
"cache": "in-memory",
"http-addr": "localhost:8080",
"dns-addr": "localhost:5353",
"recursors": ["192.168.1.15:8600", "1.1.1.1", "8.8.8.8"],
"rules": [
{
"name": "internal veldhousen.ninja",
"value": "veldhousen.(ninja|internal)",
"answer": { "type": "A", "value": "192.168.1.15" },
"ttl": 300
}
],
"rule-lists": [
{
"name": "Polish Filters Team KADhosts",
"url": "https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADhosts.txt"
},
{
"name": "FadeMind hosts",
"url": "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts"
},
{
"name": "W3kbl",
"url": "https://v.firebog.net/hosts/static/w3kbl.txt"
},
{ "name": "Adaway", "url": "https://adaway.org/hosts.txt" },
{
"name": "AdguardDNS",
"url": "https://v.firebog.net/hosts/AdguardDNS.txt"
},
{ "name": "Admiral", "url": "https://v.firebog.net/hosts/Admiral.txt" },
{
"name": "anudeepnd blacklist",
"url": "https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt"
},
{
"name": "lists.disconnect.me simple ad",
"url": "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt"
},
{
"name": "EasyList",
"url": "https://v.firebog.net/hosts/Easylist.txt"
},
{
"name": "yoyo.org adservers",
"url": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext"
},
{
"name": "FadeMind UncheckyAds",
"url": "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts"
},
{
"name": "hostsVN",
"url": "https://raw.githubusercontent.com/bigdargon/hostsVN/master/hosts"
},
{
"name": "EasyPrivacy",
"url": "https://v.firebog.net/hosts/Easyprivacy.txt"
},
{
"name": "Prigent-Ads",
"url": "https://v.firebog.net/hosts/Prigent-Ads.txt"
},
{
"name": "FadeMind add.207Net",
"url": "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts"
},
{
"name": "WindowsSpyBlocker",
"url": "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt"
},
{
"name": "Frogeye firstparty trackers",
"url": "https://hostfiles.frogeye.fr/firstparty-trackers-hosts.txt"
},
{
"name": "W3C Annual TOP EU US Ads Trackers",
"url": "https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/TOP_EU_US_Ads_Trackers_HOST"
},
{
"name": "DandelionSprout adlift",
"url": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt"
},
{
"name": "Threat Intel latest domains",
"url": "https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt"
},
{
"name": "Disconnect.me simple malvertising",
"url": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt"
},
{
"name": "Prigent Crypto",
"url": "https://v.firebog.net/hosts/Prigent-Crypto.txt"
},
{
"name": "Mandiant APT1 Report Appendix D.",
"url": "https://bitbucket.org/ethanr/dns-blacklists/raw/8575c9f96e5b4a1308f2f12394abd86d0927a4a0/bad_lists/Mandiant_APT1_Report_Appendix_D.txt"
},
{
"name": "phishing army blocklist ext",
"url": "https://phishing.army/download/phishing_army_blocklist_extended.txt"
},
{
"name": "notrack malware",
"url": "https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt"
},
{
"name": "Shalla-mal",
"url": "https://v.firebog.net/hosts/Shalla-mal.txt"
},
{
"name": "Spam404 main-blacklist",
"url": "https://raw.githubusercontent.com/Spam404/lists/master/main-blacklist.txt"
},
{
"name": "FadeMind Risky",
"url": "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts"
},
{
"name": "urlhaus abuse",
"url": "https://urlhaus.abuse.ch/downloads/hostfile/"
},
{
"name": "Coin Blocker Lists",
"url": "https://zerodot1.gitlab.io/CoinBlockerLists/hosts_browser"
}
]
}

@ -0,0 +1,31 @@
package main
import (
"encoding/json"
"fmt"
"os"
"github.com/adamveld12/gopherhole/internal"
)
type StartupConfig struct {
DatabaseURL string `json:"database"`
CacheURL string `json:"cache"`
HTTPAddr string `json:"http-addr"`
DNSAddr string `json:"dns-addr"`
Recursors []string `json:"recursors"`
Rules []internal.Rule `json:"rules"`
}
func LoadStartupConfig(conf *StartupConfig, file string) error {
data, err := os.Open(file)
if err != nil {
return fmt.Errorf("could not open file: %w", err)
}
if err := json.NewDecoder(data).Decode(conf); err != nil {
return fmt.Errorf("could not read json file: %w", err)
}
return nil
}

@ -0,0 +1,8 @@
module github.com/adamveld12/gopherhole
go 1.15
require (
github.com/mattn/go-sqlite3 v1.14.7
github.com/miekg/dns v1.1.41
)

@ -0,0 +1,13 @@
github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04 h1:cEhElsAv9LUt9ZUUocxzWe05oFLVd+AA2nstydTeI8g=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

@ -0,0 +1,57 @@
package internal
import (
"sync"
"time"
"github.com/miekg/dns"
)
type Cache interface {
LookupRecord(name string) []dns.RR
SaveAnswers(name string, answers []dns.RR)
}
type Memory struct {
sync.Mutex
cache map[string]struct {
Answers []dns.RR
Expiration time.Time
}
}
func (m *Memory) init() {
if m.cache == nil {
m.Lock()
m.cache = make(map[string]struct {
Answers []dns.RR
Expiration time.Time
})
m.Unlock()
}
}
func (m *Memory) LookupRecord(name string) []dns.RR {
m.init()
if v, ok := m.cache[name]; ok && time.Until(v.Expiration) > 0 {
return v.Answers
}
return nil
}
func (m *Memory) SaveAnswers(name string, answers []dns.RR) {
if answers == nil || name == "" {
return
}
m.init()
m.Lock()
defer m.Unlock()
ttl := time.Second * time.Duration(answers[0].Header().Ttl)
m.cache[name] = struct {
Answers []dns.RR
Expiration time.Time
}{Answers: answers, Expiration: time.Now().Add(ttl)}
}

@ -0,0 +1,98 @@
package internal
import (
"log"
"time"
"github.com/miekg/dns"
)
type DomainManager struct {
Cache
Storage
RuleEvaluator
Recursors Recursor
}
func (dm *DomainManager) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
// only grab first question: https://stackoverflow.com/questions/4082081/requesting-a-and-aaaa-records-in-single-dns-query/4083071#4083071
start := time.Now()
q := r.Question[0]
ql := QueryLog{
Started: start.UTC(),
Protocol: w.RemoteAddr().Network(),
ClientIP: w.RemoteAddr().String(),
Domain: q.Name,
}
var responseMessage *dns.Msg
var err error
// lookup in cache
if dest := dm.LookupRecord(q.Name); dest != nil {
responseMessage = new(dns.Msg)
responseMessage.Answer = dest
ql.Status = CacheHit
}
// lookup in rules engine
if rule, ok := dm.Evaluate(q.Name); ok {
responseMessage = rule.CreateAnswer(q.Name)
responseMessage.Authoritative = true
ql.Status = CustomRule
}
// recurse
if responseMessage == nil {
var resolved Resolved
if resolved, err = dm.Recursors.Resolve(r); err == nil {
dm.SaveAnswers(q.Name, resolved.Message.Answer)
ql.RecurseUpstreamIP = resolved.UpstreamUsed
ql.RecurseRoundTripTimeMs = int(resolved.RoundtripTime.Milliseconds())
ql.Status = RecursedUpstream
responseMessage = resolved.Message
} else {
ql.Status = NoAnswer
responseMessage = new(dns.Msg)
}
}
responseMessage.SetReply(r)
responseMessage.RecursionAvailable = true
responseMessage.Compress = true
ql.TotalTimeMs = int(time.Since(start).Milliseconds())
log.Printf("%+v", ql)
go func() {
if err := dm.Storage.Log(ql); err != nil {
log.Printf("ERROR WRITING LOG: %v", err)
}
}()
if err := w.WriteMsg(responseMessage); err != nil {
log.Println(err)
}
}
type ResponseStatus string
const (
CacheMiss = ResponseStatus("CACHE MISS")
CacheHit = ResponseStatus("CACHE HIT")
CustomRule = ResponseStatus("CUSTOM RULE")
RecursedUpstream = ResponseStatus("RECURSED")
NoAnswer = ResponseStatus("NO ANSWER")
)
type QueryLog struct {
Started time.Time
ClientIP string
Protocol string
Domain string
TotalTimeMs int
RecurseRoundTripTimeMs int
RecurseUpstreamIP string
Error error
Status ResponseStatus
}

@ -0,0 +1,47 @@
package internal
import (
"context"
"time"
"github.com/miekg/dns"
)
type DNSServer struct {
srv *dns.Server
dns.Handler
}
func (d *DNSServer) ListenAndServe(ctx context.Context, addr string) error {
errs := make(chan error)
c, canceller := context.WithCancel(ctx)
go d.runServer(c, "udp", addr, errs)
go d.runServer(c, "tcp", addr, errs)
defer canceller()
select {
case <-ctx.Done():
return nil
case err := <-errs:
return err
}
}
func (d *DNSServer) runServer(ctx context.Context, net, addr string, in chan<- error) {
d.srv = &dns.Server{
Addr: addr,
Net: net,
IdleTimeout: func() time.Duration { return time.Second * 5 },
Handler: d,
}
go func() {
<-ctx.Done()
d.srv.Listener.Close()
}()
if err := d.srv.ListenAndServe(); err != nil {
in <- err
}
}

@ -0,0 +1,45 @@
package internal
import (
"fmt"
"time"
"github.com/miekg/dns"
)
type Recursor struct {
Upstreams []string
Client *dns.Client
}
type Resolved struct {
RoundtripTime time.Duration
UpstreamUsed string
Attempts int
Message *dns.Msg
}
func (r Recursor) Resolve(request *dns.Msg) (Resolved, error) {
var result Resolved
errs := make([]error, len(r.Upstreams))
var err error
var upstreamsTried int
for _, upstream := range r.Upstreams {
result.Message, result.RoundtripTime, err = r.Client.Exchange(request, upstream)
upstreamsTried++
if err == nil {
result.UpstreamUsed = upstream
result.Message.RecursionAvailable = true
break
}
errs = append(errs, err)
}
result.Attempts = upstreamsTried
if result.Message == nil && len(errs) > 0 {
return result, fmt.Errorf("errors occured resolving domain: %+v", errs)
}
return result, nil
}

@ -0,0 +1,78 @@
package internal
import (
"net"
"regexp"
"github.com/miekg/dns"
)
type RuleType string
const (
A = RuleType("A")
CNAME = RuleType("CNAME")
Recurse = RuleType("recurse")
)
type RuleEvaluator interface {
Evaluate(string) (Rule, bool)
}
type RuleEngine struct {
Rules []Rule
}
func (re *RuleEngine) Evaluate(domain string) (Rule, bool) {
for _, r := range re.Rules {
regex := regexp.MustCompile(r.Value)
if regex.MatchString(domain) {
return r, true
}
}
return Rule{}, false
}
type Rule struct {
Name string `json:"name"`
Value string `json:"value"`
Answer struct {
Type RuleType
Value string
} `json:"answer"`
TTL int `json:"ttl"`
}
func getType(rt RuleType) uint16 {
switch rt {
case A:
return dns.TypeA
case CNAME:
return dns.TypeCNAME
}
return 0
}
func (r Rule) CreateAnswer(name string) *dns.Msg {
return &dns.Msg{
MsgHdr: dns.MsgHdr{},
Compress: false,
Question: []dns.Question{},
Answer: []dns.RR{
&dns.A{
Hdr: dns.RR_Header{
Name: name,
Rrtype: getType(r.Answer.Type),
Class: dns.ClassINET,
Ttl: uint32(r.TTL),
},
A: net.ParseIP(r.Answer.Value).To4(),
},
},
Ns: []dns.RR{},
Extra: []dns.RR{},
}
}

@ -0,0 +1,166 @@
package internal
import (
"database/sql"
"errors"
"fmt"
"io"
"time"
_ "github.com/mattn/go-sqlite3"
)
const ISO8601 = "2006-01-02 15:04:05.999"
type Storage interface {
io.Closer
Open() error
// AddRecursor() error
// AddRule(Rule) error
Log(QueryLog) error
GetLog(GetLogInput) ([]QueryLog, error)
}
type Sqlite struct {
Path string
*sql.DB
}
type GetLogInput struct {
Start time.Time
End time.Time
DomainFilter string
Limit uint
Page uint
}
func (ss *Sqlite) GetLog(in GetLogInput) ([]QueryLog, error) {
if in.Limit <= 0 {
in.Limit = 100
}
if in.Start.IsZero() {
in.Start = time.Now().UTC().Add(time.Hour * -24)
}
if in.End.IsZero() {
in.End = time.Now().UTC()
}
sql := `
SELECT
started, clientIp, protocol, domain, totalTimeMs, error, recurseRoundTripTimeMs, recurseUpstreamIp, status
FROM
log
WHERE
id > ? AND started > ? AND started < ?
ORDER BY started
LIMIT ?;
`
rows, err := ss.DB.Query(sql, in.Page*in.Limit, in.Start.Format(ISO8601), in.End.Format(ISO8601), in.Limit)
if err != nil {
return nil, err
}
defer rows.Close()
var ql []QueryLog
for rows.Next() {
var q QueryLog
var errStr string
var started string
rows.Scan(
&started,
&q.ClientIP,
&q.Protocol,
&q.Domain,
&q.TotalTimeMs,
&errStr,
&q.RecurseRoundTripTimeMs,
&q.RecurseUpstreamIP,
&q.Status,
)
if errStr != "" {
q.Error = errors.New(errStr)
}
if q.Started, err = time.Parse(ISO8601, started); err != nil {
return nil, fmt.Errorf("could not parse time '%s': %v", started, err)
}
ql = append(ql, q)
}
return ql, nil
}
func (ss *Sqlite) Log(ql QueryLog) error {
sql := `
INSERT INTO log
(started, clientIp, protocol, domain, totalTimeMs, error, recurseRoundTripTimeMs, recurseUpstreamIp, status)
VALUES
(?, ?, ?, ?, ?, ?, ?, ?, ?);
`
var errStr string
if ql.Error != nil {
errStr = ql.Error.Error()
}
if _, err := ss.DB.Exec(sql,
ql.Started.Format(ISO8601),
ql.ClientIP,
ql.Protocol,
ql.Domain,
ql.TotalTimeMs,
errStr,
ql.RecurseRoundTripTimeMs,
ql.RecurseUpstreamIP,
ql.Status,
); err != nil {
return err
}
return nil
}
func (ss *Sqlite) Open() error {
db, err := sql.Open("sqlite3", fmt.Sprintf("%s?cache=shared", ss.Path))
if err != nil {
return fmt.Errorf("could not open db: %w", err)
}
db.SetMaxOpenConns(1)
ss.DB = db
if err := initTable(db); err != nil {
return err
}
return nil
}
func initTable(db *sql.DB) error {
sql := `
CREATE TABLE IF NOT EXISTS log (
id INTEGER PRIMARY KEY,
started TEXT NOT NULL,
clientIp TEXT NOT NULL,
protocol TEXT NOT NULL,
domain TEXT NOT NULL,
totalTimeMs int NOT NULL,
error TEXT,
recurseRoundTripTimeMs INT,
recurseUpStreamIP TEXT,
status TEXT NOT NULL
);
`
if _, err := db.Exec(sql); err != nil {
return fmt.Errorf("could not initialize db: %w", err)
}
return nil
}

@ -0,0 +1,82 @@
package main
import (
"context"
"flag"
"fmt"
"log"
"regexp"
"strings"
"time"
"github.com/adamveld12/gopherhole/internal"
"github.com/miekg/dns"
)
var (
configFilePath = flag.String("config", "./config.json", "Config file")
)
func main() {
flag.Parse()
var conf StartupConfig
if err := LoadStartupConfig(&conf, *configFilePath); err != nil {
log.Fatalf("%+v", err)
}
log.Printf("%+v", conf)
store := &internal.Sqlite{
Path: conf.DatabaseURL,
}
if err := store.Open(); err != nil {
log.Fatalf("COULD NOT OPEN SQLITE DB: %v", err)
}
re := &internal.RuleEngine{
Rules: conf.Rules,
}
dnsClient := &dns.Client{
Net: "udp",
DialTimeout: time.Millisecond * 250,
ReadTimeout: time.Second,
WriteTimeout: time.Second,
}
dm := &internal.DomainManager{
Cache: &internal.Memory{},
Storage: store,
RuleEvaluator: re,
Recursors: internal.Recursor{
Upstreams: cleanRecursors(conf.Recursors),
Client: dnsClient,
},
}
dnsSrv := &internal.DNSServer{Handler: dm}
if err := dnsSrv.ListenAndServe(context.Background(), conf.DNSAddr); err != nil {
log.Fatal(err)
}
}
func cleanRecursors(recursors []string) []string {
cr := []string{}
reg := regexp.MustCompile(`^((?:\d{1,4}\.?){4})(?::(\d{0,5}))?`)
for _, r := range recursors {
if !reg.MatchString(r) {
log.Fatalf("%s is not a valid DNS server. Must be in ip:addr format.", r)
}
cleanedIPAddr := r
if !strings.Contains(r, ":") {
cleanedIPAddr = fmt.Sprintf("%s:53", r)
}
cr = append(cr, cleanedIPAddr)
}
log.Println(cr)
return cr
}

@ -0,0 +1,17 @@
dev: clean .bin/gopherhole .bin/config.json
cd .bin && ./gopherhole -config config.json
clean:
@rm -rf .bin
.PHONY: clean dev
.bin:
mkdir -p .bin
.bin/gopherhole: .bin
# @go build --tags "sqlite_foreign_keys fts5" -v -o .bin/gopherhole .
@go build --tags "fts5" -v -o .bin/gopherhole .
.bin/config.json:
@cp ./config.example.json .bin/config.json
Loading…
Cancel
Save