refactoring

pull/1/head
Adam Veldhousen 3 years ago
parent 943da8807b
commit 1c43e4479a
Signed by: adam
GPG Key ID: 6DB29003C6DD1E4B

@ -0,0 +1,31 @@
FROM node:lts-alpine as build-client
COPY . /build/
RUN apk add --no-cache make
RUN make build-client
FROM golang:alpine as build-server
COPY --from=build-client /build /build
RUN apk add --no-cache make
RUN make .bin/gopherhole
FROM alpine
RUN apl add --no-cache ca-certificates
RUN addgroup -g 1000 gopherhole \
&& adduser -H -D -u 1000 gopherhole gopherhole
COPY --chown=gopherhole:gopherhole --from=build-server /build/.bin/gopherhole /opt/gopherhole
USER gopherhole
VOLUME "/data"
ENTRYPOINT /opt/gopherhole

File diff suppressed because it is too large Load Diff

@ -1,5 +1,6 @@
import { sub, format } from 'date-fns';
import { readable } from "svelte/store";
import { API_HOST } from './util';
export const SearchDateFormatStr = "yyyy-MM-dd HH:mm:ss.SSS";
export const formatDateForSearch = date => format(date, SearchDateFormatStr);
@ -26,9 +27,9 @@ export const fetchMetrics = async function ({
} = {}) {
try {
const response = await fetch(
`http://localhost:8080/api/v1/metrics/stats?start=${formatDateForSearch(
API_HOST(`/metrics/stats?start=${formatDateForSearch(
start
)}&end=${formatDateForSearch(end)}&key=${key}&interval=${interval}`,
)}&end=${formatDateForSearch(end)}&key=${key}&interval=${interval}`),
{
method: "GET",
headers: { Accept: "application/json" }

@ -5,7 +5,7 @@ interface APIResponse<T> {
}
export const API_HOST = `http://localhost:8080/api/v1`;
export const API_HOST = (url = "") => `http://localhost:8000/api/v1/${url}`;
export const apiCall = async function<T>(url: string, method: string = 'GET'): Promise<APIResponse<T>> {
try {

@ -47,5 +47,8 @@
{row.TotalTimeMs}
</Column>
</Table>
{:else}
<p>No Logs yet!</p>
<p><em>TODO:</em> Link to docs on how to point your router at this server</p>
{/if}
</div>

@ -1,7 +1,7 @@
{
"database": "./db.sqlite",
"cache": "in-memory",
"http-addr": "localhost:8080",
"http-addr": "localhost:8000",
"dns-addr": "localhost:5353",
"recursors": ["192.168.1.15:8600", "1.1.1.1", "8.8.8.8"],
"rules": [

Binary file not shown.

@ -0,0 +1,26 @@
package internal
import "context"
var (
LogUpdated = EventType("LOG_UPDATE")
DNSLog = EventSource("DNS_LOG")
)
type EventType string
type EventSource string
type Event struct {
Type EventType `json:"type"`
Source EventSource `json:"source"`
Payload interface{} `json:"payload"`
}
type EventPublisher interface {
Publish(Event)
}
type EventSubscriber interface {
Subscribe(context.Context) <-chan Event
}

@ -1,12 +1,8 @@
package internal
import (
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"strconv"
"time"
"github.com/go-chi/chi/v5"
@ -23,6 +19,8 @@ var upgrader = websocket.Upgrader{
type adminHandler struct {
Cache
Storage
EventSubscriber
EventPublisher
*RuleEngine
h http.Handler
}
@ -57,17 +55,23 @@ func NewAdminHandler(c Cache, s Storage, re *RuleEngine) http.Handler {
r.Get("/metrics/log", RestHandler(a.getLog).ToHF())
r.Get("/metrics/stats", RestHandler(a.getStats).ToHF())
r.Get("/rules", RestHandler(a.getRules).ToHF())
r.Put("/rules", RestHandler(a.createRule).ToHF())
r.Get("/rules", RestHandler(a.getRules).ToHF())
r.Get("/rules/{id:[0-9]+}", RestHandler(a.getRule).ToHF())
r.Post("/rules/{id:[0-9]+}", RestHandler(a.updateRule).ToHF())
r.Delete("/rules/{id:[0-9]+}", RestHandler(a.deleteRule).ToHF())
r.Get("/recursors", RestHandler(a.getRecursors).ToHF())
r.Put("/recursors", RestHandler(a.addRecursor).ToHF())
r.Get("/recursors", RestHandler(a.getRecursors).ToHF())
r.Get("/recursors/{id:[0-9]+}", RestHandler(a.getRecursor).ToHF())
r.Delete("/recursor/{id:[0-9]+}", RestHandler(a.deleteRecursor).ToHF())
// r.Put("/rules/lists", a.addRulelist)
// r.Get("/rules/lists", a.getRuleLists)
// r.Delete("/rules/lists/{id}", a.deleteRuleList)
// r.Post("/rules/lists/reload/{id}", a.reloadRuleLists)
// r.Delete("/rules/lists/{id}", a.deleteRuleList)
// r.Get("/updates", RestHandler(a.getStats).ToHF())
// r.Delete("/cache/purgeall", RestHandler(a.purgeAll).ToHF())
// r.Delete("/cache/purge", a.purgeKey)
// r.Get("/cache", a.getCacheContents)
@ -75,7 +79,6 @@ func NewAdminHandler(c Cache, s Storage, re *RuleEngine) http.Handler {
})
return a
}
func (a *adminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -91,319 +94,7 @@ func (a *adminHandler) signal(w http.ResponseWriter, r *http.Request) {
defer c.Close()
for {
// send any updates that come through to the client
}
}
func (a *adminHandler) addRecursor(r *http.Request) (*RestResponse, error) {
var recursorHttpInput RecursorRow
if err := json.NewDecoder(r.Body).Decode(&recursorHttpInput); err != nil {
return nil, err
}
if ipAddr, port, ok := recursorHttpInput.ValidIp(); ok {
if err := a.Storage.AddRecursors(ipAddr, port, recursorHttpInput.TimeoutMs, recursorHttpInput.Weight); err != nil {
return nil, err
}
}
return &RestResponse{
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{
Success: true,
Payload: recursorHttpInput,
},
}, nil
}
func (a *adminHandler) getRecursors(r *http.Request) (*RestResponse, error) {
recursors, err := a.Storage.GetRecursors()
if err != nil {
return &RestResponse{
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{
Payload: err.Error(),
},
}, err
}
return &RestResponse{
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{
Success: true,
Payload: recursors,
},
}, nil
}
func (a *adminHandler) deleteRule(r *http.Request) (*RestResponse, error) {
ruleIdParam := chi.URLParam(r, "id")
ruleId, err := strconv.Atoi(ruleIdParam)
if err != nil {
return &RestResponse{
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{
Success: false,
Payload: "Invalid rule ID",
},
}, nil
}
if err := a.Storage.DeleteRule(ruleId); err != nil {
return &RestResponse{
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{
Success: false,
Payload: err.Error(),
},
}, nil
}
return &RestResponse{
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{
Success: true,
Payload: ruleId,
},
}, nil
}
func (a *adminHandler) createRule(r *http.Request) (*RestResponse, error) {
var rr RuleRow
if err := json.NewDecoder(r.Body).Decode(&rr); err != nil {
return &RestResponse{
Status: http.StatusUnprocessableEntity,
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{
Success: false,
Payload: err.Error(),
},
}, nil
}
if err := a.Storage.AddRule(rr); err != nil {
return &RestResponse{
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{
Success: false,
Payload: err.Error(),
},
}, nil
}
return &RestResponse{
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{
Success: true,
},
}, nil
}
func (a *adminHandler) getRules(r *http.Request) (*RestResponse, error) {
results, err := a.Storage.GetRules()
if err != nil {
return nil, err
}
if len(results) <= 0 {
results = []RuleRow{}
}
return &RestResponse{
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{
Success: true,
Payload: results,
},
}, nil
}
func (a *adminHandler) getStats(r *http.Request) (*RestResponse, error) {
q := r.URL.Query()
startFilter := q.Get("start")
endFilter := q.Get("end")
key := q.Get("key")
intervalSecondsStr := q.Get("interval")
var err error
start := time.Now().UTC().Add(time.Hour * -86400)
end := time.Now().UTC()
if startFilter != "" {
if start, err = time.Parse(ISO8601, startFilter); err != nil {
return nil, err
}
}
if endFilter != "" {
if end, err = time.Parse(ISO8601, endFilter); err != nil {
return nil, err
}
}
lai := LogAggregateInput{
Start: start,
End: end,
Column: key,
}
if intervalSecondsStr != "" {
if lai.IntervalSeconds, err = strconv.Atoi(intervalSecondsStr); err != nil {
return nil, errors.New("interval query param must be a valid whole number")
}
}
la, err := a.Storage.GetLogAggregate(lai)
if err != nil {
return nil, err
}
return &RestResponse{
Status: http.StatusOK,
Payload: struct {
Success bool `json:"success"`
Payload interface{} `json:"payload"`
}{
Success: true,
Payload: la,
},
}, nil
}
func (a *adminHandler) getLog(r *http.Request) (*RestResponse, error) {
q := r.URL.Query()
startFilter := q.Get("start")
endFilter := q.Get("end")
// filter := q.Get("filter")
pageStr := q.Get("page")
var err error
var page int
start := time.Now().UTC().Add(time.Hour * -86400)
end := time.Now().UTC()
if startFilter != "" {
if start, err = time.Parse(ISO8601, startFilter); err != nil {
return nil, err
}
}
if endFilter != "" {
if end, err = time.Parse(ISO8601, endFilter); err != nil {
return nil, err
}
}
if pageStr != "" {
page, _ = strconv.Atoi(pageStr)
}
ql, err := a.Storage.GetLog(GetLogInput{
Start: start,
End: end,
Limit: 250,
Page: page,
})
if err != nil {
return nil, err
}
return &RestResponse{
Status: http.StatusOK,
Payload: struct {
Success bool `json:"success"`
Payload interface{} `json:"payload"`
}{
Success: true,
Payload: ql,
},
}, nil
}
type RestResponse struct {
Status int
Headers http.Header
Payload struct {
Success bool `json:"success"`
Payload interface{} `json:"payload"`
}
}
func (rr *RestResponse) Write(w http.ResponseWriter) error {
if rr.Status != 0 && rr.Status != 200 {
w.WriteHeader(rr.Status)
}
for k, v := range rr.Headers {
for _, ve := range v {
w.Header().Add(k, ve)
}
}
e := json.NewEncoder(w)
e.SetIndent("\n", "\t")
if err := e.Encode(rr.Payload); err != nil {
return fmt.Errorf("could not serialize struct for http response: %w", err)
}
return nil
}
type RestHandler func(request *http.Request) (*RestResponse, error)
func (rh RestHandler) ToHF() http.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) {
rid := r.Context().Value(middleware.RequestIDKey)
rw.Header().Set(middleware.RequestIDHeader, rid.(string))
rh.ServeHTTP(rw, r)
}
}
func (rh RestHandler) Error(e error) *RestResponse {
return &RestResponse{
Status: http.StatusInternalServerError,
Payload: struct {
Success bool `json:"success"`
Payload interface{} `json:"payload"`
}{
Success: false,
Payload: e.Error(),
},
}
}
func (r RestHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
response, err := r(req)
if err != nil {
response = r.Error(err)
}
if err := response.Write(w); err != nil {
log.Printf("Error occurred handling rest response: %v", err)
// send any updates that come through to the client
}
}

@ -0,0 +1,42 @@
package internal
import (
"encoding/json"
"net/http"
)
func (a *adminHandler) addRecursor(r *http.Request) (*RestResponse, error) {
var recursorHttpInput RecursorRow
if err := json.NewDecoder(r.Body).Decode(&recursorHttpInput); err != nil {
return ErrorResponse(http.StatusUnprocessableEntity, err), nil
}
ipAddr, port, ok := recursorHttpInput.ValidIp()
if !ok {
return BasicResponse(false, "The ip address provided is invalid."), nil
}
if err := a.Storage.AddRecursors(ipAddr, port, recursorHttpInput.TimeoutMs, recursorHttpInput.Weight); err != nil {
return nil, err
}
return BasicResponse(true, recursorHttpInput), nil
}
func (a *adminHandler) getRecursors(r *http.Request) (*RestResponse, error) {
recursors, err := a.Storage.GetRecursors()
if err != nil {
return nil, err
}
return BasicResponse(true, recursors), nil
}
func (a *adminHandler) getRecursor(r *http.Request) (*RestResponse, error) {
return BasicResponse(false, "Nothing here yet"), nil
}
func (a *adminHandler) deleteRecursor(r *http.Request) (*RestResponse, error) {
return BasicResponse(false, "Nothing here yet"), nil
}

@ -0,0 +1,89 @@
package internal
import (
"encoding/json"
"net/http"
"strconv"
"github.com/go-chi/chi"
)
func (a *adminHandler) createRule(r *http.Request) (*RestResponse, error) {
var rr RuleRow
if err := json.NewDecoder(r.Body).Decode(&rr); err != nil {
return ErrorResponse(http.StatusUnprocessableEntity, err), nil
}
if err := a.Storage.AddRule(rr); err != nil {
return nil, err
}
return BasicResponse(true, nil), nil
}
func (a *adminHandler) getRule(r *http.Request) (*RestResponse, error) {
ruleIdParam := chi.URLParam(r, "id")
ruleId, err := strconv.Atoi(ruleIdParam)
if err != nil {
return BasicResponse(false, "`id` must be a valid integer."), nil
}
results, err := a.Storage.GetRule(ruleId)
if err != nil {
return nil, err
}
return BasicResponse(true, results), nil
}
func (a *adminHandler) getRules(r *http.Request) (*RestResponse, error) {
results, err := a.Storage.GetRules()
if err != nil {
return nil, err
}
if len(results) <= 0 {
results = []RuleRow{}
}
return BasicResponse(true, results), nil
}
func (a *adminHandler) deleteRule(r *http.Request) (*RestResponse, error) {
ruleIdParam := chi.URLParam(r, "id")
ruleId, err := strconv.Atoi(ruleIdParam)
if err != nil {
return BasicResponse(false, "`id` must be a valid integer"), nil
}
if err := a.Storage.DeleteRule(ruleId); err != nil {
return nil, err
}
rule, err := a.Storage.GetRule(ruleId)
if err != nil {
return nil, err
}
return BasicResponse(true, rule), nil
}
func (a *adminHandler) updateRule(r *http.Request) (*RestResponse, error) {
var rr RuleRow
if err := json.NewDecoder(r.Body).Decode(&rr); err != nil {
return ErrorResponse(http.StatusUnprocessableEntity, err), nil
}
if err := a.UpdateRule(rr.ID, rr); err != nil {
return nil, err
}
rule, err := a.GetRule(rr.ID)
if err != nil {
return nil, err
}
return BasicResponse(true, rule), nil
}

@ -0,0 +1,107 @@
package internal
import (
"fmt"
"net/http"
"strconv"
"time"
)
func (a *adminHandler) getStats(r *http.Request) (*RestResponse, error) {
q := r.URL.Query()
startFilter := q.Get("start")
endFilter := q.Get("end")
key := q.Get("key")
intervalSecondsStr := q.Get("interval")
var err error
start := time.Now().UTC().Add(time.Hour * -86400)
end := time.Now().UTC()
if startFilter != "" {
if start, err = time.Parse(ISO8601, startFilter); err != nil {
return BasicResponse(false, fmt.Sprintf("start: could not understand value `%v`", err.Error())), nil
}
}
if endFilter != "" {
if end, err = time.Parse(ISO8601, endFilter); err != nil {
return BasicResponse(false, fmt.Sprintf("end: could not understand value `%v`", err.Error())), nil
}
}
lai := LogAggregateInput{
Start: start,
End: end,
Column: key,
}
if intervalSecondsStr != "" {
if lai.IntervalSeconds, err = strconv.Atoi(intervalSecondsStr); err != nil {
return BasicResponse(false, "interval query param must be a valid whole number"), nil
}
}
la, err := a.Storage.GetLogAggregate(lai)
if err != nil {
return nil, err
}
return BasicResponse(true, la), nil
}
type LogFilter struct {
Expression string
}
func (a *adminHandler) getLog(r *http.Request) (*RestResponse, error) {
q := r.URL.Query()
startFilter := q.Get("start")
endFilter := q.Get("end")
// filter := LogFilter{Expression: q.Get("filter")}
pageStr := q.Get("page")
var err error
var page int
start := time.Now().UTC().Add(time.Hour * -86400)
end := time.Now().UTC()
if startFilter != "" {
if start, err = time.Parse(ISO8601, startFilter); err != nil {
return BasicResponse(false, "start: not a valid start time"), nil
}
}
if endFilter != "" {
if end, err = time.Parse(ISO8601, endFilter); err != nil {
return BasicResponse(false, "end: not a valid end time"), nil
}
}
if pageStr != "" {
if page, err = strconv.Atoi(pageStr); err != nil {
return BasicResponse(false, "page: must be a valid integer"), nil
}
}
gli := GetLogInput{
// Filter: filter,
Start: start,
End: end,
Limit: 250,
Page: page,
}
// if err := gli.Validate(); err != nil {
// return BasicResponse(false, err.Error()), nil
// }
ql, err := a.Storage.GetLog(gli)
if err != nil {
return nil, err
}
return BasicResponse(true, ql), nil
}

@ -0,0 +1,98 @@
package internal
import (
"encoding/json"
"fmt"
"log"
"net/http"
"github.com/go-chi/chi/middleware"
)
type RestResponse struct {
Status int
Headers http.Header
Payload struct {
Success bool `json:"success"`
Payload interface{} `json:"payload"`
}
}
func BasicResponse(success bool, model interface{}) *RestResponse {
return &RestResponse{
Status: http.StatusOK,
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{Success: success, Payload: model},
}
}
func ErrorResponse(status int, err error) *RestResponse {
return &RestResponse{
Status: status,
Payload: struct {
Success bool "json:\"success\""
Payload interface{} "json:\"payload\""
}{
Success: false,
Payload: err.Error(),
},
}
}
func (rr *RestResponse) Write(w http.ResponseWriter) error {
if rr.Status != 0 && rr.Status != 200 {
w.WriteHeader(rr.Status)
}
for k, v := range rr.Headers {
for _, ve := range v {
w.Header().Add(k, ve)
}
}
e := json.NewEncoder(w)
e.SetIndent("\n", "\t")
if err := e.Encode(rr.Payload); err != nil {
return fmt.Errorf("could not serialize struct for http response: %w", err)
}
return nil
}
type RestHandler func(request *http.Request) (*RestResponse, error)
func (rh RestHandler) ToHF() http.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) {
rid := r.Context().Value(middleware.RequestIDKey)
rw.Header().Set(middleware.RequestIDHeader, rid.(string))
rh.ServeHTTP(rw, r)
}
}
func (rh RestHandler) Error(e error) *RestResponse {
return &RestResponse{
Status: http.StatusInternalServerError,
Payload: struct {
Success bool `json:"success"`
Payload interface{} `json:"payload"`
}{
Success: false,
Payload: e.Error(),
},
}
}
func (r RestHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
response, err := r(req)
if err != nil {
response = r.Error(err)
}
if err := response.Write(w); err != nil {
log.Printf("Error occurred handling rest response: %v", err)
}
}

@ -18,13 +18,16 @@ type Storage interface {
io.Closer
Open() error
AddRecursors(net.IP, int, int, int) error
DeleteRecursors(int) error
GetRecursors() ([]RecursorRow, error)
// UpdateRule(RuleRow) error
UpdateRecursor(int, RecursorRow) error
DeleteRecursors(int) error
AddRule(RuleRow) error
GetRule(int) (RuleRow, error)
GetRules() ([]RuleRow, error)
UpdateRule(int, RuleRow) error
DeleteRule(int) error
Log(QueryLog) error
GetLog(GetLogInput) ([]QueryLog, error)
GetLogAggregate(LogAggregateInput) ([]LogAggregateDataPoint, error)
@ -97,9 +100,17 @@ func (rr RecursorRow) ValidIp() (net.IP, int, bool) {
return parsedIp, parsedPort, true
}
func (ss *Sqlite) UpdateRecursor(id int, in RecursorRow) error {
sql := `UPDATE recursors SET ipAddress = ?, timeoutMs = ?, weight = ? WHERE id = ?;`
if _, err := ss.Exec(sql, in.IpAddress, in.TimeoutMs, in.Weight); err != nil {
return fmt.Errorf("could not update recursor: %w", err)
}
return nil
}
func (ss *Sqlite) AddRecursors(ip net.IP, port, timeout, weight int) error {
sql := `INSERT INTO recursors (ipAddress, timeoutMs, weight) VALUES (?, ?, ?);`
if _, err := ss.Exec(sql, fmt.Sprintf("%s:%d", ip.String(), port), timeout, weight); err != nil {
return fmt.Errorf("could not insert recursor: %w", err)
}
@ -123,6 +134,24 @@ type RuleRow struct {
Rule
}
func (ss *Sqlite) UpdateRule(id int, in RuleRow) error {
sql := `UPDATE rules SET
name = ?,
expression = ?,
answerType = ?,
answerValue = ?,
ttl = ?,
weight = ?,
enabled = ?
WHERE id = ?;`
if _, err := ss.Exec(sql, in.Name, in.Value, in.Answer.Type, in.Answer.Value, in.TTL, in.Weight, in.Enabled); err != nil {
return fmt.Errorf("could not update rule with id: %v", id)
}
return nil
}
func (ss *Sqlite) AddRule(rr RuleRow) error {
sql := `INSERT INTO rules (name, expression, answerType, answerValue, ttl, weight, enabled, created)
VALUES (?, ? , ?, ?, ?, ?, 1, ?);`
@ -171,6 +200,10 @@ func (ss *Sqlite) GetRules() ([]RuleRow, error) {
}
defer rows.Close()
if rerr := rows.Err(); rerr != nil {
return nil, err
}
var results []RuleRow
for rows.Next() {
var rule RuleRow
@ -231,6 +264,10 @@ func (ss *Sqlite) GetLog(in GetLogInput) ([]QueryLog, error) {
}
defer rows.Close()
if rerr := rows.Err(); rerr != nil {
return nil, rerr
}
var ql []QueryLog
for rows.Next() {
var q QueryLog
@ -367,7 +404,7 @@ func (ss *Sqlite) Log(ql QueryLog) error {
}
func (ss *Sqlite) Open() error {
db, err := sql.Open("sqlite3", fmt.Sprintf("%s?cache=shared", ss.Path))
db, err := sql.Open("sqlite3", fmt.Sprintf("%s?cache=shared&_journal=WAL", ss.Path))
if err != nil {
return fmt.Errorf("could not open db: %w", err)
}

@ -16,6 +16,9 @@ import (
var (
configFilePath = flag.String("config", "./config.json", "Config file")
// dbPath = flag.String("db-path", ".", "Directory to write database files to")
// httpAddr = flag.String("http-address", ":8080", "Bind address for http server")
// dnsAddr = flag.String("dns-address", ":53", "Bind address for dns server")
)
func main() {
@ -25,6 +28,8 @@ func main() {
if err := LoadStartupConfig(&conf, *configFilePath); err != nil {
log.Fatalf("%+v", err)
}
// conf.HTTPAddr = *httpAddr
// conf.DNSAddr = *dnsAddr
log.Printf("%+v", conf)
store := &internal.Sqlite{
Path: conf.DatabaseURL,

Loading…
Cancel
Save