client/src/api/index.ts

@@ -62,20 +62,11 @@ export enum StatSearchKey {
     Protocol = "protocol"
 }
 
-export interface StatDataset {
+export interface Stat {
-    label: string
-    data: DataPoint[]
-}
-
-export interface DataPoint {
     Header: string
-    Value: Number
+    AverageTotalTime: Number
     Count: number,
     Time: string
-}
-export interface Stat {
-    labels: string[]
-    datasets: StatDataset[]
 };
 
 export const getStats = async ({
@@ -83,7 +74,7 @@ export const getStats = async ({
     end = new Date(),
     key = StatSearchKey.Domain,
     interval = 30,
-}: StatsSearchOptions) => await apiCall<Stat>('metrics/stats', 'GET', {
+}: StatsSearchOptions) => await apiCall<Stat[]>('metrics/stats', 'GET', {
     start: getUnixTime(start),
     end: getUnixTime(end),
     key,

client/src/components/TimeChart.svelte

@@ -2,23 +2,45 @@
     import { onMount } from "svelte";
     import type { Stat } from "../api";
     import randomColor from "randomcolor";
+
     import { Chart, registerables } from "chart.js";
     Chart.register(...registerables);
 
-    export let stats: Stat = null;
+    export let stats: Stat[] = [];
-    export let column: string = null;
 
-    const generateChartOptions = (s: Stat, empty: Boolean = false) => {
+    const transformStats = (ostats) => {
+        const chartData = ostats.reduce((agg, x) => {
+            let root = agg[x.Header] || {
+                labels: [],
+                dataset: {
+                    label: x.Header,
+                    borderColor: randomColor({
+                        luminosity: "dark",
+                    }), //"rgb(75,192,192)",
+                    data: [],
+                },
+            };
+            root.dataset.data = root.dataset.data.concat(x.Count);
+            root.labels = root.labels.concat(x.Time);
+            agg[x.Header] = root;
+            return agg;
+        }, {});
+
+        const finalChartData = Object.keys(chartData).map((x) => chartData[x]);
+        const finalChartLabels =
+            finalChartData.length > 0 ? finalChartData[0].labels : [];
+        return {
+            labels: finalChartLabels,
+            datasets: finalChartData.map((x) => x.dataset),
+        };
+    };
+
+    const generateChartOptions = (s: [], empty: Boolean = false) => {
         let labels = [];
         let datasets = [];
 
-        if (s) {
+        if (s && s.length > 0) {
-            labels = s.labels;
+            ({ labels, datasets } = transformStats(s));
-            datasets = s.datasets.map(({ label, data }) => ({
-                label,
-                data: data.map((x) => x.Count),
-                borderColor: randomColor(),
-            }));
         }
 
         var delayed;
@@ -32,25 +54,17 @@
                 responsive: true,
                 maintainAspectRatio: false,
                 scales: {
-                    x: {
+                    // x: {
-                        title: {
+                    //     type: "time",
-                            label: "time",
+                    //     ticks: {
-                            display: true,
+                    //         source: "auto",
-                        },
+                    //         // Disabled rotation for performance
-                        //     type: "time",
+                    //         maxRotation: 0,
-                        //     ticks: {
+                    //         autoSkip: true,
-                        //         source: "auto",
+                    //     },
-                        //         // Disabled rotation for performance
+                    // },
-                        //         maxRotation: 0,
-                        //         autoSkip: true,
-                        //     },
-                    },
                     y: {
                         stacked: true,
-                        min: 0,
-                        ticks: {
-                            stepSize: 5,
-                        },
                     },
                 },
                 hoverRadius: 5,
@@ -65,10 +79,6 @@
                         algorithm: "lttb",
                         samples: 60,
                     },
-                    title: {
-                        display: true,
-                        text: `Count by ${column}`,
-                    },
                 },
                 animations: {
                     radius: {
@@ -107,7 +117,7 @@
         chartInstance = new Chart(ctx, generateChartOptions(stats, true));
     });
 
-    const update = (s: Stat) => {
+    const update = (s) => {
         if (chartInstance) {
             const { options, data } = generateChartOptions(s, false);
             chartInstance.options = options;

client/src/routes/Home.svelte

@@ -32,7 +32,7 @@
     let chartDataLoading: Boolean = false;
     let logDataLoading: Boolean = false;
 
-    let chartData: Stat = null;
+    let chartData: Stat[] = [];
     let logs: Log[] = [];
     let pageSize: number = 50;
     let pageCount: number = 0;
@@ -62,7 +62,7 @@
         return payload;
     };
 
-    const fetchStats = async (): Promise<Stat> => {
+    const fetchStats = async () => {
         if (chartDataLoading) {
             console.warn("tried loading stats while already loading");
             return;
@@ -80,7 +80,7 @@
 
         if (error) {
             chartErrorMsg = error;
-            return null;
+            return [];
         }
 
         return payload;
@@ -137,7 +137,7 @@
         {:else if chartErrorMsg}
             <p>{chartErrorMsg}</p>
         {:else}
-            <TimeChart stats={chartData} column={chartKey} />
+            <TimeChart stats={chartData} />
         {/if}
     </section>
     <section class="my-5">

internal/dnsHandler.go

@@ -1,11 +1,13 @@
 package internal
 
 import (
+	"fmt"
 	"log"
 	"strings"
 	"time"
 
 	"github.com/miekg/dns"
+	"github.com/nakabonne/tstorage"
 )
 
 type DomainManager struct {
@@ -55,13 +57,12 @@ func (dm *DomainManager) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 	responseMessage.Compress = true
 
 	ql.TotalTimeMs = int(time.Since(start).Milliseconds())
-
 	log.Printf("%+v", ql)
-	go func(q QueryLog) {
+	go func() {
-		if err := dm.Storage.Log(q); err != nil {
+		if err := dm.Storage.Log(ql); err != nil {
 			log.Printf("ERROR WRITING LOG: %v", err)
 		}
-	}(ql)
+	}()
 
 	if err := w.WriteMsg(responseMessage); err != nil {
 		log.Println(err)
@@ -90,15 +91,36 @@ type QueryLog struct {
 	Status                 ResponseStatus
 }
 
-func GetAggregateColumnHeader(ql QueryLog, h LogAggregateColumn) string {
+func QueryLogLabels(q QueryLog) []tstorage.Label {
-	switch h {
+	return []tstorage.Label{
-	case ClientIP:
+		{
-		return ql.ClientIP
+			Name:  string(ClientIP),
-	case Status:
+			Value: q.ClientIP,
-		return string(ql.Status)
+		},
-	case Protocol:
+		{
-		return ql.Protocol
+			Name:  string(Domain),
+			Value: q.Domain,
+		},
+		{
+			Name:  string(Protocol),
+			Value: q.Protocol,
+		},
+		{
+			Name:  string(Status),
+			Value: string(q.Status),
+		},
+		{
+			Name:  string(RecursedUpstream),
+			Value: string(q.RecurseUpstreamIP),
+		},
+		{
+			Name:  string(LookupError),
+			Value: fmt.Sprintf("%s", q.Error != ""),
+		},
 	}
-
+}
-	return ql.Domain
+
+func GetLabelsFromQuery(la LogAggregateInput) []tstorage.Label {
+	return nil
+
 }

internal/sqlite.go

@@ -2,20 +2,20 @@ package internal
 
 import (
 	"database/sql"
-	"errors"
 	"fmt"
-	"log"
 	"net"
 	"strconv"
 	"strings"
 	"time"
 
 	_ "github.com/mattn/go-sqlite3"
+	"github.com/nakabonne/tstorage"
 )
 
 type Sqlite struct {
 	Path string
 	*sql.DB
+	TS tstorage.Storage
 }
 
 func (ss *Sqlite) Open() error {
@@ -26,7 +26,14 @@ func (ss *Sqlite) Open() error {
 
 	db.SetMaxOpenConns(1)
 
+	ts, _ := tstorage.NewStorage(
+		tstorage.WithTimestampPrecision(tstorage.Nanoseconds),
+		tstorage.WithPartitionDuration(time.Minute*30),
+		tstorage.WithDataPath(ss.Path),
+	)
+
 	ss.DB = db
+	ss.TS = ts
 
 	if err := initTable(db); err != nil {
 		return err
@@ -35,129 +42,6 @@ func (ss *Sqlite) Open() error {
 	return nil
 }
 
-func (ss *Sqlite) Close() error {
-	ss.DB.Close()
-	return nil
-}
-
-const defaultSamples = 64
-const maxSamples = 128
-
-func (ss *Sqlite) GetLogAggregate(la LogAggregateInput) (LogAggregate, error) {
-	if la.End.IsZero() || la.End.After(time.Now()) {
-		la.End = time.Now().UTC()
-	}
-
-	if la.Start.After(la.End) {
-		return LogAggregate{}, errors.New("Start time cannot be before end time")
-	}
-
-	if la.Start.IsZero() {
-		la.Start = time.Now().UTC().Add(time.Hour * -12)
-	}
-
-	timespanSecs := int(la.End.Sub(la.Start) / time.Second)
-
-	// how many data points to show on the line plot
-	sampleCount := defaultSamples
-
-	if la.IntervalSeconds <= 0 {
-		la.IntervalSeconds = timespanSecs / sampleCount
-	}
-
-	sampleCount = timespanSecs / la.IntervalSeconds
-
-	// cap to prevent performance issues
-	if sampleCount > maxSamples {
-		sampleCount = maxSamples
-		la.IntervalSeconds = timespanSecs / sampleCount
-		log.Printf("got %v samples, capping to 256 for perf", sampleCount)
-	}
-
-	log.Printf("%+v - samples: %v - timespan (seconds): %v", la, sampleCount, timespanSecs)
-
-	switch la.Column {
-	case string(Domain):
-	case string(Status):
-	case string(ClientIP):
-	case string(Protocol):
-		break
-	default:
-		la.Column = string(Domain)
-	}
-
-	logs, err := ss.GetLog(GetLogInput{
-		Start: la.Start,
-		End:   la.End,
-		Limit: 10000,
-		Page:  0,
-	})
-	if err != nil {
-		return LogAggregate{}, err
-	}
-
-	if logs.PageCount > 1 {
-		return LogAggregate{}, fmt.Errorf("more than one page available: %v", logs.PageCount)
-	}
-
-	lut := [][]StatsDataPoint{}
-	buckets := map[string][]StatsDataPoint{}
-	for _, l := range logs.Logs {
-		k := GetAggregateColumnHeader(l, LogAggregateColumn(la.Column))
-		if _, ok := buckets[k]; !ok {
-			buckets[k] = make([]StatsDataPoint, sampleCount)
-			lut = append(lut, buckets[k])
-		}
-		dataset := buckets[k]
-
-		timeIndex := int(l.Started.Sub(la.Start)/time.Second) / la.IntervalSeconds
-
-		ladp := dataset[timeIndex]
-		ladp.Header = k
-		offsetSecs := (timeIndex * la.IntervalSeconds)
-		ladp.Time = la.Start.Add(time.Duration(offsetSecs) * time.Second)
-		ladp.Count += 1
-		ladp.Value += float64(l.TotalTimeMs)
-
-		buckets[k][timeIndex] = ladp
-	}
-
-	laResult := LogAggregate{
-		Labels:   make([]string, sampleCount),
-		Datasets: make([]LogAggregateDataset, len(buckets)),
-	}
-
-	for idx := 0; idx < sampleCount; idx++ {
-		offsetSecs := (idx * la.IntervalSeconds)
-		ts := la.Start.Add(time.Duration(offsetSecs) * time.Second)
-		laResult.Labels[idx] = ts.Format("01-02 15:04:05")
-
-		idx := 0
-		for k, v := range buckets {
-			ladp := v[idx]
-			if ladp.Time.IsZero() {
-				v[idx].Time = ts
-			}
-			laResult.Datasets[idx].Dataset = v
-			laResult.Datasets[idx].Label = k
-			idx++
-		}
-
-	}
-
-	return laResult, nil
-}
-
-type LogAggregate struct {
-	Labels   []string              `json:"labels"`
-	Datasets []LogAggregateDataset `json:"datasets"`
-}
-
-type LogAggregateDataset struct {
-	Label   string           `json:"label"`
-	Dataset []StatsDataPoint `json:"data"`
-}
-
 func (ss *Sqlite) Log(ql QueryLog) error {
 	sql := `
 	INSERT INTO log
@@ -179,6 +63,49 @@ func (ss *Sqlite) Log(ql QueryLog) error {
 		return err
 	}
 
+	labels := QueryLogLabels(ql)
+	isError := 0
+	if ql.Error != "" {
+		isError = 1
+	}
+
+	if err := ss.TS.InsertRows([]tstorage.Row{
+		{
+			Metric: "Count",
+			Labels: labels,
+			DataPoint: tstorage.DataPoint{
+				Timestamp: ql.Started.UnixNano(),
+				Value:     1,
+			},
+		},
+		{
+			Metric: "TotalTimeMs",
+			Labels: labels,
+			DataPoint: tstorage.DataPoint{
+				Timestamp: ql.Started.UnixNano(),
+				Value:     float64(ql.TotalTimeMs),
+			},
+		},
+		{
+			Metric: "Errors",
+			Labels: labels,
+			DataPoint: tstorage.DataPoint{
+				Timestamp: ql.Started.UnixNano(),
+				Value:     float64(isError),
+			},
+		},
+		{
+			Metric: "RecurseTimeMs",
+			Labels: labels,
+			DataPoint: tstorage.DataPoint{
+				Timestamp: ql.Started.UnixNano(),
+				Value:     float64(ql.RecurseRoundTripTimeMs),
+			},
+		},
+	}); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -453,6 +380,50 @@ func (ss *Sqlite) GetPagingInfo(in GetLogInput) (totalItems, pageCount int, err
 	return
 }
 
+func (ss *Sqlite) GetLogAggregate(la LogAggregateInput) ([]LogAggregateDataPoint, error) {
+	timeWindow := int64(5 * 60)
+	column := "domain"
+
+	if lac, ok := AggregateKeys[la.Column]; ok {
+		column = string(lac)
+	}
+
+	if la.IntervalSeconds > 0 {
+		timeWindow = int64(la.IntervalSeconds)
+	}
+
+	points, err := ss.TS.Select(column, GetLabelsFromQuery(la), la.Start.UnixNano(), la.End.UnixNano())
+
+	var
+	for _, p := range points {
+		ladp := LogAggregateDataPoint{
+			Header: column,
+			Count:  int(p.Value),
+			Time:   time.Unix(0, p.Timestamp),
+		}
+		results = append(results, ladp)
+	}
+
+	// for rows.Next() {
+	// 	var ladp LogAggregateDataPoint
+	// 	var timeInterval int64
+	// 	if err := rows.Scan(
+	// 		&ladp.Header,
+	// 		&ladp.AverageTotalTime,
+	// 		&ladp.Count,
+	// 		&timeInterval,
+	// 	); err != nil {
+	// 		return nil, err
+	// 	}
+
+	// 	ladp.Time = time.Unix(timeInterval*timeWindow, 0)
+
+	// 	results = append(results, ladp)
+	// }
+
+	return results, nil
+}
+
 func initTable(db *sql.DB) error {
 	sql := `
 		CREATE TABLE IF NOT EXISTS log (

internal/storage.go

@@ -37,11 +37,11 @@ type LogAggregateInput struct {
 	Column          string
 }
 
-type StatsDataPoint struct {
+type LogAggregateDataPoint struct {
-	Header string
+	Header           string
-	Value  float64
+	AverageTotalTime float64
-	Count  int
+	Count            int
-	Time   time.Time
+	Time             time.Time
 }
 
 type Storage interface {
@@ -60,7 +60,7 @@ type Storage interface {
 
 	Log(QueryLog) error
 	GetLog(GetLogInput) (GetLogResult, error)
-	GetLogAggregate(LogAggregateInput) (LogAggregate, error)
+	GetLogAggregate(LogAggregateInput) ([]LogAggregateDataPoint, error)
 }
 
 type RecursorRow struct {

main.go

@@ -37,8 +37,6 @@ func main() {
 		Path: conf.DatabaseURL,
 	}
 
-	defer store.Close()
-
 	if err := store.Open(); err != nil {
 		log.Fatalf("COULD NOT OPEN SQLITE DB: %v", err)
 	}

makefile

@@ -25,6 +25,7 @@ test:
 	mkdir -p .bin
 
 .bin/gopherhole: .bin
+	# @go build --tags "sqlite_foreign_keys fts5" -v -o .bin/gopherhole .
 	@go build --tags "fts5" -v -o .bin/gopherhole .
 
 .bin/config.json: