runner starts up and runs migrations

.gitignore

@@ -1,9 +1,10 @@
 .kubeconfig
+.vscode
 
 
 # auto generated files
-postgres
-
 *.pb.go
 *.pb.gw.go
 *.swagger.json
+
+src/**/internal/data/postgres/*.go

env/base/runner-deployment.yaml

@@ -12,10 +12,50 @@ spec:
       labels:
         service: bh-runner
     spec:
+      initContainers:
+        - name: runner-db-migrate
+          image: barretthousen/service-runner:latest
+          command:
+            - /opt/runner
+          args:
+            - -migrate
+          resources:
+            limits:
+              cpu: "250m"
+              memory: "64Mi"
+          volumeMounts:
+            - mountPath: /config/
+              name: bh-runner-config
       containers:
         - name: runner
           image: barretthousen/service-runner:latest
+          command:
+            - /opt/runner
           resources:
             limits:
               cpu: "250m"
               memory: "128Mi"
+          volumeMounts:
+            - mountPath: /config/
+              name: bh-runner-config
+      volumes:
+        - name: bh-runner-config
+          configMap:
+            name: bh-runner-config
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: bh-runner-config
+data:
+  config.yaml: |
+    log_level: 2
+    port: 5001
+    postgres:
+      scheme: postgres
+      port: 5432
+      host: bh-db
+      name: bh
+      user: postgres
+      password: bh-admin

go.work.sum

@@ -0,0 +1 @@
+go.uber.org/dig v1.0.0/go.mod h1:z+dSd2TP9Usi48jL8M3v63iSBVkiwtVyMKxMZYYauPg=

src/Dockerfile.service

@@ -17,4 +17,4 @@ ENV SERVICE=${SERVICE}
 
 COPY --from=builder /opt/${SERVICE} /opt/${SERVICE}
 
-CMD /opt/${SERVICE}
+ENTRYPOINT ['/opt/${SERVICE}']

src/Tiltfile

@@ -29,6 +29,7 @@ helm_resource(
     'bitnami/postgresql',
     resource_deps=['bitnami'],
     flags=[
+        '--set', 'fullnameOverride=bh-db',
         '--set', 'auth.enablePostgresUser=true',
         '--set', 'auth.postgresPassword=bh-admin',
         '--set', 'auth.database=bh',

src/lib/go.mod

@@ -5,9 +5,18 @@ go 1.19
 require (
 	github.com/jackc/pgx/v4 v4.18.1
 	go.uber.org/automaxprocs v1.5.2
+	go.uber.org/dig v1.16.1
 )
 
 require (
+	github.com/BurntSushi/toml v1.1.0 // indirect
+	github.com/joho/godotenv v1.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect
+)
+
+require (
+	github.com/ilyakaznacheev/cleanenv v1.4.2
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgconn v1.14.0 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect

src/lib/go.sum

@@ -1,4 +1,6 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I=
+github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -11,6 +13,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/ilyakaznacheev/cleanenv v1.4.2 h1:nRqiriLMAC7tz7GzjzUTBHfzdzw6SQ7XvTagkFqe/zU=
+github.com/ilyakaznacheev/cleanenv v1.4.2/go.mod h1:i0owW+HDxeGKE0/JPREJOdSCPIyOnmh6C0xhWAkF/xA=
 github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
 github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
 github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
@@ -60,6 +64,8 @@ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0f
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
+github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -108,6 +114,8 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
 go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
+go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8=
+go.uber.org/dig v1.16.1/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
@@ -188,5 +196,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ=
+olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw=

src/lib/kernel/app.go

@@ -2,26 +2,30 @@ package kernel
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
+	"github.com/ilyakaznacheev/cleanenv"
 	_ "go.uber.org/automaxprocs"
+	// "go.uber.org/dig"
 )
 
 type App interface {
 	Start(context.Context) error
 	OnStop(context.Context)
+	GetLogLevel() LogLevel
 }
 
 type AppConfig struct {
-	App
-	LogLevel LogLevel
+	LogLevel LogLevel    `yaml:"log_level" env:"BH_LOG_LEVEL" env-default:"ERROR" yaml-default:"ERROR"`
+	Config   interface{} `yaml:"service" env:"BH_SERVICE"`
 }
 
-func Run(parent context.Context, cfg AppConfig) {
-	SetLogLevel(cfg.LogLevel)
+func Run(parent context.Context, app App) {
+	SetLogLevel(app.GetLogLevel())
 	ctx, canceller := context.WithCancel(parent)
 	defer canceller()
 
@@ -39,11 +43,36 @@ func Run(parent context.Context, cfg AppConfig) {
 		InfoLog.Println("Shutting down service")
 		stopCtx, stopCanceller := context.WithTimeout(parent, time.Second*5)
 		defer stopCanceller()
-		cfg.App.OnStop(stopCtx)
+		app.OnStop(stopCtx)
 	}()
 
 	InfoLog.Println("Starting service")
-	if err := cfg.App.Start(ctx); err != nil {
-		FatalErr(err)
+
+	if err := loadConfig(app); err != nil {
+		ErrorLog.Println(err)
+		return
+	}
+
+	if err := app.Start(ctx); err != nil {
+		ErrorLog.Println(err)
+		return
+	}
+}
+
+func loadConfig(cfg interface{}) error {
+	fp := os.Getenv("BH_APP_CONFIG_PATH")
+
+	if fp == "" {
+		fp = "/config/config.yaml"
+	}
+
+	if err := cleanenv.ReadConfig(fp, cfg); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("could not read config from file: %w", err)
 	}
+
+	if err := cleanenv.ReadEnv(cfg); err != nil {
+		return fmt.Errorf("could not read config from env: %w", err)
+	}
+
+	return nil
 }

src/lib/kernel/postgres.go

@@ -42,3 +42,18 @@ func NewDBConnection(ctx context.Context, pg PostgresConnection) (conn *pgx.Conn
 
 	return conn, nil
 }
+
+func Migrate(ctx context.Context, pg PostgresConnection, sql string) error {
+	conn, err := NewDBConnection(ctx, pg)
+	if err != nil {
+		return err
+	}
+
+	defer conn.PgConn().Close(ctx)
+
+	if _, err := conn.Exec(ctx, sql); err != nil {
+		return err
+	}
+
+	return nil
+}

src/runner/config.yaml

@@ -0,0 +1,3 @@
+log_level: ERROR
+
+service: {}

src/runner/internal/data/postgres/queries.sql

@@ -0,0 +1,16 @@
+-- name: GetJobs :many
+SELECT id,
+	   startedTs,
+	   completedTs,
+	   targetSiteName,
+	   auctionsFound,
+	   errors
+	FROM runner.scrapejob;
+
+-- name: CreateScrapeJob :one
+INSERT INTO runner.scrapejob (
+		targetSiteName
+	) VALUES (
+		$1
+	) RETURNING *;
+

src/runner/internal/data/postgres/schema.sql

@@ -0,0 +1,16 @@
+START TRANSACTION;
+
+CREATE USER "runner-service" WITH PASSWORD 'runner-service';
+
+CREATE SCHEMA IF NOT EXISTS runner;
+
+CREATE TABLE IF NOT EXISTS runner.scrapejob (
+	id SERIAL PRIMARY KEY,
+	startedTs TIMESTAMP NOT NULL DEFAULT NOW(),
+	completedTs TIMESTAMP,
+	targetSiteName VARCHAR(512) NOT NULL,
+	auctionsFound INT NOT NULL DEFAULT 0,
+	errors VARCHAR(5000) NOT NULL
+);
+
+COMMIT;

src/runner/main.go

@@ -2,21 +2,50 @@ package main
 
 import (
 	"context"
+	"flag"
+	"fmt"
 	"time"
 
+	_ "embed"
+
 	"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
 )
 
+type (
+	RunnerApp struct {
+		LogLevel kernel.LogLevel           `yaml:"log_level" env:"BH_LOG_LEVEL" env-default:"0" yaml-default:"0"`
+		Port     int                       `yaml:"port" env:"RUNNER_PORT"`
+		Postgres kernel.PostgresConnection `yaml:"postgres" env:"RUNNER_DB"`
+	}
+)
+
+var migrate = flag.Bool("migrate", false, "migrates postgres db")
+
+//go:embed internal/data/postgres/schema.sql
+var dbMigrateScript string
+
 func main() {
-	kernel.Run(context.Background(), kernel.AppConfig{
-		App:      &RunnerApp{},
+	flag.Parse()
+
+	kernel.Run(context.Background(), &RunnerApp{
 		LogLevel: kernel.LevelTrace,
+		Port:     5001,
 	})
 }
 
-type RunnerApp struct{}
-
 func (app *RunnerApp) Start(ctx context.Context) error {
+	if *migrate {
+		kernel.InfoLog.Printf("running db migrations on %v", app.Postgres)
+
+		kernel.InfoLog.Printf("MIGRATION SCRIPT:\n%q", dbMigrateScript)
+		if err := kernel.Migrate(ctx, app.Postgres, dbMigrateScript); err != nil {
+			return fmt.Errorf("could not execute db migration: %w", err)
+		}
+
+		return nil
+	}
+
+	kernel.InfoLog.Printf("%+v", app)
 	t := time.NewTicker(time.Second)
 
 	for {
@@ -32,3 +61,5 @@ func (app *RunnerApp) Start(ctx context.Context) error {
 
 func (app *RunnerApp) OnStop(ctx context.Context) {
 }
+
+func (app *RunnerApp) GetLogLevel() kernel.LogLevel { return app.LogLevel }