runner starts up and runs migrations
parent
6cd624ced1
commit
2699f0c14d
GPG Key ID:
6DB29003C6DD1E4B
|
|
@ -1,9 +1,10 @@
|
|||
.kubeconfig
|
||||
.vscode
|
||||
|
||||
|
||||
# auto generated files
|
||||
postgres
|
||||
|
||||
*.pb.go
|
||||
*.pb.gw.go
|
||||
*.swagger.json
|
||||
|
||||
src/**/internal/data/postgres/*.go
|
||||
|
|
|
|||
|
|
@ -12,10 +12,50 @@ spec:
|
|||
labels:
|
||||
service: bh-runner
|
||||
spec:
|
||||
initContainers:
|
||||
- name: runner-db-migrate
|
||||
image: barretthousen/service-runner:latest
|
||||
command:
|
||||
- /opt/runner
|
||||
args:
|
||||
- -migrate
|
||||
resources:
|
||||
limits:
|
||||
cpu: "250m"
|
||||
memory: "64Mi"
|
||||
volumeMounts:
|
||||
- mountPath: /config/
|
||||
name: bh-runner-config
|
||||
containers:
|
||||
- name: runner
|
||||
image: barretthousen/service-runner:latest
|
||||
command:
|
||||
- /opt/runner
|
||||
resources:
|
||||
limits:
|
||||
cpu: "250m"
|
||||
memory: "128Mi"
|
||||
volumeMounts:
|
||||
- mountPath: /config/
|
||||
name: bh-runner-config
|
||||
volumes:
|
||||
- name: bh-runner-config
|
||||
configMap:
|
||||
name: bh-runner-config
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: bh-runner-config
|
||||
data:
|
||||
config.yaml: |
|
||||
log_level: 2
|
||||
port: 5001
|
||||
postgres:
|
||||
scheme: postgres
|
||||
port: 5432
|
||||
host: bh-db
|
||||
name: bh
|
||||
user: postgres
|
||||
password: bh-admin
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
go.uber.org/dig v1.0.0/go.mod h1:z+dSd2TP9Usi48jL8M3v63iSBVkiwtVyMKxMZYYauPg=
|
||||
|
|
@ -17,4 +17,4 @@ ENV SERVICE=${SERVICE}
|
|||
|
||||
COPY --from=builder /opt/${SERVICE} /opt/${SERVICE}
|
||||
|
||||
CMD /opt/${SERVICE}
|
||||
ENTRYPOINT ['/opt/${SERVICE}']
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ helm_resource(
|
|||
'bitnami/postgresql',
|
||||
resource_deps=['bitnami'],
|
||||
flags=[
|
||||
'--set', 'fullnameOverride=bh-db',
|
||||
'--set', 'auth.enablePostgresUser=true',
|
||||
'--set', 'auth.postgresPassword=bh-admin',
|
||||
'--set', 'auth.database=bh',
|
||||
|
|
|
|||
|
|
@ -5,9 +5,18 @@ go 1.19
|
|||
require (
|
||||
github.com/jackc/pgx/v4 v4.18.1
|
||||
go.uber.org/automaxprocs v1.5.2
|
||||
go.uber.org/dig v1.16.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.1.0 // indirect
|
||||
github.com/joho/godotenv v1.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/ilyakaznacheev/cleanenv v1.4.2
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
|
||||
github.com/jackc/pgconn v1.14.0 // indirect
|
||||
github.com/jackc/pgio v1.0.0 // indirect
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I=
|
||||
github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
|
||||
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
|
|
@ -11,6 +13,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
|
|||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/ilyakaznacheev/cleanenv v1.4.2 h1:nRqiriLMAC7tz7GzjzUTBHfzdzw6SQ7XvTagkFqe/zU=
|
||||
github.com/ilyakaznacheev/cleanenv v1.4.2/go.mod h1:i0owW+HDxeGKE0/JPREJOdSCPIyOnmh6C0xhWAkF/xA=
|
||||
github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
|
||||
github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
|
||||
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
|
||||
|
|
@ -60,6 +64,8 @@ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0f
|
|||
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
|
||||
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
|
|
@ -108,6 +114,8 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
|||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
|
||||
go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
|
||||
go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8=
|
||||
go.uber.org/dig v1.16.1/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
|
||||
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
|
||||
|
|
@ -188,5 +196,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
|||
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ=
|
||||
olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw=
|
||||
|
|
|
|||
|
|
@ -2,26 +2,30 @@ package kernel
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/ilyakaznacheev/cleanenv"
|
||||
_ "go.uber.org/automaxprocs"
|
||||
// "go.uber.org/dig"
|
||||
)
|
||||
|
||||
type App interface {
|
||||
Start(context.Context) error
|
||||
OnStop(context.Context)
|
||||
GetLogLevel() LogLevel
|
||||
}
|
||||
|
||||
type AppConfig struct {
|
||||
App
|
||||
LogLevel LogLevel
|
||||
LogLevel LogLevel `yaml:"log_level" env:"BH_LOG_LEVEL" env-default:"ERROR" yaml-default:"ERROR"`
|
||||
Config interface{} `yaml:"service" env:"BH_SERVICE"`
|
||||
}
|
||||
|
||||
func Run(parent context.Context, cfg AppConfig) {
|
||||
SetLogLevel(cfg.LogLevel)
|
||||
func Run(parent context.Context, app App) {
|
||||
SetLogLevel(app.GetLogLevel())
|
||||
ctx, canceller := context.WithCancel(parent)
|
||||
defer canceller()
|
||||
|
||||
|
|
@ -39,11 +43,36 @@ func Run(parent context.Context, cfg AppConfig) {
|
|||
InfoLog.Println("Shutting down service")
|
||||
stopCtx, stopCanceller := context.WithTimeout(parent, time.Second*5)
|
||||
defer stopCanceller()
|
||||
cfg.App.OnStop(stopCtx)
|
||||
app.OnStop(stopCtx)
|
||||
}()
|
||||
|
||||
InfoLog.Println("Starting service")
|
||||
if err := cfg.App.Start(ctx); err != nil {
|
||||
FatalErr(err)
|
||||
|
||||
if err := loadConfig(app); err != nil {
|
||||
ErrorLog.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := app.Start(ctx); err != nil {
|
||||
ErrorLog.Println(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func loadConfig(cfg interface{}) error {
|
||||
fp := os.Getenv("BH_APP_CONFIG_PATH")
|
||||
|
||||
if fp == "" {
|
||||
fp = "/config/config.yaml"
|
||||
}
|
||||
|
||||
if err := cleanenv.ReadConfig(fp, cfg); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("could not read config from file: %w", err)
|
||||
}
|
||||
|
||||
if err := cleanenv.ReadEnv(cfg); err != nil {
|
||||
return fmt.Errorf("could not read config from env: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,3 +42,18 @@ func NewDBConnection(ctx context.Context, pg PostgresConnection) (conn *pgx.Conn
|
|||
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func Migrate(ctx context.Context, pg PostgresConnection, sql string) error {
|
||||
conn, err := NewDBConnection(ctx, pg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer conn.PgConn().Close(ctx)
|
||||
|
||||
if _, err := conn.Exec(ctx, sql); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
log_level: ERROR
|
||||
|
||||
service: {}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
-- name: GetJobs :many
|
||||
SELECT id,
|
||||
startedTs,
|
||||
completedTs,
|
||||
targetSiteName,
|
||||
auctionsFound,
|
||||
errors
|
||||
FROM runner.scrapejob;
|
||||
|
||||
-- name: CreateScrapeJob :one
|
||||
INSERT INTO runner.scrapejob (
|
||||
targetSiteName
|
||||
) VALUES (
|
||||
$1
|
||||
) RETURNING *;
|
||||
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
START TRANSACTION;
|
||||
|
||||
CREATE USER "runner-service" WITH PASSWORD 'runner-service';
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS runner;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS runner.scrapejob (
|
||||
id SERIAL PRIMARY KEY,
|
||||
startedTs TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
completedTs TIMESTAMP,
|
||||
targetSiteName VARCHAR(512) NOT NULL,
|
||||
auctionsFound INT NOT NULL DEFAULT 0,
|
||||
errors VARCHAR(5000) NOT NULL
|
||||
);
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -2,21 +2,50 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
_ "embed"
|
||||
|
||||
"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
|
||||
)
|
||||
|
||||
type (
|
||||
RunnerApp struct {
|
||||
LogLevel kernel.LogLevel `yaml:"log_level" env:"BH_LOG_LEVEL" env-default:"0" yaml-default:"0"`
|
||||
Port int `yaml:"port" env:"RUNNER_PORT"`
|
||||
Postgres kernel.PostgresConnection `yaml:"postgres" env:"RUNNER_DB"`
|
||||
}
|
||||
)
|
||||
|
||||
var migrate = flag.Bool("migrate", false, "migrates postgres db")
|
||||
|
||||
//go:embed internal/data/postgres/schema.sql
|
||||
var dbMigrateScript string
|
||||
|
||||
func main() {
|
||||
kernel.Run(context.Background(), kernel.AppConfig{
|
||||
App: &RunnerApp{},
|
||||
flag.Parse()
|
||||
|
||||
kernel.Run(context.Background(), &RunnerApp{
|
||||
LogLevel: kernel.LevelTrace,
|
||||
Port: 5001,
|
||||
})
|
||||
}
|
||||
|
||||
type RunnerApp struct{}
|
||||
|
||||
func (app *RunnerApp) Start(ctx context.Context) error {
|
||||
if *migrate {
|
||||
kernel.InfoLog.Printf("running db migrations on %v", app.Postgres)
|
||||
|
||||
kernel.InfoLog.Printf("MIGRATION SCRIPT:\n%q", dbMigrateScript)
|
||||
if err := kernel.Migrate(ctx, app.Postgres, dbMigrateScript); err != nil {
|
||||
return fmt.Errorf("could not execute db migration: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
kernel.InfoLog.Printf("%+v", app)
|
||||
t := time.NewTicker(time.Second)
|
||||
|
||||
for {
|
||||
|
|
@ -32,3 +61,5 @@ func (app *RunnerApp) Start(ctx context.Context) error {
|
|||
|
||||
func (app *RunnerApp) OnStop(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (app *RunnerApp) GetLogLevel() kernel.LogLevel { return app.LogLevel }
|
||||
|
|
|
|||
Loading…
Reference in New Issue