various updates for docs env

.drone.yml

@@ -7,7 +7,7 @@ trigger:
     - push
 
 steps:
-  - name: Build & Publish
+  - name: Build & Publish Catalog
     image: plugins/kaniko
     settings:
       dockerfile: "./src/Dockerfile.prod-backend"
@@ -23,29 +23,19 @@ steps:
       no-push: true
       tags:
         - ${DRONE_COMMIT_SHA}
-# ---
+  - name: Build & Publish Runner
-# kind: pipeline
+    image: plugins/kaniko
-# type: kubernetes
+    settings:
-# name: Deploy beta
+      dockerfile: "./src/Dockerfile.prod-backend"
-
+      context: "./src"
-# trigger:
+      registry: git.vdhsn.com
-#   event:
+      repo: git.vdhsn.com/barretthousen/service-runner
-#     - push
+      username:
-#   branch:
+        from_secret: DOCKER_USERNAME
-#     - trunk
+      password:
-
+        from_secret: DOCKER_PASSWORD
-# steps:
+      args:
-#   - name: discord notification
+        - "service=runner"
-#     image: appleboy/drone-discord
+      no-push: true
-#     environment:
+      tags:
-#       DISCORD_WEBHOOK_ID:
+        - ${DRONE_COMMIT_SHA}
-#         from_secret: DISCORD_WEBHOOK_ID
-#       DISCORD_WEBHOOK_TOKEN:
-#         from_secret: DISCORD_WEBHOOK_TOKEN
-#     settings:
-#       username: Drone CI
-#       message: Weekly Server Update Complete!
-#       webhook_id:
-#         from_secret: DISCORD_WEBHOOK_ID
-#       webhook_token:
-#         from_secret: DISCORD_WEBHOOK_TOKEN

Makefile

@@ -8,7 +8,8 @@ GOBIN = $(shell go env GOPATH)/bin
 ###################
 
 .PHONY: setup
-setup: $(GOBIN)/sqlc $(GOBIN)/buf ./env/.age.txt
+setup: $(GOBIN)/sqlc $(GOBIN)/buf
+	@awk '{ print $$1 }' .tool-versions | xargs -I {} asdf plugin add {} || true
 	@asdf install || true
 
 .PHONY: gen
@@ -26,13 +27,15 @@ clobber:
 	KUBECONFIG=$(KUBECONFIG) ctlptl delete cluster kind-bh-local || true
 	@rm -f $(KUBECONFIG)
 
+.PHONY: secrets
+secrets: ./env/.age.txt
 
 .PHONY: acceptance-test
 acceptance-test:
 	docker run -it --rm \
-		-v $(pwd)/src/acceptance-tests.yml:/tests/tests.yml \
+		-v $(PWD)/src/$(SERVICE)/acceptance-tests.yml:/tests/tests.yml \
-		-e "TEST_HOST=bh.localhost" \
+		-e "TEST_HOST=$(ORIGIN)" \
-		-e "TEST_ENV=local" \
+		-e "TEST_ENV=prod" \
 		nytimes/httptest
 
 

README.md

@@ -2,71 +2,85 @@
 
 Search and get alerts for items across the most popular auction sites.
 
-Built with microservice architecture, for learning purposes
+Built with microservice architecture and various fancy tools for learning purposes
 
 ### Links
 
 - [Keybase Team Chat](keybase://team/barretthousen)
 - [Source](https://git.vdhsn.com/Barretthousen/barretthousen)
-- [Auction terms glossary](https://auctionsneapolitan.com/Auction-Terms-Glossary.html)
 - [User's Site](https://barretthousen.com)
 - [Admin Panel](https://admin.barretthousen.com)
-
+- [(Staging) User's Site](https://beta.barretthousen.com)
-
+- [(Staging) Admin Panel](https://beta.admin.barretthousen.com)
-### Goals
-
-1. Ability to search upcoming and live auctions across major auction sites
-2. Get an email digest of upcoming auctions for the week
-
-Future goals
-
-1. Get email alerts when these auctions are about to go live
-2. Specify search criteria for email alerts
-
 
 ## Technology
 
-For Users
+This project is for fun and learning so there are a lot of tools at play:
-- [Prod](https://barretthousen.com)
+
-- [Staging](https://beta.barretthousen.com)
+Infra
+- [Tilt](): local development env orchestration
+- [Ctlptl](): Automate local kubernetes clusters for local dev
+- [Docker]()
+- [Postgres]():
+- [Traefik]():
+- [Kustomize]():
+- [Helm]():
+
+Backend
+- [Go]()
+- [goose](): Go SQL db migration tool
+- [Sqlc](): Generate Go code to query SQL databases from migrations in a type safe way
+- [Buf](): Nice frontend for protoc and .proto libraries
+- [Protobufs]():
+
+Frontend
+- [Typescript]():
+- [SvelteKit]():
+- [TailwindCSS]():
 
-For Admins
 
-- [Prod](https://beta.barretthousen.com)
-- [Staging](https://admin.beta.barretthousen.com)
 - [Architecture Diagram (use diagrams.net)](./doc/Barretthousen_mvp.drawio)
-
 ![Arcchitecure Diagram MVP](./doc/Barretthousen_mvp.drawio.svg)
 
 ### Services
 
-- Web-client
+> [Auction terms glossary](https://auctionsneapolitan.com/Auction-Terms-Glossary.html)
+
+- [Web-client](./src/web-client/)
 Frontend site that users access to browse upcoming auctions and subscribe for updates to their searches
 
-- Catalog
+- [Admin-client](./src/admin-client/)
+Enables site administrators to kicks off scrape and email jobs in Runner and Ringman, and view status on current/past jobs.
+
+- [Catalog](./src/catalog/)
 API for searching upcoming auctions.
 
-- Ringman
+- [Runner](./src/runner/)
-Manages email subscriptions and sending emails to users about upcoming auctions.
-
-- Runner
 Scrapes sites for upcoming auctions.
 Transforms the data into a suitable shape and stores it for use by the frontend.
 
-- Auctioneer
+- [Ringman](./src/ringman/)
-Enables site administrators to kicks off scrape and email jobs in Runner and Ringman, and view status on current/past jobs.
+Manages email subscriptions and sending emails to users about upcoming auctions.
 
-- BHDB
+- [Proxy-web](./src/proxy-web/)
-Monolithic postgres datbabase tying it all together
+Proxies JSON HTTP API requests for the web client to GRPC based services downstream
+
+- [Proxy-admin](./src/proxy-admin/)
+Proxies JSON HTTP API requests for the admin client to GRPC based services downstream
 
 
-### Contributing
+## Contributing
+
+You will need the following prereqs installed:
+- Docker
+- Make
+- [asdf](https://asdf-vm.com/guide/getting-started.html)
 
 
-Install `asdf` and run `hack/asdf_plugin_setup`
+Everything you need is in the makefile:
 
 ```sh
-# install asdf tools, build tooling, encryption key for sops
+# install asdf tools, sqlc, buf
 make setup
 
 # generate protobufs and sql boilerplate
@@ -76,8 +90,14 @@ make gen
 make dev
 
 # build production docker images for the backend microservices, optionally push to the respective env
-make build-backend-image SERVICE=[catalog, runner, proxy-client, proxy-admin] [ENV=[beta, prod]]
+make build-backend-image SERVICE=[catalog, runner, proxy-web, proxy-admin] [ENV=[beta, prod]]
 
 # build client docker image for web frontends, optionally push to the respective env
-make build-client-image SERVICE=[web-client] [ENV=[beta, prod]]
+make build-client-image SERVICE=[web-client, admin-client] [ENV=[beta, prod]]
+
+# acceptance tests
+make acceptance-test SERVICE=[runner,catalog] ORIGIN=[beta.barretthousen.com,beta.admin.barretthousen.com]
 ```
+
+
+For how to deploy [read me](./doc/deployment.md).

Tiltfile

@@ -51,32 +51,6 @@ helm_resource(
     labels=["9-data"]
 )
 
-helm_resource(
-    'kafka',
-    'bitnami/kafka',
-    namespace='barretthousen-local',
-    flags=[
-        '--set', 'numIoThreads=1',
-        '--set', 'numNetworkThreads=1',
-    ],
-    port_forwards=[port_forward(9092, 9092, name='kafka')],
-    resource_deps=['bitnami'],
-    labels=["9-data"]
-)
-
-helm_resource(
-    'kafka-ui',
-    'kafka-ui-github/kafka-ui',
-    namespace='barretthousen-local',
-    flags=[
-         '--set', 'envs.config.KAFKA_CLUSTERS_0_NAME=bh-kafka',
-         '--set', 'envs.config.KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:9092'
-    ],
-    port_forwards=[port_forward(9090, 8080, name='kafka-ui')],
-    resource_deps=['kafka'],
-    labels=["9-data"]
-)
-
 def bh_client(service="", port_forwards=[], labels=['2-services'], deps=['ingress']):
     basepath = './src/{}-client'.format(service)
     docker_build(
@@ -84,6 +58,9 @@ def bh_client(service="", port_forwards=[], labels=['2-services'], deps=['ingres
         context=basepath,
         dockerfile='./src/Dockerfile.frontend'.format(service),
         target='development',
+        build_args={
+            "service": service
+        },
         entrypoint='vite dev --port=80 --host=0.0.0.0 --strictPort --logLevel info',
         live_update=[
             sync(basepath + '/src', '/opt/{}-client/src'.format(service)),
@@ -141,7 +118,7 @@ def bh_backend_service(service="", port_forwards=[], migrateDB=False, devMode=Tr
             './src/runner',
             './src/catalog',
             './src/proxy-admin',
-            './src/proxy-client',
+            './src/proxy-web',
             './src/lib'
         ],
         live_update=[
@@ -168,7 +145,6 @@ k8s_resource(
 )
 
 
-
 bh_backend_service(service="runner", migrateDB=True, port_forwards=[
     port_forward(2345, name='Delve port')
 ])
@@ -181,7 +157,7 @@ bh_backend_service(service="proxy-admin", port_forwards=[
     port_forward(8082, 80, name="HTTP API @ localhost:8082")
 ], deps=['ingress'])
 
-bh_backend_service(service="proxy-client", port_forwards=[
+bh_backend_service(service="proxy-web", port_forwards=[
     port_forward(8081, 80, name="HTTP API @ localhost:8081")
 ], deps=['ingress'])
 

doc/deployment.md

@@ -5,7 +5,7 @@ Services deployed (using kustomize):
 - [catalog](./src/catalog)
 - [runner](./src/runner)
 - [proxy-admin](./src/proxy-admin)
-- [proxy-client](./src/proxy-client)
+- [proxy-web](./src/proxy-web)
 
 - [web-client](./src/web-client)
 - [admin-client](./src/admin-client)
@@ -61,7 +61,7 @@ helm install --upgrade bh-db bitnami/postgresql -n 'barretthousen-beta' \
 make build-backend-image SERVICE=catalog ENV=beta
 make build-backend-image SERVICE=runner ENV=beta
 make build-backend-image SERVICE=proxy-admin ENV=beta
-make build-backend-image SERVICE=proxy-client ENV=beta
+make build-backend-image SERVICE=proxy-web ENV=beta
 
 make build-client-image SERVICE=web-client ENV=beta ORIGIN=https://beta.barretthousen.com
 make build-client-image SERVICE=admin-client ENV=beta ORIGIN=https://beta.admin.barretthousen.com
@@ -78,8 +78,8 @@ kubectl rollout status -n barretthousen-beta deployment catalog-beta  -w
 kubectl rollout restart -n barretthousen-beta deployment proxy-admin-beta
 kubectl rollout status -n barretthousen-beta deployment proxy-admin-beta -w
 
-kubectl rollout restart -n barretthousen-beta deployment proxy-client-beta
+kubectl rollout restart -n barretthousen-beta deployment proxy-web-beta
-kubectl rollout status -n barretthousen-beta deployment proxy-client-beta -w
+kubectl rollout status -n barretthousen-beta deployment proxy-web-beta -w
 
 kubectl rollout restart -n barretthousen-beta deployment web-client-beta
 kubectl rollout status -n barretthousen-beta deployment web-client-beta -w

env/base/client-ingress.yaml

@@ -13,7 +13,7 @@ spec:
             path: "/api"
             backend:
               service:
-                name: proxy-client
+                name: proxy-web
                 port:
                   number: 80
           - pathType: ImplementationSpecific

env/base/kustomization.yaml

@@ -4,7 +4,7 @@ resources:
   - ./catalog-deployment.yaml
   - ./runner-deployment.yaml
   - ./proxy-admin-deployment.yaml
-  - ./proxy-client-deployment.yaml
+  - ./proxy-web-deployment.yaml
   - ./sync-cronjob.yaml
   - ./web-client-deployment.yaml
   - ./admin-client-deployment.yaml

env/base/proxy-web-deployment.yaml

@@ -1,45 +1,45 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: proxy-client
+  name: proxy-web
 spec:
   selector:
     matchLabels:
-      service: proxy-client
+      service: proxy-web
   template:
     metadata:
       labels:
-        service: proxy-client
+        service: proxy-web
     spec:
       serviceAccountName: barretthousen-service
       containers:
-        - name: proxy-client
+        - name: proxy-web
-          image: barretthousen/service-proxy-client:latest
+          image: barretthousen/service-proxy-web:latest
           imagePullPolicy: Always
           ports:
             - containerPort: 80
               name: http
           command:
-            - /opt/proxy-client
+            - /opt/proxy-web
           resources:
             limits:
               memory: "128Mi"
               cpu: "250m"
           volumeMounts:
-            - name: proxy-client-config
+            - name: proxy-web-config
               mountPath: /config/
       volumes:
-        - name: proxy-client-config
+        - name: proxy-web-config
           secret:
-            secretName: proxy-client-config
+            secretName: proxy-web-config
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: proxy-client
+  name: proxy-web
 spec:
   selector:
-    service: proxy-client
+    service: proxy-web
   ports:
     - port: 80
       targetPort: 80
@@ -47,7 +47,7 @@ spec:
 apiVersion: v1
 kind: Secret
 metadata:
-  name: proxy-client-config
+  name: proxy-web-config
 stringData:
   config.yaml: |
     log_level: 2

env/beta/kustomization.yaml

@@ -9,7 +9,7 @@ patchesStrategicMerge:
   - sync-cronjob.yaml
   - catalog-secret.yaml
   - proxy-admin-secret.yaml
-  - proxy-client-secret.yaml
+  - proxy-web-secret.yaml
   - runner-secret.yaml
 
 patches:
@@ -62,11 +62,11 @@ patches:
 
   - target:
       kind: Deployment
-      name: proxy-client
+      name: proxy-web
     patch: |-
       - op: replace
         path: /spec/template/spec/containers/0/image
-        value: git.vdhsn.com/barretthousen/service-proxy-client:beta
+        value: git.vdhsn.com/barretthousen/service-proxy-web:beta
   - target:
       kind: Deployment
       name: web-client

env/beta/proxy-client-secret.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: proxy-client-config
+  name: proxy-web-config
 stringData:
   config.yaml: |
     log_level: 2

env/local/image-pull-secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: bh-registry
+type: kubernetes.io/dockerconfigjson
+stringData:
+    .dockerconfigjson: |
+        { "auths": {} }

env/local/kustomization.yaml

@@ -11,6 +11,7 @@ patchesStrategicMerge:
   - debug-catalog.yaml
   - debug-runner.yaml
   - sync-cronjob.yaml
+  - image-pull-secret.yaml
 
 patches:
   - target:

env/prod/kustomization.yaml

@@ -9,7 +9,7 @@ patchesStrategicMerge:
   - sync-cronjob.yaml
   - catalog-configmap.yaml
   - proxy-admin-configmap.yaml
-  - proxy-client-configmap.yaml
+  - proxy-web-configmap.yaml
   - runner-configmap.yaml
 
 patches:
@@ -50,11 +50,11 @@ patches:
         value: git.vdhsn.com/barretthousen/service-proxy-admin:prod
   - target:
       kind: Deployment
-      name: proxy-client
+      name: proxy-web
     patch: |-
       - op: replace
         path: /spec/template/spec/containers/0/image
-        value: git.vdhsn.com/barretthousen/service-proxy-client:prod
+        value: git.vdhsn.com/barretthousen/service-proxy-web:prod
   - target:
       kind: Deployment
       name: web-client

env/prod/proxy-client-configmap.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: proxy-client-config
+  name: proxy-web-config
 data:
   config.yaml: |
     log_level: 2

go.work

@@ -4,6 +4,6 @@ use (
 	./src/catalog
 	./src/lib
 	./src/proxy-admin
-	./src/proxy-client
+	./src/proxy-web
 	./src/runner
 )

go.work.sum

@@ -163,7 +163,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5
 github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
 github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
 github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
-github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0=
+github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU=
+github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.uber.org/dig v1.0.0/go.mod h1:z+dSd2TP9Usi48jL8M3v63iSBVkiwtVyMKxMZYYauPg=

src/.dockerignore

@@ -1,2 +1 @@
-web-client
 .idea

src/Dockerfile.tools

@@ -1,4 +0,0 @@
-FROM golang:1.19-alpine as builder
-
-RUN go install github.com/bufbuild/buf/cmd/buf@v1.17.0
-RUN go install github.com/kyleconroy/sqlc/cmd/sqlc@latest

src/admin-client/README.md

@@ -1,38 +1,12 @@
-# create-svelte
+# Admin client
 
-Everything you need to build a Svelte project, powered by [`create-svelte`](https://github.com/sveltejs/kit/tree/master/packages/create-svelte).
+Admin panel to drive backend "administrative" functionality
 
-## Creating a project
+Uses [SvelteKit](), [TailwindCSS](), [Typescript (poorly)]()
 
-If you're seeing this, you've probably already done this step. Congrats!
 
-```bash
+## Development
-# create a new project in the current directory
-npm create svelte@latest
 
-# create a new project in my-app
+Follow the [Contributing section](../../README.md#contributing) to get the environment stood up locally.
-npm create svelte@latest my-app
-```
 
-## Developing
+Browse to [http://admin.localhost:8000](http://admin.localhost:8000).
-
-Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
-
-```bash
-npm run dev
-
-# or start the server and open the app in a new browser tab
-npm run dev -- --open
-```
-
-## Building
-
-To create a production version of your app:
-
-```bash
-npm run build
-```
-
-You can preview the production build with `npm run preview`.
-
-> To deploy your app, you may need to install an [adapter](https://kit.svelte.dev/docs/adapters) for your target environment.

src/catalog/acceptance-tests.yml

@@ -1,16 +1,7 @@
 tests:
-  - description: "trigger scrape"
+  - description: "Search for top"
     request:
-      path: "/v1/findNewUpcoming"
+      path: "/api/v1/upcoming"
-      method: "POST"
+      method: "GET"
-      headers:
-        authorization: "token ${SECRET_AUTH_TOKEN}"
-    response:
-      statusCodes: [200]
-  - description: "get list of jobs"
-    request:
-      path: "/v1/jobs"
-      headers:
-        authorization: "token ${SECRET_AUTH_TOKEN}"
     response:
       statusCodes: [200]

src/catalog/config.yaml

@@ -1,3 +1,16 @@
-log_level: ERROR
+log_level: 2
-
+port: 5001
-service: {}
+db_service:
+  scheme: postgres
+  port: 5432
+  host: bh-db
+  name: bh
+  user: catalog-service
+  password: catalog-service
+db_migrate:
+  scheme: postgres
+  port: 5432
+  host: bh-db
+  name: bh
+  user: postgres
+  password: bh-admin

src/proxy-web/go.mod

@@ -1,4 +1,4 @@
-module git.vdhsn.com/barretthousen/barretthousen/src/proxy-client
+module git.vdhsn.com/barretthousen/barretthousen/src/proxy-web
 
 go 1.19
 

src/runner/acceptance-tests.yml

@@ -1,16 +1,12 @@
 tests:
-  - description: "trigger scrape"
+  - description: "trigger sync"
     request:
-      path: "/v1/findNewUpcoming"
+      path: "/v1/sync"
-      method: "POST"
+      method: "PUT"
-      headers:
-        authorization: "token ${SECRET_AUTH_TOKEN}"
     response:
       statusCodes: [200]
   - description: "get list of jobs"
     request:
-      path: "/v1/jobs"
+      path: "/v1/sync"
-      headers:
-        authorization: "token ${SECRET_AUTH_TOKEN}"
     response:
       statusCodes: [200]

src/runner/config.yaml

@@ -1,3 +1,16 @@
-log_level: ERROR
+log_level: 2
-
+port: 5001
-service: {}
+db_service:
+  scheme: postgres
+  port: 5432
+  host: bh-db
+  name: bh
+  user: runner-service
+  password: runner-service
+db_migrate:
+  scheme: postgres
+  port: 5432
+  host: bh-db
+  name: bh
+  user: postgres
+  password: bh-admin

src/web-client/README.md

@@ -1,38 +1,12 @@
-# create-svelte
+# Web client
 
-Everything you need to build a Svelte project, powered by [`create-svelte`](https://github.com/sveltejs/kit/tree/master/packages/create-svelte).
+A SSR web app that allows users to search for upcoming auctions
 
-## Creating a project
+Uses [SvelteKit](), [TailwindCSS](), [Typescript (poorly)]()
 
-If you're seeing this, you've probably already done this step. Congrats!
 
-```bash
+## Development
-# create a new project in the current directory
-npm create svelte@latest
 
-# create a new project in my-app
+Follow the [Contributing section](../../README.md#contributing) to get the environment stood up locally.
-npm create svelte@latest my-app
-```
 
-## Developing
+Browse to [http://bh.localhost:8000](http://bh.localhost:8000).
-
-Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
-
-```bash
-npm run dev
-
-# or start the server and open the app in a new browser tab
-npm run dev -- --open
-```
-
-## Building
-
-To create a production version of your app:
-
-```bash
-npm run build
-```
-
-You can preview the production build with `npm run preview`.
-
-> To deploy your app, you may need to install an [adapter](https://kit.svelte.dev/docs/adapters) for your target environment.

src/web-client/src/routes/+page.ts

@@ -2,7 +2,7 @@ import { browser } from '$app/environment';
 import type { PageLoad } from './$types';
 
 // TODO: change to env var
-const API_HOST = `${browser ? '' : 'http://proxy-client-local'}/api/v1`;
+const API_HOST = `${browser ? '' : 'http://proxy-web-local'}/api/v1`;
 
 interface SearchPageData {
 	page: number;