remove GA for now

- drone ci build using kaniko

This builds images extremely slowly, I've implemented some custom fixes to the
plugin to improve performance but it's still slow to build images. I will
try to implement a DinD solution.
There are issues in kaniko about this: https://github.com/GoogleContainerTools/kaniko/issues/875

Squashed commit of the following:

commit 1fd65ad139
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Sat Jul 15 18:17:55 2023 -0500

    add deploy script

commit 075e15f218
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Sat Jul 15 17:53:24 2023 -0500

    bump limits

commit 2d45234e7b
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Sat Jul 15 17:51:19 2023 -0500

    sync button in tilt, support syncing all targets simultaneously

commit 75e73a1171
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Sat Jul 15 17:03:50 2023 -0500

    create admin user on startup

commit 40cf74560d
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Fri Jul 14 11:14:00 2023 -0500

    test more cache

commit 1fbbfe7548
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Fri Jul 14 11:11:34 2023 -0500

    test tmpfs

commit 0f48c098d9
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Fri Jul 14 11:08:32 2023 -0500

    add tmpfs

commit d37142cfef
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Fri Jul 14 11:02:39 2023 -0500

    crank verbosity

commit 178eb5bc38
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Fri Jul 14 10:44:08 2023 -0500

    bump requests

commit b0aa0b39f0
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Fri Jul 14 10:40:04 2023 -0500

    test fewer concurrent builds

commit 221e378a53
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Thu Jul 13 20:40:18 2023 -0500

    test

commit 6c03914341
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Thu Jul 13 19:01:22 2023 -0500

    snapshot mode for faster build

commit 91c56d430e
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 22:40:12 2023 -0500

    try script

commit 4a119c2523
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 22:28:41 2023 -0500

    test promote

commit b1062a4001
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 22:26:58 2023 -0500

    test

commit c06804dfe6
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 17:47:48 2023 -0500

    beta promote and ignore

commit 643e104c37
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 16:21:39 2023 -0500

    tweakies

commit 478e65ac70
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 16:09:54 2023 -0500

    un woops

commit 966d48c193
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 15:23:08 2023 -0500

    woops

commit 2829743d17
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 15:22:03 2023 -0500

    test internal routing

commit e08e68b8ad
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 14:07:31 2023 -0500

    push images

commit 5705bca068
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 13:40:48 2023 -0500

    ugh

commit 1699b08d5f
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 13:39:49 2023 -0500

    fixup client builds

commit 9ec8616ae1
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 13:36:36 2023 -0500

    update go mods

commit 163b25fd67
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 02:11:02 2023 -0500

    fix fronend builds

commit e5cc7d5c79
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 01:56:08 2023 -0500

    resources

commit 8088b95c1e
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 01:38:21 2023 -0500

    big test

commit 38fef91c49
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 01:26:09 2023 -0500

    testicles

commit 53baa0a647
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 01:24:50 2023 -0500

    testicles

commit 16901ec5bb
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 01:23:38 2023 -0500

    testicles

commit 3478d4f512
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 01:19:58 2023 -0500

    test 2

commit 16298ada78
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 01:13:53 2023 -0500

    more test

commit 80524a1d06
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 01:09:03 2023 -0500

    test

commit b9916204de
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 00:19:41 2023 -0500

    tools image

commit 6e414b3a1b
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Wed Jul 12 00:00:23 2023 -0500

    test

commit abb6183e88
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Tue Jul 11 22:55:58 2023 -0500

    added catawiki scrape

commit 2643817e93
Author: Adam Veldhousen <adamveld12@gmail.com>
Date:   Tue Jul 11 14:40:05 2023 -0500

    add catawiki syncer

.drone.yml

@@ -1,41 +1,296 @@
 kind: pipeline
 type: kubernetes
-name: Build Images
+name: Build & Push Images
 
 trigger:
+  branch:
+    - trunk
   event:
     - push
 
 steps:
   - name: Build & Publish Catalog
-    image: plugins/kaniko
+    image: git.vdhsn.com/barretthousen/drone-kaniko:v1.0.1
+    volumes:
+      - name: cache
+        path: /kaniko
     settings:
+      verbosity: debug
       dockerfile: "./src/Dockerfile.prod-backend"
       context: "./src"
+      target: production
       registry: git.vdhsn.com
       repo: git.vdhsn.com/barretthousen/service-catalog
+      enable_cache: true
+      cache_repo: git.vdhsn.com/barretthousen/ci-cache
+      skip_unused_stages: true
+      build_args:
+        - "service=catalog"
+      tags:
+        - ${DRONE_COMMIT_SHA}
+        - beta
+      custom_labels:
+        - com.barretthousen.service=catalog
+        - com.barretthousen.version=${DRONE_COMMIT_SHA}
+        - com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
+        - com.barretthousen.build-date=${DRONE_BUILD_STARTED}
+        - com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
       username:
         from_secret: DOCKER_USERNAME
       password:
         from_secret: DOCKER_PASSWORD
-      args:
+
-        - "service=catalog"
-      no-push: true
-      tags:
-        - ${DRONE_COMMIT_SHA}
   - name: Build & Publish Runner
     image: plugins/kaniko
     settings:
+      verbosity: debug
       dockerfile: "./src/Dockerfile.prod-backend"
       context: "./src"
+      target: production
       registry: git.vdhsn.com
       repo: git.vdhsn.com/barretthousen/service-runner
+      enable_cache: true
+      cache_repo: git.vdhsn.com/barretthousen/ci-cache
+      snapshot_mode: redo
+      skip_unused_stages: true
+      build_args:
+        - "service=runner"
+      tags:
+        - ${DRONE_COMMIT_SHA}
+      custom_labels:
+        - com.barretthousen.service=runner
+        - com.barretthousen.version=${DRONE_COMMIT_SHA}
+        - com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
+        - com.barretthousen.build-date=${DRONE_BUILD_STARTED}
+        - com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
       username:
         from_secret: DOCKER_USERNAME
       password:
         from_secret: DOCKER_PASSWORD
-      args:
+
-        - "service=runner"
+  - name: Build & Publish Auth
-      no-push: true
+    image: plugins/kaniko
+    settings:
+      verbosity: debug
+      dockerfile: "./src/Dockerfile.prod-backend"
+      context: "./src"
+      target: production
+      registry: git.vdhsn.com
+      repo: git.vdhsn.com/barretthousen/service-auth
+      enable_cache: true
+      cache_repo: git.vdhsn.com/barretthousen/ci-cache
+      snapshot_mode: redo
+      skip_unused_stages: true
+      build_args:
+        - "service=auth"
       tags:
         - ${DRONE_COMMIT_SHA}
+        - beta
+      custom_labels:
+        - com.barretthousen.service=auth
+        - com.barretthousen.version=${DRONE_COMMIT_SHA}
+        - com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
+        - com.barretthousen.build-date=${DRONE_BUILD_STARTED}
+        - com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
+      username:
+        from_secret: DOCKER_USERNAME
+      password:
+        from_secret: DOCKER_PASSWORD
+
+  - name: Build & Publish Proxy Admin
+    image: plugins/kaniko
+    settings:
+      verbosity: debug
+      dockerfile: "./src/Dockerfile.prod-backend"
+      context: "./src"
+      target: production
+      registry: git.vdhsn.com
+      repo: git.vdhsn.com/barretthousen/service-proxy-admin
+      enable_cache: true
+      cache_repo: git.vdhsn.com/barretthousen/ci-cache
+      snapshot_mode: redo
+      skip_unused_stages: true
+      build_args:
+        - "service=proxy-admin"
+      tags:
+        - ${DRONE_COMMIT_SHA}
+        - beta
+      custom_labels:
+        - com.barretthousen.service=proxy-admin
+        - com.barretthousen.version=${DRONE_COMMIT_SHA}
+        - com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
+        - com.barretthousen.build-date=${DRONE_BUILD_STARTED}
+        - com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
+      username:
+        from_secret: DOCKER_USERNAME
+      password:
+        from_secret: DOCKER_PASSWORD
+    depends_on:
+      - Build & Publish Catalog
+      - Build & Publish Runner
+      - Build & Publish Auth
+
+  - name: Build & Publish Proxy Web
+    image: plugins/kaniko
+    settings:
+      verbosity: debug
+      dockerfile: "./src/Dockerfile.prod-backend"
+      context: "./src"
+      target: production
+      registry: git.vdhsn.com
+      repo: git.vdhsn.com/barretthousen/service-proxy-web
+      enable_cache: true
+      cache_repo: git.vdhsn.com/barretthousen/ci-cache
+      snapshot_mode: redo
+      skip_unused_stages: true
+      build_args:
+        - "service=proxy-web"
+      tags:
+        - ${DRONE_COMMIT_SHA}
+        - beta
+      custom_labels:
+        - com.barretthousen.service=proxy-web
+        - com.barretthousen.version=${DRONE_COMMIT_SHA}
+        - com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
+        - com.barretthousen.build-date=${DRONE_BUILD_STARTED}
+        - com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
+      username:
+        from_secret: DOCKER_USERNAME
+      password:
+        from_secret: DOCKER_PASSWORD
+    depends_on:
+      - Build & Publish Catalog
+      - Build & Publish Runner
+      - Build & Publish Auth
+
+  - name: Build & Publish Web Client
+    image: plugins/kaniko
+    settings:
+      verbosity: debug
+      dockerfile: "./src/Dockerfile.frontend"
+      context: "./src/web-client"
+      target: production
+      registry: git.vdhsn.com
+      repo: git.vdhsn.com/barretthousen/service-proxy-web
+      enable_cache: true
+      cache_repo: git.vdhsn.com/barretthousen/ci-cache
+      snapshot_mode: redo
+      skip_unused_stages: true
+      build_args:
+        - "service=proxy-web"
+      tags:
+        - ${DRONE_COMMIT_SHA}
+        - beta
+      custom_labels:
+        - com.barretthousen.service=proxy-web
+        - com.barretthousen.version=${DRONE_COMMIT_SHA}
+        - com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
+        - com.barretthousen.build-date=${DRONE_BUILD_STARTED}
+        - com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
+      username:
+        from_secret: DOCKER_USERNAME
+      password:
+        from_secret: DOCKER_PASSWORD
+    depends_on:
+      - Build & Publish Catalog
+      - Build & Publish Runner
+      - Build & Publish Auth
+      - Build & Publish Proxy Admin
+      - Build & Publish Proxy Web
+
+  - name: Build & Publish Admin Client
+    image: plugins/kaniko
+    settings:
+      verbosity: debug
+      dockerfile: "./src/Dockerfile.frontend"
+      context: "./src/admin-client"
+      target: production
+      registry: git.vdhsn.com
+      repo: git.vdhsn.com/barretthousen/client-admin-client
+      enable_cache: true
+      cache_repo: git.vdhsn.com/barretthousen/ci-cache
+      snapshot_mode: redo
+      skip_unused_stages: true
+      build_args:
+        - "service=admin-client"
+      tags:
+        - ${DRONE_COMMIT_SHA}
+        - beta
+      username:
+        from_secret: DOCKER_USERNAME
+      password:
+        from_secret: DOCKER_PASSWORD
+    depends_on:
+      - Build & Publish Catalog
+      - Build & Publish Runner
+      - Build & Publish Auth
+      - Build & Publish Proxy Admin
+      - Build & Publish Proxy Web
+
+  - name: Build Success Notify
+    image: appleboy/drone-discord
+    environment:
+      DISCORD_WEBHOOK_ID:
+        from_secret: DISCORD_WEBHOOK_ID
+      DISCORD_WEBHOOK_TOKEN:
+        from_secret: DISCORD_WEBHOOK_TOKEN
+    settings:
+      username: Drone CI
+      message: Barretthousen Image Builds Complete
+      webhook_id:
+        from_secret: DISCORD_WEBHOOK_ID
+      webhook_token:
+        from_secret: DISCORD_WEBHOOK_TOKEN
+    depends_on:
+      - Build & Publish Catalog
+      - Build & Publish Runner
+      - Build & Publish Auth
+      - Build & Publish Proxy Admin
+      - Build & Publish Proxy Web
+      - Build & Publish Web Client
+      - Build & Publish Admin Client
+---
+kind: pipeline
+type: kubernetes
+name: Promote Beta -> Prod
+
+trigger:
+  branch:
+    - trunk
+  event:
+    - promote
+
+steps:
+  - name: Promote Services
+    image: alpine
+    environment:
+      REGISTRY_NAME: "https://git.vdhsn.com"
+      CONTENT_TYPE: "application/vnd.docker.distribution.manifest.v2+json"
+      DOCKER_USERNAME:
+        from_secret: DOCKER_USERNAME
+      DOCKER_PASSWORD:
+        from_secret: DOCKER_PASSWORD
+    commands:
+      - ./hack/promote.sh barretthousen/catalog beta prod
+
+  - name: Deploy Success
+    image: appleboy/drone-discord
+    environment:
+      DISCORD_WEBHOOK_ID:
+        from_secret: DISCORD_WEBHOOK_ID
+      DISCORD_WEBHOOK_TOKEN:
+        from_secret: DISCORD_WEBHOOK_TOKEN
+    settings:
+      username: Drone CI
+      message: Barretthousen Prod deployment success
+      webhook_id:
+        from_secret: DISCORD_WEBHOOK_ID
+      webhook_token:
+        from_secret: DISCORD_WEBHOOK_TOKEN
+    depends_on:
+      - Promote Services
+
+volumes:
+- name: cache
+  temp:
+    medium: memory

Makefile

@@ -52,6 +52,20 @@ VERSION = $(shell git rev-parse --verify --short HEAD)
 GIT_REF = $(shell git rev-parse --verify HEAD)
 BUILD_DATE := $(shell date +%Y-%m-%d-%T)
 
+.PHONY: build-tools-image
+build-tools-image:
+	@docker build \
+		--label 'com.barretthousen.version=$(VERSION)' \
+		--label 'com.barretthousen.git-ref=$(GIT_REF)' \
+		--label 'com.barretthousen.build-date=$(BUILD_DATE)' \
+		--label 'com.barrethousen.builder=$(BUILD_INITIATOR)' \
+		-t git.vdhsn.com/barretthousen/tools):$(VERSION) \
+		-t git.vdhsn.com/barretthousen/tools):v1 \
+		-f ./src/Dockerfile.tools ./src
+
+    @docker push git.vdhsn.com/barretthousen/tools:$(VERSION)
+	@docker push git.vdhsn.com/barretthousen/tools:v1
+
 .PHONY: build-client-image
 build-client-image:
 	@docker build --target=production \
@@ -66,7 +80,7 @@ build-client-image:
 		-f ./src/Dockerfile.frontend ./src/$(SERVICE)
 
 	@docker push git.vdhsn.com/barretthousen/client-$(SERVICE):$(VERSION)
-	@[ ! -z $(ENV) ] && docker tag git.vdhsn.com/barretthousen/client-$(SERVICE):$(VERSION) git.vdhsn.com/barretthousen/client-$(SERVICE):beta || true
+	@[ ! -z $(ENV) ] && docker tag git.vdhsn.com/barretthousen/client-$(SERVICE):$(VERSION) git.vdhsn.com/barretthousen/client-$(SERVICE):$(ENV) || true
 	@[ ! -z $(ENV) ] && docker push git.vdhsn.com/barretthousen/client-$(SERVICE):$(ENV) || true
 
 .PHONY: build-backend-image
@@ -84,8 +98,8 @@ build-backend-image:
 
 	@docker push git.vdhsn.com/barretthousen/service-$(SERVICE):$(VERSION)
 
-	@[ ! -z $(ENV) ] && docker tag git.vdhsn.com/barretthousen/service-$(SERVICE):$(VERSION) git.vdhsn.com/barretthousen/service-$(SERVICE):beta || true
+	@[ ! -z $(ENV) ] && docker tag git.vdhsn.com/barretthousen/service-$(SERVICE):$(VERSION) git.vdhsn.com/barretthousen/service-$(SERVICE):$(ENV) || true
-	@[ ! -z $(ENV) ] && docker push git.vdhsn.com/barretthousen/service-$(SERVICE):beta || true
+	@[ ! -z $(ENV) ] && docker push git.vdhsn.com/barretthousen/service-$(SERVICE):$(ENV) || true
 
 # TODO: investigate tagging without needing to pull? https://dille.name/blog/2018/09/20/how-to-tag-docker-images-without-pulling-them/
 .PHONY: promote-client-prod
@@ -124,3 +138,4 @@ age_identity=$(shell sops -d ./env/master.json)
 	@echo "# public key: $(shell echo '$(age_identity)' | jq -r '.public_key')" >> $@
 	@echo "$(shell echo '$(age_identity)' | jq -r '.private_key')" >> $@
 	@echo "$@ created!"
+	@echo "export SOPS_AGE_KEY_FILE=$(PWD)/env/.age.txt"

README.md

@@ -1,5 +1,6 @@
 # [Barretthousen](https://barretthousen.com)
 
+[![Build Status](https://ci.vdhsn.com/api/badges/Barretthousen/barretthousen/status.svg?ref=refs/heads/trunk)](https://ci.vdhsn.com/Barretthousen/barretthousen)
 Search and get alerts for items across the most popular auction sites.
 
 Built with microservice architecture and various fancy tools for learning purposes

Tiltfile

@@ -7,6 +7,7 @@ print("""
 load('ext://helm_resource', 'helm_resource', 'helm_repo')
 load('ext://deployment', 'deployment_create')
 load('ext://restart_process', 'docker_build_with_restart')
+load('ext://uibutton', 'cmd_button', 'location', 'text_input')
 
 helm_repo('bitnami', 'https://charts.bitnami.com/bitnami', labels=["9-repos"])
 helm_repo('traefik', 'https://traefik.github.io/charts', labels=["9-repos"])
@@ -166,3 +167,34 @@ bh_backend_service(service="proxy-web", port_forwards=[
 
 bh_client(service='web', deps=["proxy-web-local"])
 bh_client(service='admin', deps=["proxy-admin-local"])
+
+createAdminCmd = [
+    './hack/create_user.sh', 'admin@barretthousen.com', 'admin', 'ADMINISTRATOR'
+]
+
+loginAdminCmd = [
+    'curl', '-vvvv', '-X POST', '-H "Content-Type: application/json"',
+    '-d "{\"email\":\"admin@barretthousen.com\", \"password\":\"admin\"}"',
+    'http://bh.localhost:8000/api/v1/user'
+]
+
+syncCatalogsCmd = [
+    'curl', '-vvvv', '-X', 'PUT',
+    '-H', 'Content-Type: application/json',
+    '-H', 'bh-session-id: 2',
+    '-d', '{\"targetSite\":\"All\"}',
+    'http://admin.localhost:8000/api/v1/sync'
+]
+
+local(createAdminCmd, quiet=False, echo_off=True)
+local(loginAdminCmd, quiet=False, echo_off=True)
+local(syncCatalogsCmd, quiet=False, echo_off=True)
+
+cmd_button(
+    name='Sync Catalogs',
+    argv=syncCatalogsCmd,
+    text='Sync Catalogs',
+    location=location.NAV,
+    icon_name='sync')
+
+

doc/deployment.md

@@ -61,6 +61,7 @@ helm install --upgrade bh-db bitnami/postgresql -n 'barretthousen-beta' \
 export ENV=beta; # if not specified, will only build and push for the git commit
 make build-backend-image SERVICE=catalog
 make build-backend-image SERVICE=runner
+make build-backend-image SERVICE=auth
 make build-backend-image SERVICE=proxy-admin
 make build-backend-image SERVICE=proxy-web
 
@@ -78,6 +79,9 @@ kubectl rollout status -n barretthousen-beta deployment runner-beta -w
 kubectl rollout restart -n barretthousen-beta deployment catalog-beta
 kubectl rollout status -n barretthousen-beta deployment catalog-beta  -w
 
+kubectl rollout restart -n barretthousen-beta deployment auth-beta
+kubectl rollout status -n barretthousen-beta deployment auth-beta  -w
+
 kubectl rollout restart -n barretthousen-beta deployment proxy-admin-beta
 kubectl rollout status -n barretthousen-beta deployment proxy-admin-beta -w
 
@@ -95,6 +99,7 @@ kubectl rollout status -n barretthousen-beta deployment admin-client-beta -w
 ```sh
 make promote-backend-prod SERVICE=catalog
 make promote-backend-prod SERVICE=runner
+make promote-backend-prod SERVICE=auth
 make promote-backend-prod SERVICE=proxy-admin
 make promote-backend-prod SERVICE=proxy-web
 

env/base/auth-deployment.yaml

@@ -46,6 +46,7 @@ spec:
   ports:
     - port: 5001
       targetPort: 5001
+      name: grpc
 ---
 apiVersion: v1
 kind: Secret

env/base/catalog-deployment.yaml

@@ -46,6 +46,7 @@ spec:
   ports:
     - port: 5001
       targetPort: 5001
+      name: grpc
 ---
 apiVersion: v1
 kind: Secret

env/base/runner-deployment.yaml

@@ -46,7 +46,7 @@ spec:
   ports:
     - port: 5001
       targetPort: 5001
-
+      name: grpc
 ---
 apiVersion: v1
 kind: Secret

env/beta/auth-secret.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: auth-config
+stringData:
+  config.yaml: |
+    log_level: 2
+    port: 5001
+    db_service:
+        scheme: postgres
+        port: 5432
+        host: bh-db
+        name: bh
+        user: auth-service
+        password: auth-service
+    db_migrate:
+        scheme: postgres
+        port: 5432
+        host: bh-db
+        name: bh
+        user: postgres
+        password: bh-admin-beta

env/beta/kustomization.yaml

@@ -7,10 +7,11 @@ namespace: barretthousen-beta
 
 patchesStrategicMerge:
   - sync-cronjob.yaml
+  - auth-secret.yaml
   - catalog-secret.yaml
+  - runner-secret.yaml
   - proxy-admin-secret.yaml
   - proxy-web-secret.yaml
-  - runner-secret.yaml
 
 patches:
   - target:
@@ -27,6 +28,13 @@ patches:
       - op: replace
         path: /spec/rules/0/host
         value: beta.barretthousen.com
+  - target:
+      kind: Deployment
+      name: auth
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/containers/0/image
+        value: git.vdhsn.com/barretthousen/service-auth:beta
   - target:
       kind: Deployment
       name: catalog

env/beta/proxy-web-secret.yaml

@@ -8,4 +8,5 @@ stringData:
     port: 80
     access_control_allow_origin: "beta.barretthousen.com"
     endpoints:
+      auth: auth-beta:5001
       catalog: catalog-beta:5001

env/prod/kustomization.yaml

@@ -8,6 +8,13 @@ patchesStrategicMerge:
   - sync-cronjob.yaml
 
 patches:
+  - target:
+      kind: Deployment
+      name: auth
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/containers/0/image
+        value: git.vdhsn.com/barretthousen/service-auth:prod
   - target:
       kind: Deployment
       name: catalog

env/prod/sync-cronjob.yaml

@@ -14,6 +14,6 @@ spec:
                 [
                   "curl",
                   "http://proxy-admin/api/v1/sync",
-                  '-d=''{"target":"liveauctioneers"}''',
+                  '-d=''{"target":"All"}''',
                   "-vvvv",
                 ]

go.work

@@ -1,10 +1,10 @@
 go 1.19
 
 use (
-	./src/lib
 	./src/auth
 	./src/catalog
-	./src/runner
+	./src/lib
 	./src/proxy-admin
 	./src/proxy-web
+	./src/runner
 )

hack/build_all.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+
+build(){
+	export ENV=${1:-beta}; # if not specified, will only build and push for the git commit
+	make build-backend-image SERVICE=auth &
+	make build-backend-image SERVICE=catalog &
+	make build-backend-image SERVICE=runner &
+	make build-backend-image SERVICE=proxy-admin &
+	make build-backend-image SERVICE=proxy-web &
+	time wait;
+
+	# for prod drop `beta prefix
+	make build-client-image SERVICE=web-client &
+	make build-client-image SERVICE=admin-client &
+	time wait;
+
+	kubectl rollout restart -n barretthousen-beta deployment runner-beta &
+	kubectl rollout restart -n barretthousen-beta deployment catalog-beta &
+	kubectl rollout restart -n barretthousen-beta deployment auth-beta &
+	kubectl rollout restart -n barretthousen-beta deployment proxy-admin-beta &
+	kubectl rollout restart -n barretthousen-beta deployment proxy-web-beta &
+	kubectl rollout restart -n barretthousen-beta deployment web-client-beta &
+	kubectl rollout restart -n barretthousen-beta deployment admin-client-beta &
+
+	kubectl rollout status -n barretthousen-beta deployment runner-beta -w
+	kubectl rollout status -n barretthousen-beta deployment catalog-beta  -w
+	kubectl rollout status -n barretthousen-beta deployment auth-beta -w
+	kubectl rollout status -n barretthousen-beta deployment proxy-admin-beta -w
+	kubectl rollout status -n barretthousen-beta deployment proxy-web-beta -w
+	kubectl rollout status -n barretthousen-beta deployment admin-client-beta -w
+	kubectl rollout status -n barretthousen-beta deployment web-client-beta -w
+}
+
+promote() {
+	make promote-backend-prod SERVICE=catalog &
+	make promote-backend-prod SERVICE=runner &
+	make promote-backend-prod SERVICE=auth &
+	make promote-backend-prod SERVICE=proxy-admin &
+	make promote-backend-prod SERVICE=proxy-web &
+
+	make promote-client-prod SERVICE=web-client &
+	make promote-client-prod SERVICE=admin-client &
+
+	time wait;
+
+	kubectl rollout restart -n barretthousen deployment runner &
+	kubectl rollout restart -n barretthousen deployment catalog &
+	kubectl rollout restart -n barretthousen deployment auth &
+	kubectl rollout restart -n barretthousen deployment proxy-admin &
+	kubectl rollout restart -n barretthousen deployment proxy-web &
+	kubectl rollout restart -n barretthousen deployment web-client &
+	kubectl rollout restart -n barretthousen deployment admin-client &
+
+	kubectl rollout status -n barretthousen deployment runner -w
+	kubectl rollout status -n barretthousen deployment catalog  -w
+	kubectl rollout status -n barretthousen deployment auth -w
+	kubectl rollout status -n barretthousen deployment proxy-admin -w
+	kubectl rollout status -n barretthousen deployment proxy-web -w
+	kubectl rollout status -n barretthousen deployment admin-client -w
+	kubectl rollout status -n barretthousen deployment web-client -w
+}
+
+build $@

hack/create_user.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+
+main(){
+	local EMAIL=${1};
+	if [ -z "${EMAIL}" ]; then
+		echo "first argument must be an EMAIL address"
+		exit 1;
+	fi
+
+	local PASSWORD=${2};
+	if [ -z "${PASSWORD}" ]; then
+		echo "second argument must be a PASSWORD"
+		exit 1;
+	fi
+
+	local ROLE=${3:-USER};
+	local HOST=${4:-"http://bh.localhost:8000"};
+
+	curl -X PUT -H 'Content-Type: application/json' \
+		-d "{ \"email\":\"${EMAIL}\", \"password\":\"${PASSWORD}\", \"role\":\"${ROLE}\" }" \
+		"${HOST}/api/v1/user";
+}
+
+
+main $@;

hack/kaniko-test.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+docker run --rm \
+    -e DRONE_COMMIT_REF=refs/heads/trunk \
+    -e DRONE_REPO_BRANCH=trunk \
+	-e PLUGIN_REGISTRY=git.vdhsn.com \
+    -e PLUGIN_REPO=git.vdhsn.com/barretthousen/service-catalog \
+    -e PLUGIN_BUILD_ARGS="service=catalog" \
+    -e PLUGIN_TAGS=$(git rev-parse --verify --short HEAD) \
+	-e PLUGIN_CONTEXT="./src" \
+	-e PLUGIN_DOCKERFILE="./src/Dockerfile.prod-backend" \
+	-e PLUGIN_USERNAME=${DOCKER_USERNAME} \
+    -e PLUGIN_PASSWORD=${DOCKER_PASSWORD} \
+	-e PLUGIN_ENABLE_CACHE=true \
+	-e PLUGIN_CACHE_REPO='git.vdhsn.com/barretthousen/ci-cache' \
+	-e PLUGIN_SKIP_UNUSED_STAGES=true \
+    -v $(pwd):/drone \
+    -w /drone \
+    plugins/kaniko:linux-amd64

hack/promote.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+
+main(){
+    local REPOSITORY=${1};
+    if [ -z "${REPOSITORY}" ]; then
+        echo "First argument must be container repository";
+        exit 1;
+    fi
+
+    local FROM=${2:-beta};
+    local TO=${3:-prod};
+
+    local CONTENT_TYPE="application/vnd.docker.distribution.manifest.v2+json";
+    local REGISTRY_URL="https://git.vdhsn.com/v2/${REPOSITORY}/manifests";
+
+    export MANIFEST=$(curl -u "${DOCKER_USERNAME}:${DOCKER_PASSWORD}" \
+                           -H "Accept: ${CONTENT_TYPE}" "${REGISTRY_URL}/${FROM}");
+
+
+    curl -u "${DOCKER_USERNAME}:${DOCKER_PASSWORD}" \
+         -X PUT -H "Content-Type: ${CONTENT_TYPE}" -d "${MANIFEST}" "${REGISTRY_URL}/${TO}";
+
+    echo "";
+}
+

src/.dockerignore

@@ -1 +1,4 @@
 .idea
+web-client/*
+admin-client/*
+

src/Dockerfile.prod-backend

@@ -1,14 +1,11 @@
-FROM golang:1.19 as prod-builder
+FROM git.vdhsn.com/barretthousen/tools:v1 as prod-builder
-
-RUN go install github.com/bufbuild/buf/cmd/buf@v1.17.0
-RUN go install github.com/kyleconroy/sqlc/cmd/sqlc@latest
-
-COPY . /go/src
 
 WORKDIR /go/src/
 
-RUN /go/bin/sqlc generate -f /go/src/sqlc.yaml
+COPY . /go/src/
-RUN /go/bin/buf mod update /go/src && /go/bin/buf generate
+
+RUN /go/bin/sqlc generate -f /go/src/sqlc.yaml \
+ && /go/bin/buf mod update /go/src && /go/bin/buf generate
 
 ARG service
 

src/Dockerfile.tools

@@ -0,0 +1,4 @@
+FROM golang:1.19 
+
+RUN go install github.com/bufbuild/buf/cmd/buf@v1.17.0
+RUN go install github.com/kyleconroy/sqlc/cmd/sqlc@latest

src/auth/go.mod

@@ -6,7 +6,6 @@ require (
 	git.vdhsn.com/barretthousen/barretthousen/src/lib v1.0.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2
 	go.uber.org/dig v1.16.1
-	golang.org/x/sync v0.2.0
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1
 	google.golang.org/protobuf v1.30.0
 )
@@ -19,6 +18,7 @@ require (
 	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/pressly/goose/v3 v3.11.0 // indirect
 	golang.org/x/net v0.9.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect

src/catalog/go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/pressly/goose/v3 v3.11.0 // indirect
 	golang.org/x/net v0.9.0 // indirect
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
+	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect

src/catalog/go.sum

@@ -180,8 +180,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

src/lib/go.mod

@@ -5,7 +5,7 @@ go 1.19
 require (
 	github.com/jackc/pgx/v4 v4.18.1
 	go.uber.org/automaxprocs v1.5.2
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
+	golang.org/x/sync v0.2.0
 	google.golang.org/grpc v1.55.0
 )
 

src/lib/go.sum

@@ -175,8 +175,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

src/proxy-admin/go.mod

@@ -3,10 +3,11 @@ module git.vdhsn.com/barretthousen/barretthousen/src/proxy-admin
 go 1.19
 
 require (
+	git.vdhsn.com/barretthousen/barretthousen/src/auth v1.0.0
 	git.vdhsn.com/barretthousen/barretthousen/src/lib v1.0.0
 	git.vdhsn.com/barretthousen/barretthousen/src/runner v1.0.0
-	git.vdhsn.com/barretthousen/barretthousen/src/auth v1.0.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2
+	go.uber.org/dig v1.16.1
 	google.golang.org/grpc v1.55.0
 )
 

src/proxy-admin/go.sum

@@ -143,6 +143,7 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
 go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
+go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=

src/proxy-web/go.mod

@@ -27,7 +27,7 @@ require (
 	go.uber.org/automaxprocs v1.5.2 // indirect
 	golang.org/x/crypto v0.8.0 // indirect
 	golang.org/x/net v0.9.0 // indirect
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
+	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
@@ -39,3 +39,5 @@ require (
 replace git.vdhsn.com/barretthousen/barretthousen/src/lib v1.0.0 => ../lib
 
 replace git.vdhsn.com/barretthousen/barretthousen/src/catalog v1.0.0 => ../catalog
+
+replace git.vdhsn.com/barretthousen/barretthousen/src/auth v1.0.0 => ../auth

src/proxy-web/go.sum

@@ -178,8 +178,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

src/runner/internal/domain/catawiki/upcoming.go

@@ -0,0 +1,158 @@
+package catawiki
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+
+	catalog "git.vdhsn.com/barretthousen/barretthousen/src/catalog/api"
+	"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
+	"git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/domain"
+	"golang.org/x/sync/errgroup"
+)
+
+func init() {
+	kernel.TraceLog.Println("Registering AuctionFinder catawiki")
+	domain.RegisterAuctionFinder(
+		CatawikiAuctionFinder("catawiki"),
+	)
+}
+
+type CatawikiAuctionFinder string
+
+func (cw CatawikiAuctionFinder) String() string {
+	return string(cw)
+}
+
+func (cw CatawikiAuctionFinder) Find(ctx context.Context, limit int, results chan<- catalog.Auction) (err error) {
+	defer close(results)
+
+	pageSize := 50
+
+	if limit > 2500 {
+		limit = 2500
+	}
+
+	if limit <= 0 {
+		limit = 2500
+	}
+
+	errg, errgCtx := errgroup.WithContext(ctx)
+	errg.SetLimit(5)
+
+	kernel.TraceLog.Printf("[Catawiki] fetching %d pages or up to %d results in %d batches", limit/pageSize, limit, pageSize)
+	for p := 1; p <= limit/pageSize; p++ {
+		pageIdx := p
+		errg.Go(func() error {
+			auctionR, err := GetUpcomingAuctions(errgCtx, limit, pageIdx)
+			if errors.Is(err, ErrNoResults) {
+				kernel.TraceLog.Println("no more results from catawiki api")
+				return err
+			} else if err != nil {
+				return err
+			}
+
+			for _, r := range auctionR {
+				select {
+				case <-errgCtx.Done():
+					kernel.TraceLog.Println("[Catawiki] group context exited early")
+					return nil
+				default:
+					results <- r
+					kernel.TraceLog.Printf("[Catawiki]: %+v", r)
+				}
+			}
+			return nil
+		})
+	}
+
+	if err = errg.Wait(); err != nil {
+		kernel.ErrorLog.Printf("[Catawiki] could not get results f: %v", err)
+		return
+	}
+
+	return
+}
+
+var ErrNoResults = errors.New("No results found")
+
+func GetUpcomingAuctions(ctx context.Context, limit int, page int) (results []catalog.Auction, err error) {
+	if limit <= 25 {
+		limit = 25
+	} else if limit >= 50 {
+		limit = 50
+	}
+
+	if page < 1 {
+		page = 1
+	}
+
+	url := fmt.Sprintf("https://www.catawiki.com/buyer/api/v1/auctions?locale=en&=published_at_desc&per_page=%d&page=%d", limit, page)
+
+	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Accept-Language", "en-US,en;q=0.5")
+	req.Header.Add("User-Agent", "barretthousen.com Auction Search Engine")
+
+	kernel.TraceLog.Printf("[Catawiki] Loading page %d of %d results", page, limit)
+	var res *http.Response
+	if res, err = http.DefaultClient.Do(req); err != nil {
+		return
+	}
+
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		err = fmt.Errorf("bad response code: %d", res.StatusCode)
+		return
+	}
+
+	type Payload struct {
+		Auctions []struct {
+			ID       int    `json:"id"`
+			StartAt  string `json:"start_at"`
+			CloseAt  string `json:"close_at"`
+			LotCount int    `json:"lot_count"`
+			Title    string `json:"title"`
+			URL      string `json:"url"`
+			Status   string `json:"status"`
+			Sellers  []struct {
+				ID   int    `json:"id"`
+				Name string `json:"name"`
+			} `json:"sellers"`
+		} `json:"auctions"`
+	}
+
+	var p Payload
+	if err = json.NewDecoder(res.Body).Decode(&p); err != nil {
+		return
+	}
+
+	if len(p.Auctions) <= 0 {
+		err = ErrNoResults
+		return
+	}
+
+	results = make([]catalog.Auction, len(p.Auctions))
+	for idx, auction := range p.Auctions {
+		results[idx] = catalog.Auction{
+			Title:          auction.Title,
+			Description:    "",
+			SourceSiteURL:  "https://www.catawiki.com",
+			SourceSiteName: "CataWiki",
+			SourceURL:      auction.URL,
+			Country:        "",
+			Province:       "",
+			ItemCount:      auction.LotCount,
+		}
+
+		results[idx].Start, _ = time.Parse(time.RFC3339, auction.StartAt)
+		results[idx].End, _ = time.Parse(time.RFC3339, auction.CloseAt)
+	}
+
+	return
+}

src/runner/internal/domain/catawiki/upcoming_test.go

@@ -0,0 +1,44 @@
+package catawiki
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	catalog "git.vdhsn.com/barretthousen/barretthousen/src/catalog/api"
+)
+
+func TestGetUpcomingAuctions(t *testing.T) {
+	type args struct {
+		limit int
+		page  int
+	}
+	tests := []struct {
+		name        string
+		args        args
+		wantResults []catalog.Auction
+		wantErr     bool
+	}{
+		{
+			name: "1 Page, 25 Results",
+			args: args{
+				limit: 25,
+				page:  1,
+			},
+			wantResults: make([]catalog.Auction, 25),
+			wantErr:     false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotResults, err := GetUpcomingAuctions(context.TODO(), tt.args.limit, tt.args.page)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GetUpcomingAuctions() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotResults, tt.wantResults) {
+				t.Errorf("GetUpcomingAuctions() = %v, want %v", gotResults, tt.wantResults)
+			}
+		})
+	}
+}

src/runner/internal/domain/logic.go

@@ -2,6 +2,7 @@ package domain
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -60,17 +61,32 @@ type (
 func (domain Domain) StartSync(ctx context.Context, in FindNewUpcomingInput) (out FindNewUpcomingOutput, err error) {
 	kernel.TraceLog.Printf("%+v", in)
 
-	finder := targetsImpls["liveauctioneers"]
+	if in.TargetSite == "All" || in.TargetSite == "" {
+		for k, v := range targetsImpls {
 
+			if out.Job, err = domain.Storage.CreateScrapeJob(ctx, k); err != nil {
+				err = fmt.Errorf("could not create new scrape job record: %w", err)
+				continue
+			}
+
+			kernel.InfoLog.Printf("Scrape Job %d starting", out.Job.ID)
+
+			go domain.executeScrapeJob(v, out.Job.ID)
+		}
+	} else if finder, ok := targetsImpls[in.TargetSite]; ok {
 		if out.Job, err = domain.Storage.CreateScrapeJob(ctx, in.TargetSite); err != nil {
 			err = fmt.Errorf("could not create new scrape job record: %w", err)
 			return
 		}
 
 		kernel.InfoLog.Printf("Scrape Job %d starting", out.Job.ID)
-
-	// TODO: make everything after this line async and run after return
 		go domain.executeScrapeJob(finder, out.Job.ID)
+	} else {
+		kernel.TraceLog.Println("could not find target")
+		err = errors.New("No scrape job found by name")
+		return
+	}
+
 	return
 }
 
@@ -116,13 +132,13 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
 	errs := &strings.Builder{}
 	for auction := range found {
 		total++
-		if !auction.Start.After(time.Now()) {
+		if !auction.Start.After(time.Now().Add(-24 * time.Hour)) {
 			continue
 		}
 
 		ace, err := domain.CatalogService.UpdateUpcomingAuction(ctx, auction)
 		if err != nil {
-			kernel.TraceLog.Printf("could not import upcoming auction: %s", err.Error())
+			kernel.ErrorLog.Printf("[%s] could not import upcoming auction: %s", finder.String(), err.Error())
 			fmt.Fprintf(errs, "{ \"AuctionFingerprint\": \"%s\", \"error\": \"%s\" }\n", ace.Fingerprint, err.Error())
 			continue
 		}
@@ -132,9 +148,10 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
 		}
 	}
 
+	kernel.TraceLog.Printf("[%s] waiting for results...", finder.String())
 	if err := errGroup.Wait(); err != nil {
 		err = fmt.Errorf("an issue occurred while finding upcoming items iteration: %w", err)
-		fmt.Fprintf(errs, "{ \"error\": \"%s\" }", err.Error())
+		fmt.Fprintf(errs, "{\"error\": \"%s\" }", err.Error())
 	}
 
 	var completedJob ScrapeJob
@@ -143,9 +160,9 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
 		AuctionCount: count,
 		Errors:       errs.String(),
 	}); err != nil {
-		kernel.ErrorLog.Printf("Could not complete scrape job, failing: %v", err)
+		kernel.ErrorLog.Printf("[%s] Could not complete scrape job, failing: %v", finder.String(), err)
 	}
 
-	kernel.InfoLog.Printf("Scrape Job %d completed in %v. Successfully imported %d/%d", jobID, completedJob.Completed.Sub(completedJob.Started), count, total)
+	kernel.InfoLog.Printf("[%s] Scrape Job %d completed in %v. Successfully imported %d/%d", finder.String(), jobID, completedJob.Completed.Sub(completedJob.Started), count, total)
 	return
 }

src/runner/main.go

@@ -16,6 +16,7 @@ import (
 	"go.uber.org/dig"
 	"google.golang.org/grpc"
 
+	_ "git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/domain/catawiki"
 	_ "git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/domain/liveauctioneers"
 )
 

src/web-client/package.json

@@ -39,10 +39,10 @@
 	},
 	"type": "module",
 	"dependencies": {
-		"analytics": "^0.8.9",
-		"analytics-plugin-do-not-track": "^0.1.5",
 		"@analytics/google-analytics": "^1.0.7",
 		"@segment/snippet": "^4.16.2",
+		"analytics": "^0.8.9",
+		"analytics-plugin-do-not-track": "^0.1.5",
 		"luxon": "^3.3.0"
 	}
 }

src/web-client/src/routes/+layout.svelte

@@ -4,16 +4,16 @@
 
 	import Analytics from 'analytics';
 	// import doNotTrack from 'analytics-plugin-do-not-track';
-	import googleAnalytics from '@analytics/google-analytics';
+	// import googleAnalytics from '@analytics/google-analytics';
 
 	const analytics = Analytics({
 		app: 'Barretthousen - Web Client',
 		version: '1.0',
 		debug: process.env.NODE_ENV !== 'production',
 		plugins: [
-			googleAnalytics({
+			// googleAnalytics({
-				measurementIds: ['UA-143763293-1']
+			// 	measurementIds: ['UA-143763293-1']
-			})
+			// })
 			// doNotTrack()
 		]
 	});