Compare commits

..

1 Commits

Author SHA1 Message Date
Adam Veldhousen cff1ee17f1
attempt 1
ci.vdhsn.com/push Build is failing Details
2023-07-10 01:47:53 -05:00
42 changed files with 173 additions and 754 deletions

View File

@ -1,296 +1,41 @@
kind: pipeline
type: kubernetes
name: Build & Push Images
name: Build Images
trigger:
branch:
- trunk
event:
- push
steps:
- name: Build & Publish Catalog
image: git.vdhsn.com/barretthousen/drone-kaniko:v1.0.1
volumes:
- name: cache
path: /kaniko
image: plugins/kaniko
settings:
verbosity: debug
dockerfile: "./src/Dockerfile.prod-backend"
context: "./src"
target: production
registry: git.vdhsn.com
repo: git.vdhsn.com/barretthousen/service-catalog
enable_cache: true
cache_repo: git.vdhsn.com/barretthousen/ci-cache
skip_unused_stages: true
build_args:
- "service=catalog"
tags:
- ${DRONE_COMMIT_SHA}
- beta
custom_labels:
- com.barretthousen.service=catalog
- com.barretthousen.version=${DRONE_COMMIT_SHA}
- com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
- com.barretthousen.build-date=${DRONE_BUILD_STARTED}
- com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
username:
from_secret: DOCKER_USERNAME
password:
from_secret: DOCKER_PASSWORD
args:
- "service=catalog"
no-push: true
tags:
- ${DRONE_COMMIT_SHA}
- name: Build & Publish Runner
image: plugins/kaniko
settings:
verbosity: debug
dockerfile: "./src/Dockerfile.prod-backend"
context: "./src"
target: production
registry: git.vdhsn.com
repo: git.vdhsn.com/barretthousen/service-runner
enable_cache: true
cache_repo: git.vdhsn.com/barretthousen/ci-cache
snapshot_mode: redo
skip_unused_stages: true
build_args:
username:
from_secret: DOCKER_USERNAME
password:
from_secret: DOCKER_PASSWORD
args:
- "service=runner"
no-push: true
tags:
- ${DRONE_COMMIT_SHA}
custom_labels:
- com.barretthousen.service=runner
- com.barretthousen.version=${DRONE_COMMIT_SHA}
- com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
- com.barretthousen.build-date=${DRONE_BUILD_STARTED}
- com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
username:
from_secret: DOCKER_USERNAME
password:
from_secret: DOCKER_PASSWORD
- name: Build & Publish Auth
image: plugins/kaniko
settings:
verbosity: debug
dockerfile: "./src/Dockerfile.prod-backend"
context: "./src"
target: production
registry: git.vdhsn.com
repo: git.vdhsn.com/barretthousen/service-auth
enable_cache: true
cache_repo: git.vdhsn.com/barretthousen/ci-cache
snapshot_mode: redo
skip_unused_stages: true
build_args:
- "service=auth"
tags:
- ${DRONE_COMMIT_SHA}
- beta
custom_labels:
- com.barretthousen.service=auth
- com.barretthousen.version=${DRONE_COMMIT_SHA}
- com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
- com.barretthousen.build-date=${DRONE_BUILD_STARTED}
- com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
username:
from_secret: DOCKER_USERNAME
password:
from_secret: DOCKER_PASSWORD
- name: Build & Publish Proxy Admin
image: plugins/kaniko
settings:
verbosity: debug
dockerfile: "./src/Dockerfile.prod-backend"
context: "./src"
target: production
registry: git.vdhsn.com
repo: git.vdhsn.com/barretthousen/service-proxy-admin
enable_cache: true
cache_repo: git.vdhsn.com/barretthousen/ci-cache
snapshot_mode: redo
skip_unused_stages: true
build_args:
- "service=proxy-admin"
tags:
- ${DRONE_COMMIT_SHA}
- beta
custom_labels:
- com.barretthousen.service=proxy-admin
- com.barretthousen.version=${DRONE_COMMIT_SHA}
- com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
- com.barretthousen.build-date=${DRONE_BUILD_STARTED}
- com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
username:
from_secret: DOCKER_USERNAME
password:
from_secret: DOCKER_PASSWORD
depends_on:
- Build & Publish Catalog
- Build & Publish Runner
- Build & Publish Auth
- name: Build & Publish Proxy Web
image: plugins/kaniko
settings:
verbosity: debug
dockerfile: "./src/Dockerfile.prod-backend"
context: "./src"
target: production
registry: git.vdhsn.com
repo: git.vdhsn.com/barretthousen/service-proxy-web
enable_cache: true
cache_repo: git.vdhsn.com/barretthousen/ci-cache
snapshot_mode: redo
skip_unused_stages: true
build_args:
- "service=proxy-web"
tags:
- ${DRONE_COMMIT_SHA}
- beta
custom_labels:
- com.barretthousen.service=proxy-web
- com.barretthousen.version=${DRONE_COMMIT_SHA}
- com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
- com.barretthousen.build-date=${DRONE_BUILD_STARTED}
- com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
username:
from_secret: DOCKER_USERNAME
password:
from_secret: DOCKER_PASSWORD
depends_on:
- Build & Publish Catalog
- Build & Publish Runner
- Build & Publish Auth
- name: Build & Publish Web Client
image: plugins/kaniko
settings:
verbosity: debug
dockerfile: "./src/Dockerfile.frontend"
context: "./src/web-client"
target: production
registry: git.vdhsn.com
repo: git.vdhsn.com/barretthousen/service-proxy-web
enable_cache: true
cache_repo: git.vdhsn.com/barretthousen/ci-cache
snapshot_mode: redo
skip_unused_stages: true
build_args:
- "service=proxy-web"
tags:
- ${DRONE_COMMIT_SHA}
- beta
custom_labels:
- com.barretthousen.service=proxy-web
- com.barretthousen.version=${DRONE_COMMIT_SHA}
- com.barretthousen.git-ref=${DRONE_COMMIT_SHA}
- com.barretthousen.build-date=${DRONE_BUILD_STARTED}
- com.barrethousen.builder=${DRONE_COMMIT_AUTHOR}
username:
from_secret: DOCKER_USERNAME
password:
from_secret: DOCKER_PASSWORD
depends_on:
- Build & Publish Catalog
- Build & Publish Runner
- Build & Publish Auth
- Build & Publish Proxy Admin
- Build & Publish Proxy Web
- name: Build & Publish Admin Client
image: plugins/kaniko
settings:
verbosity: debug
dockerfile: "./src/Dockerfile.frontend"
context: "./src/admin-client"
target: production
registry: git.vdhsn.com
repo: git.vdhsn.com/barretthousen/client-admin-client
enable_cache: true
cache_repo: git.vdhsn.com/barretthousen/ci-cache
snapshot_mode: redo
skip_unused_stages: true
build_args:
- "service=admin-client"
tags:
- ${DRONE_COMMIT_SHA}
- beta
username:
from_secret: DOCKER_USERNAME
password:
from_secret: DOCKER_PASSWORD
depends_on:
- Build & Publish Catalog
- Build & Publish Runner
- Build & Publish Auth
- Build & Publish Proxy Admin
- Build & Publish Proxy Web
- name: Build Success Notify
image: appleboy/drone-discord
environment:
DISCORD_WEBHOOK_ID:
from_secret: DISCORD_WEBHOOK_ID
DISCORD_WEBHOOK_TOKEN:
from_secret: DISCORD_WEBHOOK_TOKEN
settings:
username: Drone CI
message: Barretthousen Image Builds Complete
webhook_id:
from_secret: DISCORD_WEBHOOK_ID
webhook_token:
from_secret: DISCORD_WEBHOOK_TOKEN
depends_on:
- Build & Publish Catalog
- Build & Publish Runner
- Build & Publish Auth
- Build & Publish Proxy Admin
- Build & Publish Proxy Web
- Build & Publish Web Client
- Build & Publish Admin Client
---
kind: pipeline
type: kubernetes
name: Promote Beta -> Prod
trigger:
branch:
- trunk
event:
- promote
steps:
- name: Promote Services
image: alpine
environment:
REGISTRY_NAME: "https://git.vdhsn.com"
CONTENT_TYPE: "application/vnd.docker.distribution.manifest.v2+json"
DOCKER_USERNAME:
from_secret: DOCKER_USERNAME
DOCKER_PASSWORD:
from_secret: DOCKER_PASSWORD
commands:
- ./hack/promote.sh barretthousen/catalog beta prod
- name: Deploy Success
image: appleboy/drone-discord
environment:
DISCORD_WEBHOOK_ID:
from_secret: DISCORD_WEBHOOK_ID
DISCORD_WEBHOOK_TOKEN:
from_secret: DISCORD_WEBHOOK_TOKEN
settings:
username: Drone CI
message: Barretthousen Prod deployment success
webhook_id:
from_secret: DISCORD_WEBHOOK_ID
webhook_token:
from_secret: DISCORD_WEBHOOK_TOKEN
depends_on:
- Promote Services
volumes:
- name: cache
temp:
medium: memory

View File

@ -52,19 +52,23 @@ VERSION = $(shell git rev-parse --verify --short HEAD)
GIT_REF = $(shell git rev-parse --verify HEAD)
BUILD_DATE := $(shell date +%Y-%m-%d-%T)
.PHONY: build-tools-image
build-tools-image:
.PHONY: build-swagger-image
build-swagger-image:
@docker build \
--label 'com.barretthousen.service=swagger' \
--label 'com.barretthousen.version=$(VERSION)' \
--label 'com.barretthousen.git-ref=$(GIT_REF)' \
--label 'com.barretthousen.build-date=$(BUILD_DATE)' \
--label 'com.barrethousen.builder=$(BUILD_INITIATOR)' \
-t git.vdhsn.com/barretthousen/tools):$(VERSION) \
-t git.vdhsn.com/barretthousen/tools):v1 \
-f ./src/Dockerfile.tools ./src
-t barretthousen/service-swagger:$(VERSION) \
-t git.vdhsn.com/barretthousen/service-swagger:$(VERSION) \
-f ./src/Dockerfile.swagger ./src
@docker push git.vdhsn.com/barretthousen/service-swagger:$(VERSION)
@[ ! -z $(ENV) ] && docker tag git.vdhsn.com/barretthousen/service-swagger:$(VERSION) git.vdhsn.com/barretthousen/service-swagger:$(ENV) || true
@[ ! -z $(ENV) ] && docker push git.vdhsn.com/barretthousen/service-swagger:$(ENV) || true
@docker push git.vdhsn.com/barretthousen/tools:$(VERSION)
@docker push git.vdhsn.com/barretthousen/tools:v1
.PHONY: build-client-image
build-client-image:
@ -80,7 +84,7 @@ build-client-image:
-f ./src/Dockerfile.frontend ./src/$(SERVICE)
@docker push git.vdhsn.com/barretthousen/client-$(SERVICE):$(VERSION)
@[ ! -z $(ENV) ] && docker tag git.vdhsn.com/barretthousen/client-$(SERVICE):$(VERSION) git.vdhsn.com/barretthousen/client-$(SERVICE):$(ENV) || true
@[ ! -z $(ENV) ] && docker tag git.vdhsn.com/barretthousen/client-$(SERVICE):$(VERSION) git.vdhsn.com/barretthousen/client-$(SERVICE):beta || true
@[ ! -z $(ENV) ] && docker push git.vdhsn.com/barretthousen/client-$(SERVICE):$(ENV) || true
.PHONY: build-backend-image
@ -98,8 +102,8 @@ build-backend-image:
@docker push git.vdhsn.com/barretthousen/service-$(SERVICE):$(VERSION)
@[ ! -z $(ENV) ] && docker tag git.vdhsn.com/barretthousen/service-$(SERVICE):$(VERSION) git.vdhsn.com/barretthousen/service-$(SERVICE):$(ENV) || true
@[ ! -z $(ENV) ] && docker push git.vdhsn.com/barretthousen/service-$(SERVICE):$(ENV) || true
@[ ! -z $(ENV) ] && docker tag git.vdhsn.com/barretthousen/service-$(SERVICE):$(VERSION) git.vdhsn.com/barretthousen/service-$(SERVICE):beta || true
@[ ! -z $(ENV) ] && docker push git.vdhsn.com/barretthousen/service-$(SERVICE):beta || true
# TODO: investigate tagging without needing to pull? https://dille.name/blog/2018/09/20/how-to-tag-docker-images-without-pulling-them/
.PHONY: promote-client-prod
@ -138,4 +142,3 @@ age_identity=$(shell sops -d ./env/master.json)
@echo "# public key: $(shell echo '$(age_identity)' | jq -r '.public_key')" >> $@
@echo "$(shell echo '$(age_identity)' | jq -r '.private_key')" >> $@
@echo "$@ created!"
@echo "export SOPS_AGE_KEY_FILE=$(PWD)/env/.age.txt"

View File

@ -1,6 +1,5 @@
# [Barretthousen](https://barretthousen.com)
[![Build Status](https://ci.vdhsn.com/api/badges/Barretthousen/barretthousen/status.svg?ref=refs/heads/trunk)](https://ci.vdhsn.com/Barretthousen/barretthousen)
Search and get alerts for items across the most popular auction sites.
Built with microservice architecture and various fancy tools for learning purposes

View File

@ -7,7 +7,6 @@ print("""
load('ext://helm_resource', 'helm_resource', 'helm_repo')
load('ext://deployment', 'deployment_create')
load('ext://restart_process', 'docker_build_with_restart')
load('ext://uibutton', 'cmd_button', 'location', 'text_input')
helm_repo('bitnami', 'https://charts.bitnami.com/bitnami', labels=["9-repos"])
helm_repo('traefik', 'https://traefik.github.io/charts', labels=["9-repos"])
@ -132,6 +131,9 @@ def bh_backend_service(service="", port_forwards=[], migrateDB=False, devMode=Tr
port_forwards=port_forwards,
labels=labels,
resource_deps=deps,
links=[
link(url='http://{}.localhost:8000'.format(service), name='Swagger'),
]
)
@ -144,6 +146,21 @@ k8s_resource(
labels='2-services'
)
docker_build(
ref="barretthousen/service-swagger",
dockerfile="./src/Dockerfile.swagger",
context="./src",
only = [
'./auth/api/grpc/auth.swagger.json',
'./catalog/api/grpc/catalog.swagger.json',
'./runner/api/grpc/runner.swagger.json',
],
live_update=[
sync('./src/auth/api/grpc/auth.swagger.json', '/var/doc'),
sync('./src/catalog/api/grpc/catalog.swagger.json', '/var/doc'),
sync('./src/runner/api/grpc/runner.swagger.json', '/var/doc')
]
)
bh_backend_service(service="auth", migrateDB=True, port_forwards=[
port_forward(2345, name='Delve port')
@ -167,34 +184,3 @@ bh_backend_service(service="proxy-web", port_forwards=[
bh_client(service='web', deps=["proxy-web-local"])
bh_client(service='admin', deps=["proxy-admin-local"])
createAdminCmd = [
'./hack/create_user.sh', 'admin@barretthousen.com', 'admin', 'ADMINISTRATOR'
]
loginAdminCmd = [
'curl', '-vvvv', '-X POST', '-H "Content-Type: application/json"',
'-d "{\"email\":\"admin@barretthousen.com\", \"password\":\"admin\"}"',
'http://bh.localhost:8000/api/v1/user'
]
syncCatalogsCmd = [
'curl', '-vvvv', '-X', 'PUT',
'-H', 'Content-Type: application/json',
'-H', 'bh-session-id: 2',
'-d', '{\"targetSite\":\"All\"}',
'http://admin.localhost:8000/api/v1/sync'
]
local(createAdminCmd, quiet=False, echo_off=True)
local(loginAdminCmd, quiet=False, echo_off=True)
local(syncCatalogsCmd, quiet=False, echo_off=True)
cmd_button(
name='Sync Catalogs',
argv=syncCatalogsCmd,
text='Sync Catalogs',
location=location.NAV,
icon_name='sync')

View File

@ -61,7 +61,6 @@ helm install --upgrade bh-db bitnami/postgresql -n 'barretthousen-beta' \
export ENV=beta; # if not specified, will only build and push for the git commit
make build-backend-image SERVICE=catalog
make build-backend-image SERVICE=runner
make build-backend-image SERVICE=auth
make build-backend-image SERVICE=proxy-admin
make build-backend-image SERVICE=proxy-web
@ -79,9 +78,6 @@ kubectl rollout status -n barretthousen-beta deployment runner-beta -w
kubectl rollout restart -n barretthousen-beta deployment catalog-beta
kubectl rollout status -n barretthousen-beta deployment catalog-beta -w
kubectl rollout restart -n barretthousen-beta deployment auth-beta
kubectl rollout status -n barretthousen-beta deployment auth-beta -w
kubectl rollout restart -n barretthousen-beta deployment proxy-admin-beta
kubectl rollout status -n barretthousen-beta deployment proxy-admin-beta -w
@ -99,7 +95,6 @@ kubectl rollout status -n barretthousen-beta deployment admin-client-beta -w
```sh
make promote-backend-prod SERVICE=catalog
make promote-backend-prod SERVICE=runner
make promote-backend-prod SERVICE=auth
make promote-backend-prod SERVICE=proxy-admin
make promote-backend-prod SERVICE=proxy-web

View File

@ -46,7 +46,6 @@ spec:
ports:
- port: 5001
targetPort: 5001
name: grpc
---
apiVersion: v1
kind: Secret

View File

@ -46,7 +46,6 @@ spec:
ports:
- port: 5001
targetPort: 5001
name: grpc
---
apiVersion: v1
kind: Secret

View File

@ -46,7 +46,7 @@ spec:
ports:
- port: 5001
targetPort: 5001
name: grpc
---
apiVersion: v1
kind: Secret

View File

@ -1,22 +0,0 @@
apiVersion: v1
kind: Secret
metadata:
name: auth-config
stringData:
config.yaml: |
log_level: 2
port: 5001
db_service:
scheme: postgres
port: 5432
host: bh-db
name: bh
user: auth-service
password: auth-service
db_migrate:
scheme: postgres
port: 5432
host: bh-db
name: bh
user: postgres
password: bh-admin-beta

View File

@ -7,11 +7,10 @@ namespace: barretthousen-beta
patchesStrategicMerge:
- sync-cronjob.yaml
- auth-secret.yaml
- catalog-secret.yaml
- runner-secret.yaml
- proxy-admin-secret.yaml
- proxy-web-secret.yaml
- runner-secret.yaml
patches:
- target:
@ -30,11 +29,11 @@ patches:
value: beta.barretthousen.com
- target:
kind: Deployment
name: auth
name: swagger
patch: |-
- op: replace
path: /spec/template/spec/containers/0/image
value: git.vdhsn.com/barretthousen/service-auth:beta
value: git.vdhsn.com/barretthousen/service-swagger:beta
- target:
kind: Deployment
name: catalog

View File

@ -8,5 +8,4 @@ stringData:
port: 80
access_control_allow_origin: "beta.barretthousen.com"
endpoints:
auth: auth-beta:5001
catalog: catalog-beta:5001

View File

@ -6,6 +6,23 @@ spec:
template:
spec:
containers:
- name: swagger
image: barretthousen/service-swagger:latest
env:
- name: SWAGGER_JSON
value: /var/doc/auth.swagger.json
ports:
- containerPort: 8080
- name: auth
ports:
- containerPort: 2345
---
apiVersion: v1
kind: Service
metadata:
name: auth
spec:
ports:
- port: 81
targetPort: 8080
name: swagger

View File

@ -6,6 +6,23 @@ spec:
template:
spec:
containers:
- name: swagger
image: barretthousen/service-swagger:latest
env:
- name: SWAGGER_JSON
value: /var/doc/catalog.swagger.json
ports:
- containerPort: 8080
- name: catalog
ports:
- containerPort: 2345
---
apiVersion: v1
kind: Service
metadata:
name: catalog
spec:
ports:
- port: 81
targetPort: 8080
name: swagger

View File

@ -6,6 +6,23 @@ spec:
template:
spec:
containers:
- name: swagger
image: barretthousen/service-swagger:latest
env:
- name: SWAGGER_JSON
value: /var/doc/runner.swagger.json
ports:
- containerPort: 8080
- name: runner
ports:
- containerPort: 2345
---
apiVersion: v1
kind: Service
metadata:
name: runner
spec:
ports:
- port: 81
targetPort: 8080
name: swagger

View File

@ -16,6 +16,7 @@ patchesStrategicMerge:
- image-pull-secret.yaml
- proxy-admin-secret.yaml
- proxy-web-secret.yaml
- swagger.yaml
patches:
- target:

30
env/local/swagger.yaml vendored Normal file
View File

@ -0,0 +1,30 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: web
spec:
rules:
- host: bh.localhost
http:
paths:
- pathType: Prefix
path: "/swagger/auth"
backend:
service:
name: auth
port:
number: 81
- pathType: Prefix
path: "/swagger/catalog"
backend:
service:
name: catalog
port:
number: 81
- pathType: Prefix
path: "/swagger/runner"
backend:
service:
name: runner
port:
number: 81

View File

@ -8,13 +8,6 @@ patchesStrategicMerge:
- sync-cronjob.yaml
patches:
- target:
kind: Deployment
name: auth
patch: |-
- op: replace
path: /spec/template/spec/containers/0/image
value: git.vdhsn.com/barretthousen/service-auth:prod
- target:
kind: Deployment
name: catalog

View File

@ -14,6 +14,6 @@ spec:
[
"curl",
"http://proxy-admin/api/v1/sync",
'-d=''{"target":"All"}''',
'-d=''{"target":"liveauctioneers"}''',
"-vvvv",
]

View File

@ -1,10 +1,10 @@
go 1.19
use (
./src/lib
./src/auth
./src/catalog
./src/lib
./src/runner
./src/proxy-admin
./src/proxy-web
./src/runner
)

View File

@ -1,64 +0,0 @@
#!/bin/bash
build(){
export ENV=${1:-beta}; # if not specified, will only build and push for the git commit
make build-backend-image SERVICE=auth &
make build-backend-image SERVICE=catalog &
make build-backend-image SERVICE=runner &
make build-backend-image SERVICE=proxy-admin &
make build-backend-image SERVICE=proxy-web &
time wait;
# for prod drop `beta prefix
make build-client-image SERVICE=web-client &
make build-client-image SERVICE=admin-client &
time wait;
kubectl rollout restart -n barretthousen-beta deployment runner-beta &
kubectl rollout restart -n barretthousen-beta deployment catalog-beta &
kubectl rollout restart -n barretthousen-beta deployment auth-beta &
kubectl rollout restart -n barretthousen-beta deployment proxy-admin-beta &
kubectl rollout restart -n barretthousen-beta deployment proxy-web-beta &
kubectl rollout restart -n barretthousen-beta deployment web-client-beta &
kubectl rollout restart -n barretthousen-beta deployment admin-client-beta &
kubectl rollout status -n barretthousen-beta deployment runner-beta -w
kubectl rollout status -n barretthousen-beta deployment catalog-beta -w
kubectl rollout status -n barretthousen-beta deployment auth-beta -w
kubectl rollout status -n barretthousen-beta deployment proxy-admin-beta -w
kubectl rollout status -n barretthousen-beta deployment proxy-web-beta -w
kubectl rollout status -n barretthousen-beta deployment admin-client-beta -w
kubectl rollout status -n barretthousen-beta deployment web-client-beta -w
}
promote() {
make promote-backend-prod SERVICE=catalog &
make promote-backend-prod SERVICE=runner &
make promote-backend-prod SERVICE=auth &
make promote-backend-prod SERVICE=proxy-admin &
make promote-backend-prod SERVICE=proxy-web &
make promote-client-prod SERVICE=web-client &
make promote-client-prod SERVICE=admin-client &
time wait;
kubectl rollout restart -n barretthousen deployment runner &
kubectl rollout restart -n barretthousen deployment catalog &
kubectl rollout restart -n barretthousen deployment auth &
kubectl rollout restart -n barretthousen deployment proxy-admin &
kubectl rollout restart -n barretthousen deployment proxy-web &
kubectl rollout restart -n barretthousen deployment web-client &
kubectl rollout restart -n barretthousen deployment admin-client &
kubectl rollout status -n barretthousen deployment runner -w
kubectl rollout status -n barretthousen deployment catalog -w
kubectl rollout status -n barretthousen deployment auth -w
kubectl rollout status -n barretthousen deployment proxy-admin -w
kubectl rollout status -n barretthousen deployment proxy-web -w
kubectl rollout status -n barretthousen deployment admin-client -w
kubectl rollout status -n barretthousen deployment web-client -w
}
build $@

View File

@ -1,26 +0,0 @@
#!/bin/bash
main(){
local EMAIL=${1};
if [ -z "${EMAIL}" ]; then
echo "first argument must be an EMAIL address"
exit 1;
fi
local PASSWORD=${2};
if [ -z "${PASSWORD}" ]; then
echo "second argument must be a PASSWORD"
exit 1;
fi
local ROLE=${3:-USER};
local HOST=${4:-"http://bh.localhost:8000"};
curl -X PUT -H 'Content-Type: application/json' \
-d "{ \"email\":\"${EMAIL}\", \"password\":\"${PASSWORD}\", \"role\":\"${ROLE}\" }" \
"${HOST}/api/v1/user";
}
main $@;

View File

@ -1,19 +0,0 @@
#!/bin/bash
docker run --rm \
-e DRONE_COMMIT_REF=refs/heads/trunk \
-e DRONE_REPO_BRANCH=trunk \
-e PLUGIN_REGISTRY=git.vdhsn.com \
-e PLUGIN_REPO=git.vdhsn.com/barretthousen/service-catalog \
-e PLUGIN_BUILD_ARGS="service=catalog" \
-e PLUGIN_TAGS=$(git rev-parse --verify --short HEAD) \
-e PLUGIN_CONTEXT="./src" \
-e PLUGIN_DOCKERFILE="./src/Dockerfile.prod-backend" \
-e PLUGIN_USERNAME=${DOCKER_USERNAME} \
-e PLUGIN_PASSWORD=${DOCKER_PASSWORD} \
-e PLUGIN_ENABLE_CACHE=true \
-e PLUGIN_CACHE_REPO='git.vdhsn.com/barretthousen/ci-cache' \
-e PLUGIN_SKIP_UNUSED_STAGES=true \
-v $(pwd):/drone \
-w /drone \
plugins/kaniko:linux-amd64

View File

@ -1,26 +0,0 @@
#!/bin/sh
main(){
local REPOSITORY=${1};
if [ -z "${REPOSITORY}" ]; then
echo "First argument must be container repository";
exit 1;
fi
local FROM=${2:-beta};
local TO=${3:-prod};
local CONTENT_TYPE="application/vnd.docker.distribution.manifest.v2+json";
local REGISTRY_URL="https://git.vdhsn.com/v2/${REPOSITORY}/manifests";
export MANIFEST=$(curl -u "${DOCKER_USERNAME}:${DOCKER_PASSWORD}" \
-H "Accept: ${CONTENT_TYPE}" "${REGISTRY_URL}/${FROM}");
curl -u "${DOCKER_USERNAME}:${DOCKER_PASSWORD}" \
-X PUT -H "Content-Type: ${CONTENT_TYPE}" -d "${MANIFEST}" "${REGISTRY_URL}/${TO}";
echo "";
}

View File

@ -1,4 +1 @@
.idea
web-client/*
admin-client/*

View File

@ -1,11 +1,14 @@
FROM git.vdhsn.com/barretthousen/tools:v1 as prod-builder
FROM golang:1.19 as prod-builder
RUN go install github.com/bufbuild/buf/cmd/buf@v1.17.0
RUN go install github.com/kyleconroy/sqlc/cmd/sqlc@latest
COPY . /go/src
WORKDIR /go/src/
COPY . /go/src/
RUN /go/bin/sqlc generate -f /go/src/sqlc.yaml \
&& /go/bin/buf mod update /go/src && /go/bin/buf generate
RUN /go/bin/sqlc generate -f /go/src/sqlc.yaml
RUN /go/bin/buf mod update /go/src && /go/bin/buf generate
ARG service

5
src/Dockerfile.swagger Normal file
View File

@ -0,0 +1,5 @@
FROM swaggerapi/swagger-ui
COPY ./auth/api/grpc/auth.swagger.json /var/doc/
COPY ./catalog/api/grpc/catalog.swagger.json /var/doc/
COPY ./runner/api/grpc/runner.swagger.json /var/doc

View File

@ -1,4 +0,0 @@
FROM golang:1.19
RUN go install github.com/bufbuild/buf/cmd/buf@v1.17.0
RUN go install github.com/kyleconroy/sqlc/cmd/sqlc@latest

View File

@ -6,6 +6,7 @@ require (
git.vdhsn.com/barretthousen/barretthousen/src/lib v1.0.0
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2
go.uber.org/dig v1.16.1
golang.org/x/sync v0.2.0
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1
google.golang.org/protobuf v1.30.0
)
@ -18,7 +19,6 @@ require (
github.com/joho/godotenv v1.5.1 // indirect
github.com/pressly/goose/v3 v3.11.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.8.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect

View File

@ -18,7 +18,7 @@ require (
github.com/joho/godotenv v1.5.1 // indirect
github.com/pressly/goose/v3 v3.11.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
golang.org/x/sys v0.8.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect

View File

@ -180,8 +180,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

View File

@ -5,7 +5,7 @@ go 1.19
require (
github.com/jackc/pgx/v4 v4.18.1
go.uber.org/automaxprocs v1.5.2
golang.org/x/sync v0.2.0
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
google.golang.org/grpc v1.55.0
)

View File

@ -175,8 +175,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

View File

@ -3,11 +3,10 @@ module git.vdhsn.com/barretthousen/barretthousen/src/proxy-admin
go 1.19
require (
git.vdhsn.com/barretthousen/barretthousen/src/auth v1.0.0
git.vdhsn.com/barretthousen/barretthousen/src/lib v1.0.0
git.vdhsn.com/barretthousen/barretthousen/src/runner v1.0.0
git.vdhsn.com/barretthousen/barretthousen/src/auth v1.0.0
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2
go.uber.org/dig v1.16.1
google.golang.org/grpc v1.55.0
)

View File

@ -143,7 +143,6 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=

View File

@ -27,7 +27,7 @@ require (
go.uber.org/automaxprocs v1.5.2 // indirect
golang.org/x/crypto v0.8.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
@ -39,5 +39,3 @@ require (
replace git.vdhsn.com/barretthousen/barretthousen/src/lib v1.0.0 => ../lib
replace git.vdhsn.com/barretthousen/barretthousen/src/catalog v1.0.0 => ../catalog
replace git.vdhsn.com/barretthousen/barretthousen/src/auth v1.0.0 => ../auth

View File

@ -178,8 +178,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

View File

@ -1,158 +0,0 @@
package catawiki
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"time"
catalog "git.vdhsn.com/barretthousen/barretthousen/src/catalog/api"
"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
"git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/domain"
"golang.org/x/sync/errgroup"
)
func init() {
kernel.TraceLog.Println("Registering AuctionFinder catawiki")
domain.RegisterAuctionFinder(
CatawikiAuctionFinder("catawiki"),
)
}
type CatawikiAuctionFinder string
func (cw CatawikiAuctionFinder) String() string {
return string(cw)
}
func (cw CatawikiAuctionFinder) Find(ctx context.Context, limit int, results chan<- catalog.Auction) (err error) {
defer close(results)
pageSize := 50
if limit > 2500 {
limit = 2500
}
if limit <= 0 {
limit = 2500
}
errg, errgCtx := errgroup.WithContext(ctx)
errg.SetLimit(5)
kernel.TraceLog.Printf("[Catawiki] fetching %d pages or up to %d results in %d batches", limit/pageSize, limit, pageSize)
for p := 1; p <= limit/pageSize; p++ {
pageIdx := p
errg.Go(func() error {
auctionR, err := GetUpcomingAuctions(errgCtx, limit, pageIdx)
if errors.Is(err, ErrNoResults) {
kernel.TraceLog.Println("no more results from catawiki api")
return err
} else if err != nil {
return err
}
for _, r := range auctionR {
select {
case <-errgCtx.Done():
kernel.TraceLog.Println("[Catawiki] group context exited early")
return nil
default:
results <- r
kernel.TraceLog.Printf("[Catawiki]: %+v", r)
}
}
return nil
})
}
if err = errg.Wait(); err != nil {
kernel.ErrorLog.Printf("[Catawiki] could not get results f: %v", err)
return
}
return
}
var ErrNoResults = errors.New("No results found")
func GetUpcomingAuctions(ctx context.Context, limit int, page int) (results []catalog.Auction, err error) {
if limit <= 25 {
limit = 25
} else if limit >= 50 {
limit = 50
}
if page < 1 {
page = 1
}
url := fmt.Sprintf("https://www.catawiki.com/buyer/api/v1/auctions?locale=en&=published_at_desc&per_page=%d&page=%d", limit, page)
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
req.Header.Add("Accept", "application/json")
req.Header.Add("Accept-Language", "en-US,en;q=0.5")
req.Header.Add("User-Agent", "barretthousen.com Auction Search Engine")
kernel.TraceLog.Printf("[Catawiki] Loading page %d of %d results", page, limit)
var res *http.Response
if res, err = http.DefaultClient.Do(req); err != nil {
return
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
err = fmt.Errorf("bad response code: %d", res.StatusCode)
return
}
type Payload struct {
Auctions []struct {
ID int `json:"id"`
StartAt string `json:"start_at"`
CloseAt string `json:"close_at"`
LotCount int `json:"lot_count"`
Title string `json:"title"`
URL string `json:"url"`
Status string `json:"status"`
Sellers []struct {
ID int `json:"id"`
Name string `json:"name"`
} `json:"sellers"`
} `json:"auctions"`
}
var p Payload
if err = json.NewDecoder(res.Body).Decode(&p); err != nil {
return
}
if len(p.Auctions) <= 0 {
err = ErrNoResults
return
}
results = make([]catalog.Auction, len(p.Auctions))
for idx, auction := range p.Auctions {
results[idx] = catalog.Auction{
Title: auction.Title,
Description: "",
SourceSiteURL: "https://www.catawiki.com",
SourceSiteName: "CataWiki",
SourceURL: auction.URL,
Country: "",
Province: "",
ItemCount: auction.LotCount,
}
results[idx].Start, _ = time.Parse(time.RFC3339, auction.StartAt)
results[idx].End, _ = time.Parse(time.RFC3339, auction.CloseAt)
}
return
}

View File

@ -1,44 +0,0 @@
package catawiki
import (
"context"
"reflect"
"testing"
catalog "git.vdhsn.com/barretthousen/barretthousen/src/catalog/api"
)
func TestGetUpcomingAuctions(t *testing.T) {
type args struct {
limit int
page int
}
tests := []struct {
name string
args args
wantResults []catalog.Auction
wantErr bool
}{
{
name: "1 Page, 25 Results",
args: args{
limit: 25,
page: 1,
},
wantResults: make([]catalog.Auction, 25),
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotResults, err := GetUpcomingAuctions(context.TODO(), tt.args.limit, tt.args.page)
if (err != nil) != tt.wantErr {
t.Errorf("GetUpcomingAuctions() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(gotResults, tt.wantResults) {
t.Errorf("GetUpcomingAuctions() = %v, want %v", gotResults, tt.wantResults)
}
})
}
}

View File

@ -2,7 +2,6 @@ package domain
import (
"context"
"errors"
"fmt"
"strings"
"time"
@ -61,32 +60,17 @@ type (
func (domain Domain) StartSync(ctx context.Context, in FindNewUpcomingInput) (out FindNewUpcomingOutput, err error) {
kernel.TraceLog.Printf("%+v", in)
if in.TargetSite == "All" || in.TargetSite == "" {
for k, v := range targetsImpls {
finder := targetsImpls["liveauctioneers"]
if out.Job, err = domain.Storage.CreateScrapeJob(ctx, k); err != nil {
err = fmt.Errorf("could not create new scrape job record: %w", err)
continue
}
kernel.InfoLog.Printf("Scrape Job %d starting", out.Job.ID)
go domain.executeScrapeJob(v, out.Job.ID)
}
} else if finder, ok := targetsImpls[in.TargetSite]; ok {
if out.Job, err = domain.Storage.CreateScrapeJob(ctx, in.TargetSite); err != nil {
err = fmt.Errorf("could not create new scrape job record: %w", err)
return
}
kernel.InfoLog.Printf("Scrape Job %d starting", out.Job.ID)
go domain.executeScrapeJob(finder, out.Job.ID)
} else {
kernel.TraceLog.Println("could not find target")
err = errors.New("No scrape job found by name")
if out.Job, err = domain.Storage.CreateScrapeJob(ctx, in.TargetSite); err != nil {
err = fmt.Errorf("could not create new scrape job record: %w", err)
return
}
kernel.InfoLog.Printf("Scrape Job %d starting", out.Job.ID)
// TODO: make everything after this line async and run after return
go domain.executeScrapeJob(finder, out.Job.ID)
return
}
@ -132,13 +116,13 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
errs := &strings.Builder{}
for auction := range found {
total++
if !auction.Start.After(time.Now().Add(-24 * time.Hour)) {
if !auction.Start.After(time.Now()) {
continue
}
ace, err := domain.CatalogService.UpdateUpcomingAuction(ctx, auction)
if err != nil {
kernel.ErrorLog.Printf("[%s] could not import upcoming auction: %s", finder.String(), err.Error())
kernel.TraceLog.Printf("could not import upcoming auction: %s", err.Error())
fmt.Fprintf(errs, "{ \"AuctionFingerprint\": \"%s\", \"error\": \"%s\" }\n", ace.Fingerprint, err.Error())
continue
}
@ -148,10 +132,9 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
}
}
kernel.TraceLog.Printf("[%s] waiting for results...", finder.String())
if err := errGroup.Wait(); err != nil {
err = fmt.Errorf("an issue occurred while finding upcoming items iteration: %w", err)
fmt.Fprintf(errs, "{\"error\": \"%s\" }", err.Error())
fmt.Fprintf(errs, "{ \"error\": \"%s\" }", err.Error())
}
var completedJob ScrapeJob
@ -160,9 +143,9 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
AuctionCount: count,
Errors: errs.String(),
}); err != nil {
kernel.ErrorLog.Printf("[%s] Could not complete scrape job, failing: %v", finder.String(), err)
kernel.ErrorLog.Printf("Could not complete scrape job, failing: %v", err)
}
kernel.InfoLog.Printf("[%s] Scrape Job %d completed in %v. Successfully imported %d/%d", finder.String(), jobID, completedJob.Completed.Sub(completedJob.Started), count, total)
kernel.InfoLog.Printf("Scrape Job %d completed in %v. Successfully imported %d/%d", jobID, completedJob.Completed.Sub(completedJob.Started), count, total)
return
}

View File

@ -16,7 +16,6 @@ import (
"go.uber.org/dig"
"google.golang.org/grpc"
_ "git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/domain/catawiki"
_ "git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/domain/liveauctioneers"
)

View File

@ -39,10 +39,10 @@
},
"type": "module",
"dependencies": {
"@analytics/google-analytics": "^1.0.7",
"@segment/snippet": "^4.16.2",
"analytics": "^0.8.9",
"analytics-plugin-do-not-track": "^0.1.5",
"@analytics/google-analytics": "^1.0.7",
"@segment/snippet": "^4.16.2",
"luxon": "^3.3.0"
}
}

View File

@ -4,16 +4,16 @@
import Analytics from 'analytics';
// import doNotTrack from 'analytics-plugin-do-not-track';
// import googleAnalytics from '@analytics/google-analytics';
import googleAnalytics from '@analytics/google-analytics';
const analytics = Analytics({
app: 'Barretthousen - Web Client',
version: '1.0',
debug: process.env.NODE_ENV !== 'production',
plugins: [
// googleAnalytics({
// measurementIds: ['UA-143763293-1']
// })
googleAnalytics({
measurementIds: ['UA-143763293-1']
})
// doNotTrack()
]
});