Barretthousen
/
barretthousen
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

WIP: feat/kafka #11

Closed
adam wants to merge 7 commits from feat/kafka into feat/mvp
pull from: feat/kafka
merge into: Barretthousen:feat/mvp
Conversation 0 Commits 7 Files Changed 59
59 changed files with 7621 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
Tiltfile
View File

@ -11,6 +11,7 @@ load('ext://restart_process', 'docker_build_with_restart')
helm_repo('bitnami', 'https://charts.bitnami.com/bitnami', labels=["9-repos"]) helm_repo('bitnami', 'https://charts.bitnami.com/bitnami', labels=["9-repos"])
helm_repo('traefik', 'https://traefik.github.io/charts', labels=["9-repos"]) helm_repo('traefik', 'https://traefik.github.io/charts', labels=["9-repos"])
helm_repo('grafana', 'https://grafana.github.io/helm-charts', labels=["9-repos"]) helm_repo('grafana', 'https://grafana.github.io/helm-charts', labels=["9-repos"])
helm_repo('kafka-ui-github', 'https://provectus.github.io/kafka-ui', labels=["9-repos"])
helm_resource( helm_resource(
'ingress', 'ingress',
@ -64,6 +65,56 @@ helm_resource(
labels=["9-data"] labels=["9-data"]
) )
helm_resource(
'kafka-ui',
'kafka-ui-github/kafka-ui',
namespace='barretthousen-local',
flags=[
'--set', 'envs.config.KAFKA_CLUSTERS_0_NAME=bh-kafka',
'--set', 'envs.config.KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:9092'
],
port_forwards=[port_forward(9090, 8080, name='kafka-ui')],
resource_deps=['kafka'],
labels=["9-data"]
)
def bh_client(service="", port_forwards=[], labels=['2-services'], deps=['ingress']):
# docker_build('example-nodejs-image', '.',
# build_args={'node_env': 'development'},
# entrypoint='yarn run nodemon --ext js,mustache /app/index.js',
# live_update=[
# sync('.', '/app'),
# run('cd /app && yarn install', trigger=['./package.json', './yarn.lock']),
# # if all that changed was start-time.txt, make sure the server
# # reloads so that it will reflect the new startup time
# run('touch /app/index.js', trigger='./start-time.txt'),
# # add a congrats message!
# run('sed -i "s/Hello cats!/{}/g" /app/views/index.mustache'.
# format(congrats)),
# ])
basepath = './src/{}-client'.format(service)
docker_build(
ref='barretthousen/client-{}-client'.format(service),
context=basepath,
dockerfile=basepath +'/Dockerfile.dev-frontend'.format(service),
target='development',
entrypoint='vite dev --port=80 --host=0.0.0.0 --strictPort --logLevel info',
live_update=[
sync(basepath + '/src', '/opt/{}-client/src'.format(service)),
sync(basepath + '/static', '/opt/{}-client/static'.format(service)),
run('cd {} && npm install'.format(basepath), trigger=[basepath+'/package.json', basepath+'/package.lock.json'])
]
)
k8s_resource(
workload='{}-client-local'.format(service),
port_forwards=port_forwards,
labels=['2-services'],
resource_deps=deps
)
def bh_backend_service(service="", port_forwards=[], migrateDB=False, devMode=True, labels=['2-services'], deps=['postgres']): def bh_backend_service(service="", port_forwards=[], migrateDB=False, devMode=True, labels=['2-services'], deps=['postgres']):
local_resource( local_resource(
@ -74,6 +125,7 @@ def bh_backend_service(service="", port_forwards=[], migrateDB=False, devMode=Tr
labels=['3-compilation'] labels=['3-compilation']
) )
# complains about grpc port still being in use, so maybe the service isn't exiting cleanly
entry_cmd = [ entry_cmd = [
'/go/bin/dlv', '/go/bin/dlv',
'--headless', '--headless',
@ -111,7 +163,7 @@ def bh_backend_service(service="", port_forwards=[], migrateDB=False, devMode=Tr
], ],
live_update=[ live_update=[
sync('./.bin', '/opt'), sync('./.bin', '/opt'),
sync('./src', '/go/src') sync('./src', '/go/src'),
] ]
) )
@ -132,27 +184,15 @@ k8s_resource(
labels='2-services' labels='2-services'
) )
docker_build(
ref='barretthousen/service-web-client',
context='./src/web-client',
dockerfile='./src/web-client/Dockerfile.dev-frontend',
target='development'
)
k8s_resource(
workload='web-client-local',
port_forwards=['8080:80'],
labels=['2-services'],
resource_deps=['ingress']
)
bh_backend_service(service="runner", migrateDB=True, port_forwards=[ bh_backend_service(service="runner", migrateDB=True, port_forwards=[
port_forward(2345, name='Delve port') port_forward(2345, name='Delve port')
]) ], deps=["postgres", "kafka"])
bh_backend_service(service="catalog", migrateDB=True, port_forwards=[ bh_backend_service(service="catalog", migrateDB=True, port_forwards=[
port_forward(2346, 2345, name='Delve port') port_forward(2346, 2345, name='Delve port')
]) ], deps=["postgres", "kafka"])
bh_backend_service(service="proxy-admin", port_forwards=[ bh_backend_service(service="proxy-admin", port_forwards=[
port_forward(8082, 80, name="HTTP API @ localhost:8082") port_forward(8082, 80, name="HTTP API @ localhost:8082")
@ -162,6 +202,10 @@ bh_backend_service(service="proxy-client", port_forwards=[
port_forward(8081, 80, name="HTTP API @ localhost:8081") port_forward(8081, 80, name="HTTP API @ localhost:8081")
], deps=['ingress']) ], deps=['ingress'])
bh_client(service='web')
bh_client(service='admin')
# local_resource( # local_resource(
# 'dev-web-client', # 'dev-web-client',

41
env/base/admin-client-deployment.yaml vendored Normal file
View File

@ -0,0 +1,41 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: admin-client
spec:
selector:
matchLabels:
app: admin-client
template:
metadata:
labels:
app: admin-client
spec:
serviceAccountName: barretthousen-service
containers:
- name: admin-client
image: barretthousen/client-admin-client:latest
imagePullPolicy: Always
stdin: true
tty: true
env:
- name: ORIGIN
value: https://admin.barretthousen.com
ports:
- containerPort: 80
name: http
resources:
limits:
memory: "512Mi"
cpu: "500m"
---
apiVersion: v1
kind: Service
metadata:
name: admin-client
spec:
selector:
app: admin-client
ports:
- port: 80
targetPort: 80

7
env/base/admin-ingress.yaml vendored
View File

@ -16,3 +16,10 @@ spec:
name: proxy-admin name: proxy-admin
port: port:
number: 80 number: 80
- pathType: ImplementationSpecific
path: "/"
backend:
service:
name: admin-client
port:
number: 80

3
env/base/kustomization.yaml vendored
View File

@ -5,7 +5,8 @@ resources:
- ./runner-deployment.yaml - ./runner-deployment.yaml
- ./proxy-admin-deployment.yaml - ./proxy-admin-deployment.yaml
- ./proxy-client-deployment.yaml - ./proxy-client-deployment.yaml
- ./sync-cronjob.yaml
- ./web-client-deployment.yaml - ./web-client-deployment.yaml
- ./admin-client-deployment.yaml
- ./client-ingress.yaml - ./client-ingress.yaml
- ./admin-ingress.yaml - ./admin-ingress.yaml
- ./scrape-cronjob.yaml

0
env/base/scrape-cronjob.yaml → env/base/sync-cronjob.yaml vendored
View File

2
env/base/web-client-deployment.yaml vendored
View File

@ -14,7 +14,7 @@ spec:
serviceAccountName: barretthousen-service serviceAccountName: barretthousen-service
containers: containers:
- name: web-client - name: web-client
image: barretthousen/service-web-client:latest image: barretthousen/client-web-client:latest
imagePullPolicy: Always imagePullPolicy: Always
stdin: true stdin: true
tty: true tty: true

2
env/beta/kustomization.yaml vendored
View File

@ -6,7 +6,7 @@ nameSuffix: -beta
namespace: barretthousen-beta namespace: barretthousen-beta
patchesStrategicMerge: patchesStrategicMerge:
- scrape-cronjob.yaml - sync-cronjob.yaml
- catalog-secret.yaml - catalog-secret.yaml
- proxy-admin-secret.yaml - proxy-admin-secret.yaml
- proxy-client-secret.yaml - proxy-client-secret.yaml

0
env/beta/scrape-cronjob.yaml → env/beta/sync-cronjob.yaml vendored
View File

2
env/local/kustomization.yaml vendored
View File

@ -10,7 +10,7 @@ namespace: barretthousen-local
patchesStrategicMerge: patchesStrategicMerge:
- debug-catalog.yaml - debug-catalog.yaml
- debug-runner.yaml - debug-runner.yaml
- scrape-cronjob.yaml - sync-cronjob.yaml
patches: patches:
- target: - target:

0
env/local/scrape-cronjob.yaml → env/local/sync-cronjob.yaml vendored
View File

2
env/prod/kustomization.yaml vendored
View File

@ -6,7 +6,7 @@ commonLabels:
namespace: barretthousen namespace: barretthousen
patchesStrategicMerge: patchesStrategicMerge:
- scrape-cronjob.yaml - sync-cronjob.yaml
- catalog-configmap.yaml - catalog-configmap.yaml
- proxy-admin-configmap.yaml - proxy-admin-configmap.yaml
- proxy-client-configmap.yaml - proxy-client-configmap.yaml

0
env/prod/scrape-cronjob.yaml → env/prod/sync-cronjob.yaml vendored
View File

201
go.work.sum
View File

@ -1,224 +1,29 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E=
cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ=
cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw=
cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE=
cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8=
cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8=
cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc=
cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8=
cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E=
cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k=
cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08=
cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw=
cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU=
cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss=
cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g=
cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU=
cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU=
cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc=
cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q=
cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8=
cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU=
cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s=
cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA=
cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs=
cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs=
cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w=
cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA=
cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s=
cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8=
cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE=
cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE=
cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8=
cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM=
cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs=
cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4=
cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c=
cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c=
cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww=
cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ=
cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE=
cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4=
cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs=
cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE=
cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY=
cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU=
cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M=
cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY=
cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg=
cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE=
cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c=
cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0=
cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg=
cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw=
cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw=
cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y=
cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo=
cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0=
cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74=
cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4=
cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE=
cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI=
cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY=
cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA=
cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY=
cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I=
cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM=
cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo=
cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw=
cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM=
cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY=
cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU=
cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ=
cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI=
cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ=
cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc=
cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw=
cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs=
cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk=
cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc=
cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs=
cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4=
cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM=
cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c=
cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac=
cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ=
cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ=
cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI=
cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA=
cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14=
cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg=
cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc=
cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0=
cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag=
cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk=
cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s=
cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4=
cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA=
cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A=
cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M=
cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI=
cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw=
cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c=
cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc=
cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM=
cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk=
cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos=
cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ=
cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU=
cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0=
cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY=
cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY=
cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes=
cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg=
cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng=
cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/elastic/elastic-transport-go/v8 v8.0.0-alpha/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/elastic/go-elasticsearch/v8 v8.0.0/go.mod h1:8NCWP26meGbncX+R9sxo2JD8IqBjRTuS7yXMstHpd40=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.uber.org/dig v1.0.0/go.mod h1:z+dSd2TP9Usi48jL8M3v63iSBVkiwtVyMKxMZYYauPg= go.uber.org/dig v1.0.0/go.mod h1:z+dSd2TP9Usi48jL8M3v63iSBVkiwtVyMKxMZYYauPg=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

1
src/admin-client/.dockerignore Normal file
View File

@ -0,0 +1 @@
build

13
src/admin-client/.eslintignore Normal file
View File

@ -0,0 +1,13 @@
.DS_Store
node_modules
/build
/.svelte-kit
/package
.env
.env.*
!.env.example
# Ignore files for PNPM, NPM and YARN
pnpm-lock.yaml
package-lock.json
yarn.lock

30
src/admin-client/.eslintrc.cjs Normal file
View File

@ -0,0 +1,30 @@
module.exports = {
root: true,
extends: [
'eslint:recommended',
'plugin:@typescript-eslint/recommended',
'plugin:svelte/recommended',
'prettier'
],
parser: '@typescript-eslint/parser',
plugins: ['@typescript-eslint'],
parserOptions: {
sourceType: 'module',
ecmaVersion: 2020,
extraFileExtensions: ['.svelte']
},
env: {
browser: true,
es2017: true,
node: true
},
overrides: [
{
files: ['*.svelte'],
parser: 'svelte-eslint-parser',
parserOptions: {
parser: '@typescript-eslint/parser'
}
}
]
};

10
src/admin-client/.gitignore vendored Normal file
View File

@ -0,0 +1,10 @@
.DS_Store
node_modules
/build
/.svelte-kit
/package
.env
.env.*
!.env.example
vite.config.js.timestamp-*
vite.config.ts.timestamp-*

13
src/admin-client/.prettierignore Normal file
View File

@ -0,0 +1,13 @@
.DS_Store
node_modules
/build
/.svelte-kit
/package
.env
.env.*
!.env.example
# Ignore files for PNPM, NPM and YARN
pnpm-lock.yaml
package-lock.json
yarn.lock

9
src/admin-client/.prettierrc Normal file
View File

@ -0,0 +1,9 @@
{
"useTabs": true,
"singleQuote": true,
"trailingComma": "none",
"printWidth": 100,
"plugins": ["prettier-plugin-svelte"],
"pluginSearchDirs": ["."],
"overrides": [{ "files": "*.svelte", "options": { "parser": "svelte" } }]
}

9
src/admin-client/Dockerfile.dev-frontend Normal file
View File

@ -0,0 +1,9 @@
FROM node:lts AS development
COPY . /opt/admin-client
WORKDIR /opt/admin-client
RUN npm install && npm install -g vite
CMD vite dev --port=80 --host=0.0.0.0 --strictPort --logLevel info

42
src/admin-client/Dockerfile.frontend Normal file
View File

@ -0,0 +1,42 @@
FROM node:lts AS build
ARG origin=https://barretthousen.com
ENV PROTOCOL_HEADER=x-forwarded-proto
ENV HOST_HEADER=x-forwarded-host
ENV ORIGIN=${origin}
WORKDIR /opt/admin-client
COPY . /opt/admin-client
RUN npm run build \
&& cp -v /opt/admin-client/package.json /opt/admin-client/build \
&& cp -v /opt/admin-client/package-lock.json /opt/admin-client/build \
&& cd /opt/admin-client/build && npm ci --omit dev
FROM node:lts AS production
LABEL com.barretthousen.service="admin-client"
LABEL com.barretthousen.tags="frontend,public"
LABEL com.barretthousen.release-date="2023-05-01"
LABEL com.barretthousen.version="alpha-0.0.1"
LABEL com.barretthousen.git-ref="000000000000000000000000000000000000000000000000"
ARG origin=https://barretthousen.com
ENV ENV=production
ENV PROTOCOL_HEADER=x-forwarded-proto
ENV HOST_HEADER=x-forwarded-host
ENV ORIGIN=${origin}
WORKDIR /opt
COPY --from=build /opt/admin-client/build/ /opt/admin-client/
ENV PORT 80
EXPOSE 80
ENTRYPOINT ["node", "admin-client"]

38
src/admin-client/README.md Normal file
View File

@ -0,0 +1,38 @@
# create-svelte
Everything you need to build a Svelte project, powered by [`create-svelte`](https://github.com/sveltejs/kit/tree/master/packages/create-svelte).
## Creating a project
If you're seeing this, you've probably already done this step. Congrats!
```bash
# create a new project in the current directory
npm create svelte@latest
# create a new project in my-app
npm create svelte@latest my-app
```
## Developing
Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
```bash
npm run dev
# or start the server and open the app in a new browser tab
npm run dev -- --open
```
## Building
To create a production version of your app:
```bash
npm run build
```
You can preview the production build with `npm run preview`.
> To deploy your app, you may need to install an [adapter](https://kit.svelte.dev/docs/adapters) for your target environment.

4251
src/admin-client/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

44
src/admin-client/package.json Normal file
View File

@ -0,0 +1,44 @@
{
"name": "adminb-client",
"version": "0.0.1",
"private": true,
"main": "index.js",
"scripts": {
"dev": "vite dev --port 8080",
"build": "vite build",
"preview": "vite preview",
"test": "playwright test",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
"test:unit": "vitest",
"lint": "prettier --plugin-search-dir . --check . && eslint .",
"format": "prettier --plugin-search-dir . --write ."
},
"devDependencies": {
"@playwright/test": "^1.28.1",
"@sveltejs/adapter-node": "^1.2.4",
"@sveltejs/kit": "^1.5.0",
"@types/luxon": "^3.3.0",
"@typescript-eslint/eslint-plugin": "^5.45.0",
"@typescript-eslint/parser": "^5.45.0",
"autoprefixer": "^10.4.14",
"eslint": "^8.28.0",
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-svelte": "^2.26.0",
"less": "^4.1.3",
"postcss": "^8.4.23",
"prettier": "^2.8.0",
"prettier-plugin-svelte": "^2.8.1",
"svelte": "^3.54.0",
"svelte-check": "^3.0.1",
"tailwindcss": "^3.3.2",
"tslib": "^2.4.1",
"typescript": "^5.0.0",
"vite": "^4.3.0",
"vitest": "^0.25.3"
},
"type": "module",
"dependencies": {
"luxon": "^3.3.0"
}
}

12
src/admin-client/playwright.config.ts Normal file
View File

@ -0,0 +1,12 @@
import type { PlaywrightTestConfig } from '@playwright/test';
const config: PlaywrightTestConfig = {
webServer: {
command: 'npm run build && npm run preview',
port: 4173
},
testDir: 'tests',
testMatch: /(.+\.)?(test|spec)\.[jt]s/
};
export default config;

6
src/admin-client/postcss.config.js Normal file
View File

@ -0,0 +1,6 @@
export default {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
}

28
src/admin-client/src/app.css Normal file
View File

@ -0,0 +1,28 @@
@tailwind base;
@tailwind components;
@tailwind utilities;
body,
html {
margin: 0;
padding: 0;
background: #222222;
color: #FFFFFF;
}
button,
input {
background: #222222;
border: #C2A661 solid 1px;
color: #FFFFFF;
}
a:hover {
color: #ebca78;
}
a {
color: #C2A661;
text-decoration: underline;
}

12
src/admin-client/src/app.d.ts vendored Normal file
View File

@ -0,0 +1,12 @@
// See https://kit.svelte.dev/docs/types#app
// for information about these interfaces
declare global {
namespace App {
// interface Error {}
// interface Locals {}
// interface PageData {}
// interface Platform {}
}
}
export {};

12
src/admin-client/src/app.html Normal file
View File

@ -0,0 +1,12 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="icon" href="%sveltekit.assets%/favicon.png" />
<meta name="viewport" content="width=device-width" />
%sveltekit.head%
</head>
<body data-sveltekit-preload-data="hover">
<div style="display: contents">%sveltekit.body%</div>
</body>
</html>

7
src/admin-client/src/index.test.ts Normal file
View File

@ -0,0 +1,7 @@
import { describe, it, expect } from 'vitest';
describe('sum test', () => {
it('adds 1 + 2 to equal 3', () => {
expect(1 + 2).toBe(3);
});
});

35
src/admin-client/src/lib/ScrapeJobResult.svelte Normal file
View File

@ -0,0 +1,35 @@
<script lang="ts">
import { DateTime } from 'luxon';
interface ScrapeJob {
id: Number;
targetSiteName: string;
createdTs: string;
completedTs: string;
auctionsFound: Number;
errors: string;
}
export let job: ScrapeJob;
const { id, targetSiteName, createdTs, completedTs, auctionsFound, errors } = job;
const createdDT = DateTime.fromISO(createdTs);
const completedDT = DateTime.fromISO(completedTs);
</script>
<article class="scrape-job-result flex flex-col py-3 shadow-sm border-b-2 border-bh-gold m-2">
<header class="pb-2">
Job #{id}
</header>
<section>
<ul>
<li class="text-sm">Target: {targetSiteName}</li>
<li class="text-sm">Started: {createdDT?.toLocaleString(DateTime.DATETIME_FULL)}</li>
{#if completedTs !== null}
<li class="text-sm">Completed: {completedDT?.toLocaleString(DateTime.DATETIME_FULL)}</li>
{/if}
<li class="text-sm">Found: {auctionsFound}</li>
<li class="text-sm">Errors: {errors}</li>
</ul>
</section>
</article>

20
src/admin-client/src/lib/StartScrapeForm.svelte Normal file
View File

@ -0,0 +1,20 @@
<script lang="ts">
import { createEventDispatcher } from 'svelte';
let target: string = 'All';
const dispatch = createEventDispatcher();
function execScrape() {
dispatch('scrape', { target });
}
</script>
<form class="flex rounded-md" on:submit|preventDefault={() => execScrape()}>
<select name="target" class="bg-bh-gold text-bh-black text-center px-4" bind:value={target}>
<option>All</option>
<option>liveauctioneers</option>
</select>
<button class="border-none rounded-r-md bg-bh-gold text-bh-black py-1 px-2">Start Sync</button>
</form>

48
src/admin-client/src/routes/+layout.svelte Normal file
View File

@ -0,0 +1,48 @@
<script lang="ts">
import '../app.css';
</script>
<svelte:head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>BH Admin Console</title>
<meta name="description" content="" />
</svelte:head>
<main class="flex flex-col w-full">
<nav
class="flex w-full bg-bh-gold text-bh-black fixed top-0 left-0 right-0"
style="height: 80px;"
>
<img
class="pr-5"
style="width: 450px; height: 80px;"
src="/logo_black_gold.svg"
alt="BarrettHousen"
/>
<ul class="flex w-full py-3 items-center justify-between">
<li class="flex grow justify-center">
<span class="flex grow" style="max-width: 75%;"> Admin Stuff </span>
</li>
<li class="px-6">
<!-- <span>I want email alerts!</span> -->
</li>
</ul>
</nav>
<div class="flex flex-row flex-grow h-full" style="padding-top: 80px;">
<div id="gutter" class="h-full" style="min-width: 195px; max-width: 195px;" />
<section id="content" class="py-6 pl-4 grow border-l border-bh-gold" style="min-height: 100vh;">
<slot />
</section>
</div>
<footer
class="flex justify-center bg-bh-gold text-bh-black w-full py-10 shadow-inner shadnow-md shadow-bh-black"
>
<p>
We find great auctions of rare and one of a kind collectibles from around the web and compile
them in one easy place.
</p>
</footer>
</main>

11
src/admin-client/src/routes/+layout.ts Normal file
View File

@ -0,0 +1,11 @@
import type { LayoutLoad } from './$types';
export const load = (({ url }) => {
return {
// TODO: refactor to one source of truth for all query param values
query: url.searchParams.get('query') || '',
page: parseInt(url.searchParams.get('page') || '0'),
limit: parseInt(url.searchParams.get('pageSize') || '64'),
};
}) satisfies LayoutLoad;

61
src/admin-client/src/routes/+page.svelte Normal file
View File

@ -0,0 +1,61 @@
<script lang="ts">
import type { PageData } from './$types';
import { fade } from 'svelte/transition';
import { invalidateAll } from '$app/navigation';
import StartScrapeForm from '$lib/StartScrapeForm.svelte';
import ScrapeJobResult from '$lib/ScrapeJobResult.svelte';
export let data: PageData;
$: completedJobs = data.jobs.filter(({ completedTs }) => completedTs !== null);
$: activeJobs = data.jobs.filter(({ completedTs }) => completedTs === null);
$: activeJobCount = activeJobs.length;
$: completedJobCount = completedJobs.length;
async function onScrape({ detail }) {
try {
const response = await fetch(
new Request('/api/v1/sync', {
method: 'PUT',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ targetSite: detail.target })
})
);
const scrapeJob = await response.json();
data.jobs.push(scrapeJob);
console.log(scrapeJob);
} catch (error) {}
}
</script>
<section class="flex w-full flex-col justify-center" in:fade out:fade>
<h1 class="text-2xl mb-8">Sync Status</h1>
<div class="mb-8">
<StartScrapeForm on:scrape={onScrape} />
</div>
<section>
<h2>{activeJobCount} In Progress Jobs</h2>
<ul class="flex">
{#each activeJobs as j, i}
<li in:fade>
<ScrapeJobResult job={j} />
</li>
<!-- {#if i < 10}
{/if} -->
{/each}
</ul>
<h2>{completedJobCount} Complete</h2>
<ul class="flex flex-wrap justify-between">
{#each completedJobs as j, i}
<li in:fade>
<ScrapeJobResult job={j} />
</li>
<!-- {#if i < 5}
{/if} -->
{/each}
</ul>
</section>
</section>

32
src/admin-client/src/routes/+page.ts Normal file
View File

@ -0,0 +1,32 @@
import { browser } from '$app/environment';
import type { PageLoad } from './$types';
// TODO: change to env var
const API_HOST = `${browser ? '' : 'http://proxy-admin-local'}/api/v1`;
interface Job {
id: Number;
targetSiteName: string;
createdTs: string;
completedTs: string;
auctionsFound: Number;
errors: string;
}
interface ScrapeStatusPageData {
jobs: Job[]
}
export const load = (async ({ fetch, url }): Promise<ScrapeStatusPageData> => {
try {
const response = await fetch(API_HOST + `/sync`);
const {results } = await response.json();
return {
jobs: results || []
};
} catch (e) {
console.log(e);
return { jobs:[] };
}
}) satisfies PageLoad;

BIN
src/admin-client/static/favicon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.1 KiB

21
src/admin-client/static/logo_black_gold.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

21
src/admin-client/static/logo_black_white.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

23
src/admin-client/svelte.config.js Normal file
View File

@ -0,0 +1,23 @@
import adapter from '@sveltejs/adapter-node';
import { vitePreprocess } from '@sveltejs/kit/vite';
/** @type {import('@sveltejs/kit').Config} */
const config = {
// Consult https://kit.svelte.dev/docs/integrations#preprocessors
// for more information about preprocessors
preprocess: vitePreprocess(),
kit: {
env: {
publicPrefix: 'BH_WEB_CLIENT'
},
// adapter-auto only supports some environments, see https://kit.svelte.dev/docs/adapter-auto for a list.
// If your environment is not supported or you settled on a specific environment, switch out the adapter.
// See https://kit.svelte.dev/docs/adapters for more information about adapters.
adapter: adapter({
precompress: true
})
}
};
export default config;

13
src/admin-client/tailwind.config.js Normal file
View File

@ -0,0 +1,13 @@
/** @type {import('tailwindcss').Config} */
export default {
content: ['./src/**/*.{html,js,svelte,ts}'],
theme: {
extend: {
colors: {
"bh-gold": "#C2A661",
"bh-black": "#222222",
}
}
},
plugins: []
};

8
src/admin-client/tests/test.ts Normal file
View File

@ -0,0 +1,8 @@
import { expect, test } from '@playwright/test';
test('Start Scape control exists', async ({ page }) => {
await page.goto('/');
await expect(page.getByRole('textbox', { name: 'target' })).toBeVisible();
await expect(page.getByRole('button', { name: 'Run Job' })).toBeVisible();
});

17
src/admin-client/tsconfig.json Normal file
View File

@ -0,0 +1,17 @@
{
"extends": "./.svelte-kit/tsconfig.json",
"compilerOptions": {
"allowJs": true,
"checkJs": true,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"skipLibCheck": true,
"sourceMap": true,
"strict": true
}
// Path aliases are handled by https://kit.svelte.dev/docs/configuration#alias
//
// If you want to overwrite includes/excludes, make sure to copy over the relevant includes/excludes
// from the referenced tsconfig.json - TypeScript does not merge them in
}

9
src/admin-client/vite.config.ts Normal file
View File

@ -0,0 +1,9 @@
import { sveltekit } from '@sveltejs/kit/vite';
import { defineConfig } from 'vitest/config';
export default defineConfig({
plugins: [sveltekit()],
test: {
include: ['src/**/*.{test,spec}.{js,ts}']
}
});

4
src/buf.lock
View File

@ -9,5 +9,5 @@ deps:
- remote: buf.build - remote: buf.build
owner: grpc-ecosystem owner: grpc-ecosystem
repository: grpc-gateway repository: grpc-gateway
commit: a1ecdc58eccd49aa8bea2a7a9022dc27 commit: 11c9972ea0fd4c95a2c38d29bb1dc817
digest: shake256:efdd86fbdc42e8b7259fe461a49656827a03fb7cba0b3b9eb622ca10654ec6beccb9a051229c1553ccd89ed3e95d69ad4d7c799f1da3f3f1bd447b7947a4893e digest: shake256:9c7ce822dff52ad28714465396fbe98e879409677a61687b7dd9bb3d1484aa5d1704c013698b24f34c5d51023dbff47287ecd9676271953c25e646b42ebb76c5

78
src/catalog/api/client.go
View File

@ -1,64 +1,38 @@
package api package api
import ( import (
"context"
"time" "time"
capi "git.vdhsn.com/barretthousen/barretthousen/src/catalog/api/grpc"
"google.golang.org/grpc"
"google.golang.org/protobuf/types/known/timestamppb"
) )
func NewCatalogServiceClient(conn grpc.ClientConnInterface) *CatalogServiceClient {
return &CatalogServiceClient{
CatalogClient: capi.NewCatalogClient(conn),
}
}
type CatalogServiceClient struct {
capi.CatalogClient
}
type Auction struct { type Auction struct {
Fingerprint string `json:"fingerprint,omitempty"` ID int `json:"id,omitempty"`
Title string `json:"title,omitempty"` Fingerprint string `json:"fingerprint,omitempty"`
Description string `json:"description,omitempty"` Title string `json:"title,omitempty"`
SourceSiteURL string `json:"source_site_url,omitempty"` Description string `json:"description,omitempty"`
SourceSiteName string `json:"source_site_name,omitempty"` ItemCount int `json:"itemCount,omitempty"`
SourceURL string `json:"source_url,omitempty"` Start time.Time `json:"start,omitempty"`
Country string `json:"country,omitempty"` End time.Time `json:"end,omitempty"`
Province string `json:"province,omitempty"` Source SourceDetail `json:"source"`
ItemCount int `json:"itemCount,omitempty"` Seller SellerDetail `json:"seller"`
Start time.Time `json:"start,omitempty"` Address AddressDetail `json:"address"`
End time.Time `json:"end,omitempty"`
} }
type AuctionCreatedEvent struct { type SourceDetail struct {
Fingerprint string Name string `json:"name"`
ID int AuctionURL string `json:"auctionUrl"`
Duplicate bool SiteURL string `json:"siteURL"`
} }
func (css *CatalogServiceClient) UpdateUpcomingAuction(ctx context.Context, a Auction) (AuctionCreatedEvent, error) { type SellerDetail struct {
ac, err := css.ImportAuction(ctx, &capi.ImportAuctionMessage{ ID int `json:"id"`
Items: int32(a.ItemCount), Name string `json:"name"`
Start: timestamppb.New(a.Start), SiteURL string `json:"siteUrl"`
End: timestamppb.New(a.End), }
Title: a.Title,
Description: a.Description, type AddressDetail struct {
SourceSiteURL: a.SourceSiteURL, CountryCode string `json:"country"`
SourceSiteName: a.SourceSiteName, Lat float64 `json:"lat"`
SourceURL: a.SourceURL, Long float64 `json:"lng"`
Country: a.Country, State string `json:"state"`
Province: a.Province, City string `json:"city"`
})
if err != nil {
return AuctionCreatedEvent{}, err
}
return AuctionCreatedEvent{
Fingerprint: ac.Auction.Fingerprint,
ID: int(ac.Auction.Id),
Duplicate: ac.Duplicate,
}, nil
} }

4
src/catalog/api/grpc/catalog.proto
View File

@ -14,8 +14,8 @@ service Catalog {
}; };
} }
rpc ImportAuction(ImportAuctionMessage) returns (AuctionCreated) { // rpc ImportAuction(ImportAuctionMessage) returns (AuctionCreated) {
} // }
} }
message AuctionSearchCriteria { message AuctionSearchCriteria {

58
src/catalog/internal/data/kafka/consumer.go Normal file
View File

@ -0,0 +1,58 @@
package kafka
import (
"context"
"errors"
"io"
"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
kafka "github.com/segmentio/kafka-go"
)
type kafkaConsumer struct {
*kafka.Conn
}
func (c *kafkaConsumer) Close() error {
return c.Conn.Close()
}
func (c *kafkaConsumer) ConsumeAsync(ctx context.Context, output chan<- []byte) {
for {
batch := c.ReadBatch(1024, 1024*50)
for {
msg, err := batch.ReadMessage()
if err != nil && !errors.Is(err, io.EOF) || len(msg.Value) == 0 {
break
}
kernel.TraceLog.Printf("header: %+v, value: %+v", msg.Headers, msg.Value)
select {
case output <- msg.Value:
case <-ctx.Done():
batch.Close()
return
}
}
batch.Close()
select {
case <-ctx.Done():
return
}
}
}
func NewConsumer(topic string) (*kafkaConsumer, error) {
conn, err := kafka.DialLeader(context.Background(), "tcp", "kafka:9092", topic, 0)
if err != nil {
return nil, err
}
return &kafkaConsumer{
Conn: conn,
}, nil
}

55
src/catalog/internal/indexer.go Normal file
View File

@ -0,0 +1,55 @@
package internal
import (
"context"
"encoding/json"
"io"
api "git.vdhsn.com/barretthousen/barretthousen/src/catalog/api"
"git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal/domain"
"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
)
type KafkaConsumer interface {
io.Closer
ConsumeAsync(context.Context, chan<- []byte)
}
type AuctionImporter interface {
ImportAuction(context.Context, domain.ImportAuctionMessage) (domain.AuctionCreated, error)
}
func RunIndexer(ctx context.Context, c KafkaConsumer, importer AuctionImporter) {
defer c.Close()
msgs := make(chan []byte, 64)
go c.ConsumeAsync(ctx, msgs)
for msg := range msgs {
var auction api.Auction
if err := json.Unmarshal(msg, &auction); err != nil {
kernel.TraceLog.Printf("could not ingest: %w", err)
continue
}
_, err := importer.ImportAuction(ctx, domain.ImportAuctionMessage{
Auction: domain.Auction{
ItemCount: auction.ItemCount,
Start: auction.Start,
End: auction.End,
Title: auction.Title,
Description: auction.Description,
SourceSiteURL: auction.Source.SiteURL,
SourceSiteName: auction.Source.Name,
SourceURL: auction.Source.AuctionURL,
Country: auction.Address.CountryCode,
Province: auction.Address.State,
},
})
if err != nil {
kernel.ErrorLog.Printf("could not import auction: %w", err)
continue
}
}
}

11
src/catalog/main.go
View File

@ -8,6 +8,7 @@ import (
"git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal" "git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal"
"git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal/data" "git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal/data"
"git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal/data/kafka"
"git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal/data/postgres" "git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal/data/postgres"
"git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal/domain" "git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal/domain"
"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel" "git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
@ -80,7 +81,15 @@ func (app *catalogApp) Start(ctx context.Context) error {
return err return err
} }
return ioc.Invoke(func(d *domain.Usecase) error { if err = ioc.Provide(func() (internal.KafkaConsumer, error) {
return kafka.NewConsumer("runner.sync_results")
}); err != nil {
return err
}
return ioc.Invoke(func(d *domain.Usecase, consumer internal.KafkaConsumer) error {
go internal.RunIndexer(ctx, consumer, d)
catalogService := internal.NewCatalogServer(d) catalogService := internal.NewCatalogServer(d)
if _, err := kernel.StartGRPCServer(ctx, app.Port, catalogService); err != nil { if _, err := kernel.StartGRPCServer(ctx, app.Port, catalogService); err != nil {

22
src/runner/config.yaml
View File

@ -1,3 +1,21 @@
log_level: ERROR log_level: 2
port: 5001
service: {} kafka_bootstrap_servers:
- kafka
db_service:
scheme: postgres
port: 5432
host: bh-db
name: bh
user: runner-service
password: runner-service
db_migrate:
scheme: postgres
port: 5432
host: bh-db
name: bh
user: postgres
password: bh-admin

5
src/runner/go.mod
View File

@ -18,6 +18,10 @@ require (
github.com/ilyakaznacheev/cleanenv v1.4.2 // indirect github.com/ilyakaznacheev/cleanenv v1.4.2 // indirect
github.com/jackc/puddle v1.3.0 // indirect github.com/jackc/puddle v1.3.0 // indirect
github.com/joho/godotenv v1.5.1 // indirect github.com/joho/godotenv v1.5.1 // indirect
github.com/klauspost/compress v1.16.4 // indirect
github.com/moby/patternmatcher v0.5.0 // indirect
github.com/moby/sys/sequential v0.5.0 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pressly/goose/v3 v3.11.0 // indirect github.com/pressly/goose/v3 v3.11.0 // indirect
golang.org/x/net v0.9.0 // indirect golang.org/x/net v0.9.0 // indirect
golang.org/x/sys v0.8.0 // indirect golang.org/x/sys v0.8.0 // indirect
@ -34,6 +38,7 @@ require (
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgtype v1.14.0 // indirect github.com/jackc/pgtype v1.14.0 // indirect
github.com/jackc/pgx/v4 v4.18.1 github.com/jackc/pgx/v4 v4.18.1
github.com/segmentio/kafka-go v0.4.40
go.uber.org/automaxprocs v1.5.2 // indirect go.uber.org/automaxprocs v1.5.2 // indirect
golang.org/x/crypto v0.8.0 // indirect golang.org/x/crypto v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect golang.org/x/text v0.9.0 // indirect

2352
src/runner/go.sum
View File

File diff suppressed because it is too large Load Diff

55
src/runner/internal/data/kafka/producer.go Normal file
View File

@ -0,0 +1,55 @@
package kafka
import (
"context"
"encoding/json"
"fmt"
"time"
kafka "github.com/segmentio/kafka-go"
)
type kafkaProducer struct {
*kafka.Conn
}
func (c *kafkaProducer) Close() error {
return c.Conn.Close()
}
func (c *kafkaProducer) SendMessageJSON(headers map[string]string, msg interface{}) (err error) {
var data []byte
if data, err = json.Marshal(msg); err != nil {
err = fmt.Errorf("could not marshal message into JSON: %w", err)
return
}
h := []kafka.Header{}
for k, v := range headers {
h = append(h, kafka.Header{
Key: k,
Value: []byte(v),
})
}
if _, err = c.WriteMessages(kafka.Message{
Time: time.Now().UTC(),
Headers: h,
Value: data,
}); err != nil {
return
}
return
}
func NewProducer(topic string) (*kafkaProducer, error) {
conn, err := kafka.DialLeader(context.Background(), "tcp", "kafka:9092", topic, 0)
if err != nil {
return nil, err
}
return &kafkaProducer{
Conn: conn,
}, nil
}

4
src/runner/internal/data/storage.go
View File

@ -15,6 +15,10 @@ type PGRunnerStorage struct {
} }
func (db *PGRunnerStorage) CreateScrapeJob(ctx context.Context, target string) (sj domain.ScrapeJob, err error) { func (db *PGRunnerStorage) CreateScrapeJob(ctx context.Context, target string) (sj domain.ScrapeJob, err error) {
if target == "" {
target = "ALL"
}
rsj, err := db.Queries.CreateScrapeJob(ctx, target) rsj, err := db.Queries.CreateScrapeJob(ctx, target)
if err != nil { if err != nil {
return domain.ScrapeJob{}, err return domain.ScrapeJob{}, err

30
src/runner/internal/domain/liveauctioneers/upcoming.go
View File

@ -199,16 +199,26 @@ func LAGetSaleInfo(ctx context.Context, catIDs LACatalogIDs) (results []catalog.
results = make([]catalog.Auction, len(apiResults.Data.Catalogs)) results = make([]catalog.Auction, len(apiResults.Data.Catalogs))
for idx, c := range apiResults.Data.Catalogs { for idx, c := range apiResults.Data.Catalogs {
results[idx] = catalog.Auction{ results[idx] = catalog.Auction{
Title: c.Title, Title: c.Title,
Description: c.Description, Description: c.Description,
SourceSiteURL: "https://www.liveauctioneers.com", Start: time.Unix(c.SaleStartTS, 0),
SourceSiteName: "Live Auctioneers", End: time.Unix(c.SaleStartTS, 0).Add(time.Hour * 8),
SourceURL: fmt.Sprintf("https://www.liveauctioneers.com/catalog/%d", c.ID), ItemCount: c.ItemCount,
Start: time.Unix(c.SaleStartTS, 0), Source: catalog.SourceDetail{
End: time.Unix(c.SaleStartTS, 0).Add(time.Hour * 8), SiteURL: "https://www.liveauctioneers.com",
ItemCount: c.ItemCount, Name: "Live Auctioneers",
Country: c.Address.CountryCode, AuctionURL: fmt.Sprintf("https://www.liveauctioneers.com/catalog/%d", c.ID),
Province: c.Address.City, },
Address: catalog.AddressDetail{
CountryCode: c.Address.CountryCode,
City: c.Address.City,
State: c.Address.State,
Lat: c.Address.Lat,
Long: c.Address.Long,
},
Seller: catalog.SellerDetail{
ID: int(c.SellerID),
},
} }
} }

53
src/runner/internal/domain/logic.go
View File

@ -3,11 +3,13 @@ package domain
import ( import (
"context" "context"
"fmt" "fmt"
"io"
"strings" "strings"
"time" "time"
catalog "git.vdhsn.com/barretthousen/barretthousen/src/catalog/api" catalog "git.vdhsn.com/barretthousen/barretthousen/src/catalog/api"
"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel" "git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
"git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/data/kafka"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
@ -30,7 +32,6 @@ func RegisterAuctionFinder(finder UpcomingAuctionFinder) {
type ( type (
Domain struct { Domain struct {
Storage Storage
CatalogService
} }
CompleteScrapeJobStatus struct { CompleteScrapeJobStatus struct {
@ -44,8 +45,9 @@ type (
GetJobs(context.Context) ([]ScrapeJob, error) GetJobs(context.Context) ([]ScrapeJob, error)
} }
CatalogService interface { KafkaProducer interface {
UpdateUpcomingAuction(context.Context, catalog.Auction) (catalog.AuctionCreatedEvent, error) io.Closer
SendMessageJSON(map[string]string, interface{}) error
} }
FindNewUpcomingInput struct { FindNewUpcomingInput struct {
@ -60,17 +62,23 @@ type (
func (domain Domain) StartSync(ctx context.Context, in FindNewUpcomingInput) (out FindNewUpcomingOutput, err error) { func (domain Domain) StartSync(ctx context.Context, in FindNewUpcomingInput) (out FindNewUpcomingOutput, err error) {
kernel.TraceLog.Printf("%+v", in) kernel.TraceLog.Printf("%+v", in)
finder := targetsImpls["liveauctioneers"] runAllFinders := in.TargetSite == "" || in.TargetSite == "all"
for targetSite, finder := range targetsImpls {
if !runAllFinders && in.TargetSite != targetSite {
continue
}
if out.Job, err = domain.Storage.CreateScrapeJob(ctx, in.TargetSite); err != nil { if out.Job, err = domain.Storage.CreateScrapeJob(ctx, targetSite); err != nil {
err = fmt.Errorf("could not create new scrape job record: %w", err) err = fmt.Errorf("could not create new scrape job record: %w", err)
return return
}
kernel.InfoLog.Printf("Scrape Job %d starting", out.Job.ID)
// TODO: make everything after this line async and run after return
go domain.executeScrapeJob(finder, out.Job.ID)
} }
kernel.InfoLog.Printf("Scrape Job %d starting", out.Job.ID)
// TODO: make everything after this line async and run after return
go domain.executeScrapeJob(finder, out.Job.ID)
return return
} }
@ -99,7 +107,7 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
defer cancel() defer cancel()
found := make(chan catalog.Auction, 2048) found := make(chan catalog.Auction, 2048)
deadlineCtx, deadlineCancel := context.WithDeadline(ctx, time.Now().Add(time.Minute)) deadlineCtx, deadlineCancel := context.WithTimeout(ctx, time.Minute*5)
defer deadlineCancel() defer deadlineCancel()
errGroup, innerCtx := errgroup.WithContext(deadlineCtx) errGroup, innerCtx := errgroup.WithContext(deadlineCtx)
@ -107,6 +115,13 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
return finder.Find(innerCtx, 0, found) return finder.Find(innerCtx, 0, found)
}) })
var p KafkaProducer
var err error
if p, err = kafka.NewProducer("runner.sync_results"); err != nil {
return
}
defer p.Close()
count := 0 count := 0
total := 0 total := 0
errs := &strings.Builder{} errs := &strings.Builder{}
@ -116,16 +131,15 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
continue continue
} }
ace, err := domain.CatalogService.UpdateUpcomingAuction(ctx, auction) if err := p.SendMessageJSON(map[string]string{
if err != nil { "scrape-job-id": fmt.Sprintf("%d", jobID),
kernel.TraceLog.Printf("could not import upcoming auction: %s", err.Error()) "target-site": finder.String(),
fmt.Fprintf(errs, "{ \"AuctionFingerprint\": \"%s\", \"error\": \"%s\" }\n", ace.Fingerprint, err.Error()) }, auction); err != nil {
kernel.TraceLog.Printf("could not publish auction to kafka: %v", err)
continue continue
} }
if !ace.Duplicate { count++
count++
}
} }
if err := errGroup.Wait(); err != nil { if err := errGroup.Wait(); err != nil {
@ -134,7 +148,6 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
} }
var completedJob ScrapeJob var completedJob ScrapeJob
var err error
if completedJob, err = domain.Storage.CompleteScrapeJob(ctx, jobID, CompleteScrapeJobStatus{ if completedJob, err = domain.Storage.CompleteScrapeJob(ctx, jobID, CompleteScrapeJobStatus{
AuctionCount: count, AuctionCount: count,
Errors: errs.String(), Errors: errs.String(),

11
src/runner/internal/handler.go
View File

@ -34,7 +34,7 @@ func (rh *runnerHandler) StartSync(ctx context.Context, cmd *api.SyncParameters)
Id: int32(out.Job.ID), Id: int32(out.Job.ID),
AuctionsFound: int32(out.Job.AuctionsFound), AuctionsFound: int32(out.Job.AuctionsFound),
CreatedTs: timestamppb.New(out.Job.Started), CreatedTs: timestamppb.New(out.Job.Started),
CompletedTs: timestamppb.New(out.Job.Completed), CompletedTs: nil,
TargetSiteName: out.Job.TargetSite, TargetSiteName: out.Job.TargetSite,
Errors: out.Job.Errors, Errors: out.Job.Errors,
}, nil }, nil
@ -51,14 +51,21 @@ func (rh *runnerHandler) Status(ctx context.Context, cmd *api.StatusFilter) (*ap
} }
for _, j := range out.Jobs { for _, j := range out.Jobs {
var completedTime *timestamppb.Timestamp
if !j.Completed.IsZero() {
completedTime = timestamppb.New(j.Completed)
}
result.Results = append(result.Results, &api.SyncStatus{ result.Results = append(result.Results, &api.SyncStatus{
Id: int32(j.ID), Id: int32(j.ID),
AuctionsFound: int32(j.AuctionsFound), AuctionsFound: int32(j.AuctionsFound),
CreatedTs: timestamppb.New(j.Started), CreatedTs: timestamppb.New(j.Started),
CompletedTs: timestamppb.New(j.Completed), CompletedTs: completedTime,
TargetSiteName: j.TargetSite, TargetSiteName: j.TargetSite,
Errors: j.Errors, Errors: j.Errors,
}) })
} }
return result, nil return result, nil

12
src/runner/main.go
View File

@ -6,7 +6,6 @@ import (
"flag" "flag"
"fmt" "fmt"
capi "git.vdhsn.com/barretthousen/barretthousen/src/catalog/api"
"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel" "git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
"git.vdhsn.com/barretthousen/barretthousen/src/runner/internal" "git.vdhsn.com/barretthousen/barretthousen/src/runner/internal"
"git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/data" "git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/data"
@ -93,16 +92,9 @@ func (app *runnerApp) Start(ctx context.Context) error {
return err return err
} }
if err = ioc.Provide(func(conn grpc.ClientConnInterface) domain.CatalogService { if err = ioc.Provide(func(rs domain.Storage) *domain.Domain {
return capi.NewCatalogServiceClient(conn)
}); err != nil {
return err
}
if err = ioc.Provide(func(css domain.CatalogService, rs domain.Storage) *domain.Domain {
return &domain.Domain{ return &domain.Domain{
Storage: rs, Storage: rs,
CatalogService: css,
} }
}); err != nil { }); err != nil {
return err return err