add catawiki catalog sync, DX improvements, ci
ci.vdhsn.com/push Build was killed
Details
ci.vdhsn.com/push Build was killed
Details
- drone ci build using kaniko This builds images extremely slowly, I've implemented some custom fixes to the plugin to improve performance but it's still slow to build images. I will try to implement a DinD solution. There are issues in kaniko about this: https://github.com/GoogleContainerTools/kaniko/issues/875 Squashed commit of the following: committrunk1fd65ad139Author: Adam Veldhousen <adamveld12@gmail.com> Date: Sat Jul 15 18:17:55 2023 -0500 add deploy script commit075e15f218Author: Adam Veldhousen <adamveld12@gmail.com> Date: Sat Jul 15 17:53:24 2023 -0500 bump limits commit2d45234e7bAuthor: Adam Veldhousen <adamveld12@gmail.com> Date: Sat Jul 15 17:51:19 2023 -0500 sync button in tilt, support syncing all targets simultaneously commit75e73a1171Author: Adam Veldhousen <adamveld12@gmail.com> Date: Sat Jul 15 17:03:50 2023 -0500 create admin user on startup commit40cf74560dAuthor: Adam Veldhousen <adamveld12@gmail.com> Date: Fri Jul 14 11:14:00 2023 -0500 test more cache commit1fbbfe7548Author: Adam Veldhousen <adamveld12@gmail.com> Date: Fri Jul 14 11:11:34 2023 -0500 test tmpfs commit0f48c098d9Author: Adam Veldhousen <adamveld12@gmail.com> Date: Fri Jul 14 11:08:32 2023 -0500 add tmpfs commitd37142cfefAuthor: Adam Veldhousen <adamveld12@gmail.com> Date: Fri Jul 14 11:02:39 2023 -0500 crank verbosity commit178eb5bc38Author: Adam Veldhousen <adamveld12@gmail.com> Date: Fri Jul 14 10:44:08 2023 -0500 bump requests commitb0aa0b39f0Author: Adam Veldhousen <adamveld12@gmail.com> Date: Fri Jul 14 10:40:04 2023 -0500 test fewer concurrent builds commit221e378a53Author: Adam Veldhousen <adamveld12@gmail.com> Date: Thu Jul 13 20:40:18 2023 -0500 test commit6c03914341Author: Adam Veldhousen <adamveld12@gmail.com> Date: Thu Jul 13 19:01:22 2023 -0500 snapshot mode for faster build commit91c56d430eAuthor: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 22:40:12 2023 -0500 try script commit4a119c2523Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 22:28:41 2023 -0500 test promote commitb1062a4001Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 22:26:58 2023 -0500 test commitc06804dfe6Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 17:47:48 2023 -0500 beta promote and ignore commit643e104c37Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 16:21:39 2023 -0500 tweakies commit478e65ac70Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 16:09:54 2023 -0500 un woops commit966d48c193Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 15:23:08 2023 -0500 woops commit2829743d17Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 15:22:03 2023 -0500 test internal routing commite08e68b8adAuthor: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 14:07:31 2023 -0500 push images commit5705bca068Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 13:40:48 2023 -0500 ugh commit1699b08d5fAuthor: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 13:39:49 2023 -0500 fixup client builds commit9ec8616ae1Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 13:36:36 2023 -0500 update go mods commit163b25fd67Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 02:11:02 2023 -0500 fix fronend builds commite5cc7d5c79Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 01:56:08 2023 -0500 resources commit8088b95c1eAuthor: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 01:38:21 2023 -0500 big test commit38fef91c49Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 01:26:09 2023 -0500 testicles commit53baa0a647Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 01:24:50 2023 -0500 testicles commit16901ec5bbAuthor: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 01:23:38 2023 -0500 testicles commit3478d4f512Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 01:19:58 2023 -0500 test 2 commit16298ada78Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 01:13:53 2023 -0500 more test commit80524a1d06Author: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 01:09:03 2023 -0500 test commitb9916204deAuthor: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 00:19:41 2023 -0500 tools image commit6e414b3a1bAuthor: Adam Veldhousen <adamveld12@gmail.com> Date: Wed Jul 12 00:00:23 2023 -0500 test commitabb6183e88Author: Adam Veldhousen <adamveld12@gmail.com> Date: Tue Jul 11 22:55:58 2023 -0500 added catawiki scrape commit2643817e93Author: Adam Veldhousen <adamveld12@gmail.com> Date: Tue Jul 11 14:40:05 2023 -0500 add catawiki syncer
parent
389c6ed5e8
commit
70d300df5e
GPG Key ID:
6DB29003C6DD1E4B
@ -1,10 +1,10 @@
|
||||
go 1.19
|
||||
|
||||
use (
|
||||
./src/lib
|
||||
./src/auth
|
||||
./src/catalog
|
||||
./src/runner
|
||||
./src/lib
|
||||
./src/proxy-admin
|
||||
./src/proxy-web
|
||||
./src/runner
|
||||
)
|
||||
|
||||
@ -0,0 +1,64 @@
|
||||
#!/bin/bash
|
||||
|
||||
|
||||
build(){
|
||||
export ENV=${1:-beta}; # if not specified, will only build and push for the git commit
|
||||
make build-backend-image SERVICE=auth &
|
||||
make build-backend-image SERVICE=catalog &
|
||||
make build-backend-image SERVICE=runner &
|
||||
make build-backend-image SERVICE=proxy-admin &
|
||||
make build-backend-image SERVICE=proxy-web &
|
||||
time wait;
|
||||
|
||||
# for prod drop `beta prefix
|
||||
make build-client-image SERVICE=web-client &
|
||||
make build-client-image SERVICE=admin-client &
|
||||
time wait;
|
||||
|
||||
kubectl rollout restart -n barretthousen-beta deployment runner-beta &
|
||||
kubectl rollout restart -n barretthousen-beta deployment catalog-beta &
|
||||
kubectl rollout restart -n barretthousen-beta deployment auth-beta &
|
||||
kubectl rollout restart -n barretthousen-beta deployment proxy-admin-beta &
|
||||
kubectl rollout restart -n barretthousen-beta deployment proxy-web-beta &
|
||||
kubectl rollout restart -n barretthousen-beta deployment web-client-beta &
|
||||
kubectl rollout restart -n barretthousen-beta deployment admin-client-beta &
|
||||
|
||||
kubectl rollout status -n barretthousen-beta deployment runner-beta -w
|
||||
kubectl rollout status -n barretthousen-beta deployment catalog-beta -w
|
||||
kubectl rollout status -n barretthousen-beta deployment auth-beta -w
|
||||
kubectl rollout status -n barretthousen-beta deployment proxy-admin-beta -w
|
||||
kubectl rollout status -n barretthousen-beta deployment proxy-web-beta -w
|
||||
kubectl rollout status -n barretthousen-beta deployment admin-client-beta -w
|
||||
kubectl rollout status -n barretthousen-beta deployment web-client-beta -w
|
||||
}
|
||||
|
||||
promote() {
|
||||
make promote-backend-prod SERVICE=catalog &
|
||||
make promote-backend-prod SERVICE=runner &
|
||||
make promote-backend-prod SERVICE=auth &
|
||||
make promote-backend-prod SERVICE=proxy-admin &
|
||||
make promote-backend-prod SERVICE=proxy-web &
|
||||
|
||||
make promote-client-prod SERVICE=web-client &
|
||||
make promote-client-prod SERVICE=admin-client &
|
||||
|
||||
time wait;
|
||||
|
||||
kubectl rollout restart -n barretthousen deployment runner &
|
||||
kubectl rollout restart -n barretthousen deployment catalog &
|
||||
kubectl rollout restart -n barretthousen deployment auth &
|
||||
kubectl rollout restart -n barretthousen deployment proxy-admin &
|
||||
kubectl rollout restart -n barretthousen deployment proxy-web &
|
||||
kubectl rollout restart -n barretthousen deployment web-client &
|
||||
kubectl rollout restart -n barretthousen deployment admin-client &
|
||||
|
||||
kubectl rollout status -n barretthousen deployment runner -w
|
||||
kubectl rollout status -n barretthousen deployment catalog -w
|
||||
kubectl rollout status -n barretthousen deployment auth -w
|
||||
kubectl rollout status -n barretthousen deployment proxy-admin -w
|
||||
kubectl rollout status -n barretthousen deployment proxy-web -w
|
||||
kubectl rollout status -n barretthousen deployment admin-client -w
|
||||
kubectl rollout status -n barretthousen deployment web-client -w
|
||||
}
|
||||
|
||||
build $@
|
||||
@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
|
||||
|
||||
main(){
|
||||
local EMAIL=${1};
|
||||
if [ -z "${EMAIL}" ]; then
|
||||
echo "first argument must be an EMAIL address"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
local PASSWORD=${2};
|
||||
if [ -z "${PASSWORD}" ]; then
|
||||
echo "second argument must be a PASSWORD"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
local ROLE=${3:-USER};
|
||||
local HOST=${4:-"http://bh.localhost:8000"};
|
||||
|
||||
curl -X PUT -H 'Content-Type: application/json' \
|
||||
-d "{ \"email\":\"${EMAIL}\", \"password\":\"${PASSWORD}\", \"role\":\"${ROLE}\" }" \
|
||||
"${HOST}/api/v1/user";
|
||||
}
|
||||
|
||||
|
||||
main $@;
|
||||
@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
|
||||
docker run --rm \
|
||||
-e DRONE_COMMIT_REF=refs/heads/trunk \
|
||||
-e DRONE_REPO_BRANCH=trunk \
|
||||
-e PLUGIN_REGISTRY=git.vdhsn.com \
|
||||
-e PLUGIN_REPO=git.vdhsn.com/barretthousen/service-catalog \
|
||||
-e PLUGIN_BUILD_ARGS="service=catalog" \
|
||||
-e PLUGIN_TAGS=$(git rev-parse --verify --short HEAD) \
|
||||
-e PLUGIN_CONTEXT="./src" \
|
||||
-e PLUGIN_DOCKERFILE="./src/Dockerfile.prod-backend" \
|
||||
-e PLUGIN_USERNAME=${DOCKER_USERNAME} \
|
||||
-e PLUGIN_PASSWORD=${DOCKER_PASSWORD} \
|
||||
-e PLUGIN_ENABLE_CACHE=true \
|
||||
-e PLUGIN_CACHE_REPO='git.vdhsn.com/barretthousen/ci-cache' \
|
||||
-e PLUGIN_SKIP_UNUSED_STAGES=true \
|
||||
-v $(pwd):/drone \
|
||||
-w /drone \
|
||||
plugins/kaniko:linux-amd64
|
||||
@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
|
||||
|
||||
main(){
|
||||
local REPOSITORY=${1};
|
||||
if [ -z "${REPOSITORY}" ]; then
|
||||
echo "First argument must be container repository";
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
local FROM=${2:-beta};
|
||||
local TO=${3:-prod};
|
||||
|
||||
local CONTENT_TYPE="application/vnd.docker.distribution.manifest.v2+json";
|
||||
local REGISTRY_URL="https://git.vdhsn.com/v2/${REPOSITORY}/manifests";
|
||||
|
||||
export MANIFEST=$(curl -u "${DOCKER_USERNAME}:${DOCKER_PASSWORD}" \
|
||||
-H "Accept: ${CONTENT_TYPE}" "${REGISTRY_URL}/${FROM}");
|
||||
|
||||
|
||||
curl -u "${DOCKER_USERNAME}:${DOCKER_PASSWORD}" \
|
||||
-X PUT -H "Content-Type: ${CONTENT_TYPE}" -d "${MANIFEST}" "${REGISTRY_URL}/${TO}";
|
||||
|
||||
echo "";
|
||||
}
|
||||
|
||||
@ -1 +1,4 @@
|
||||
.idea
|
||||
web-client/*
|
||||
admin-client/*
|
||||
|
||||
|
||||
@ -0,0 +1,4 @@
|
||||
FROM golang:1.19
|
||||
|
||||
RUN go install github.com/bufbuild/buf/cmd/buf@v1.17.0
|
||||
RUN go install github.com/kyleconroy/sqlc/cmd/sqlc@latest
|
||||
@ -0,0 +1,158 @@
|
||||
package catawiki
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
catalog "git.vdhsn.com/barretthousen/barretthousen/src/catalog/api"
|
||||
"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
|
||||
"git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/domain"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
func init() {
|
||||
kernel.TraceLog.Println("Registering AuctionFinder catawiki")
|
||||
domain.RegisterAuctionFinder(
|
||||
CatawikiAuctionFinder("catawiki"),
|
||||
)
|
||||
}
|
||||
|
||||
type CatawikiAuctionFinder string
|
||||
|
||||
func (cw CatawikiAuctionFinder) String() string {
|
||||
return string(cw)
|
||||
}
|
||||
|
||||
func (cw CatawikiAuctionFinder) Find(ctx context.Context, limit int, results chan<- catalog.Auction) (err error) {
|
||||
defer close(results)
|
||||
|
||||
pageSize := 50
|
||||
|
||||
if limit > 2500 {
|
||||
limit = 2500
|
||||
}
|
||||
|
||||
if limit <= 0 {
|
||||
limit = 2500
|
||||
}
|
||||
|
||||
errg, errgCtx := errgroup.WithContext(ctx)
|
||||
errg.SetLimit(5)
|
||||
|
||||
kernel.TraceLog.Printf("[Catawiki] fetching %d pages or up to %d results in %d batches", limit/pageSize, limit, pageSize)
|
||||
for p := 1; p <= limit/pageSize; p++ {
|
||||
pageIdx := p
|
||||
errg.Go(func() error {
|
||||
auctionR, err := GetUpcomingAuctions(errgCtx, limit, pageIdx)
|
||||
if errors.Is(err, ErrNoResults) {
|
||||
kernel.TraceLog.Println("no more results from catawiki api")
|
||||
return err
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, r := range auctionR {
|
||||
select {
|
||||
case <-errgCtx.Done():
|
||||
kernel.TraceLog.Println("[Catawiki] group context exited early")
|
||||
return nil
|
||||
default:
|
||||
results <- r
|
||||
kernel.TraceLog.Printf("[Catawiki]: %+v", r)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err = errg.Wait(); err != nil {
|
||||
kernel.ErrorLog.Printf("[Catawiki] could not get results f: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var ErrNoResults = errors.New("No results found")
|
||||
|
||||
func GetUpcomingAuctions(ctx context.Context, limit int, page int) (results []catalog.Auction, err error) {
|
||||
if limit <= 25 {
|
||||
limit = 25
|
||||
} else if limit >= 50 {
|
||||
limit = 50
|
||||
}
|
||||
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("https://www.catawiki.com/buyer/api/v1/auctions?locale=en&=published_at_desc&per_page=%d&page=%d", limit, page)
|
||||
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Accept-Language", "en-US,en;q=0.5")
|
||||
req.Header.Add("User-Agent", "barretthousen.com Auction Search Engine")
|
||||
|
||||
kernel.TraceLog.Printf("[Catawiki] Loading page %d of %d results", page, limit)
|
||||
var res *http.Response
|
||||
if res, err = http.DefaultClient.Do(req); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
err = fmt.Errorf("bad response code: %d", res.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
type Payload struct {
|
||||
Auctions []struct {
|
||||
ID int `json:"id"`
|
||||
StartAt string `json:"start_at"`
|
||||
CloseAt string `json:"close_at"`
|
||||
LotCount int `json:"lot_count"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Status string `json:"status"`
|
||||
Sellers []struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"sellers"`
|
||||
} `json:"auctions"`
|
||||
}
|
||||
|
||||
var p Payload
|
||||
if err = json.NewDecoder(res.Body).Decode(&p); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if len(p.Auctions) <= 0 {
|
||||
err = ErrNoResults
|
||||
return
|
||||
}
|
||||
|
||||
results = make([]catalog.Auction, len(p.Auctions))
|
||||
for idx, auction := range p.Auctions {
|
||||
results[idx] = catalog.Auction{
|
||||
Title: auction.Title,
|
||||
Description: "",
|
||||
SourceSiteURL: "https://www.catawiki.com",
|
||||
SourceSiteName: "CataWiki",
|
||||
SourceURL: auction.URL,
|
||||
Country: "",
|
||||
Province: "",
|
||||
ItemCount: auction.LotCount,
|
||||
}
|
||||
|
||||
results[idx].Start, _ = time.Parse(time.RFC3339, auction.StartAt)
|
||||
results[idx].End, _ = time.Parse(time.RFC3339, auction.CloseAt)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
package catawiki
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
catalog "git.vdhsn.com/barretthousen/barretthousen/src/catalog/api"
|
||||
)
|
||||
|
||||
func TestGetUpcomingAuctions(t *testing.T) {
|
||||
type args struct {
|
||||
limit int
|
||||
page int
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
wantResults []catalog.Auction
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "1 Page, 25 Results",
|
||||
args: args{
|
||||
limit: 25,
|
||||
page: 1,
|
||||
},
|
||||
wantResults: make([]catalog.Auction, 25),
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotResults, err := GetUpcomingAuctions(context.TODO(), tt.args.limit, tt.args.page)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("GetUpcomingAuctions() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(gotResults, tt.wantResults) {
|
||||
t.Errorf("GetUpcomingAuctions() = %v, want %v", gotResults, tt.wantResults)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue