Barretthousen
/
barretthousen
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

paging broken

pull/5/head
Adam Veldhousen 2023-05-16 18:52:07 -05:00
parent d504ad8b47
commit faec39add8
Signed by: adam
GPG Key ID: 6DB29003C6DD1E4B
16 changed files with 136 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,30 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Delve into Runner",
"type": "go",
"request": "attach",
"mode": "remote",
"port": 2345,
"host": "127.0.0.1",
"trace": "verbose",
// "remotePath": "/go",
"cwd": "${workspaceFolder}",
},
{
"name": "Delve into Catalog",
"type": "go",
"request": "attach",
"mode": "remote",
"port": 2346,
"host": "127.0.0.1",
"trace": "verbose",
"cwd": "${workspaceFolder}",
// "remotePath": "/go"
}
]
}

12
Tiltfile
View File

@ -46,7 +46,7 @@ helm_resource(
def bh_backend_service(service="", port_forwards=[], migrateDB=False, devMode=True, labels=['2-services'], deps=['postgres']):
local_resource(
'{}-go-compile'.format(service),
'CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -gcflags=\'-N -l\' -o .bin/{}-debug ./src/{}'.format(service, service),
'CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -gcflags "all=-N -l" -o .bin/{}-debug ./src/{}'.format(service, service),
deps=['./src/{}'.format(service), './src/lib'],
resource_deps=deps
)
@ -79,10 +79,16 @@ def bh_backend_service(service="", port_forwards=[], migrateDB=False, devMode=Tr
'./.bin/{}-debug'.format(service)
],
only = [
'./.bin/{}-debug'.format(service)
'./.bin/{}-debug'.format(service),
'./src/runner',
'./src/catalog',
'./src/proxy-admin',
'./src/proxy-client',
'./src/lib'
],
live_update=[
sync('./.bin', '/opt')
sync('./.bin', '/opt'),
sync('./src', '/go/src')
]
)

2
env/base/runner-deployment.yaml vendored
View File

@ -15,6 +15,8 @@ spec:
containers:
- name: runner
image: barretthousen/service-runner
tty: true
stdin: true
ports:
- containerPort: 5001
name: grpc

7
src/Dockerfile.dev-backend
View File

@ -1,13 +1,16 @@
FROM golang:1.19-alpine as builder
RUN go install github.com/go-delve/delve/cmd/dlv@latest
ARG service
RUN go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest
FROM alpine as development
ARG service
COPY --from=builder /go/bin/dlv /go/bin/dlv
COPY .bin/${service}-debug /opt/${service}-debug
COPY .bin/${service}-debug /opt/
COPY ./src /go/src/
ENV SERVICE=${service}

2
src/Dockerfile.service
View File

@ -14,7 +14,7 @@ RUN /go/bin/buf mod update /go/src/src \
&& /go/bin/buf generate
RUN go mod tidy \
&& CGO_ENABLED=0 go build -v -o /opt/${service} /go/src/${service}
&& CGO_ENABLED=0 go build -v -gcflags="-trimpath=$(go env GOPATH)" -asmflags="-trimpath=$(go env GOPATH)" -o /opt/${service} /go/src/${service}
FROM alpine as production

16
src/catalog/api/client.go
View File

@ -33,7 +33,13 @@ type Auction struct {
End time.Time `json:"end,omitempty"`
}
func (css *CatalogServiceClient) UpdateUpcomingAuction(ctx context.Context, a Auction) (string, error) {
type AuctionCreatedEvent struct {
Fingerprint string
ID int
Duplicate bool
}
func (css *CatalogServiceClient) UpdateUpcomingAuction(ctx context.Context, a Auction) (AuctionCreatedEvent, error) {
ac, err := css.ImportAuction(ctx, &capi.ImportAuctionMessage{
Items: int32(a.ItemCount),
Start: timestamppb.New(a.Start),
@ -47,8 +53,12 @@ func (css *CatalogServiceClient) UpdateUpcomingAuction(ctx context.Context, a Au
Province: a.Province,
})
if err != nil {
return "", err
return AuctionCreatedEvent{}, err
}
return ac.Auction.GetFingerprint(), nil
return AuctionCreatedEvent{
Fingerprint: ac.Auction.Fingerprint,
ID: int(ac.Auction.Id),
Duplicate: ac.Duplicate,
}, nil
}

7
src/catalog/internal/data/postgres/queries.sql
View File

@ -4,9 +4,10 @@ FROM catalog.upcoming_auctions ua
LEFT JOIN catalog.upcoming_auctions_fts fts on ua.id = fts.auctionid
WHERE
ua.endts >= DATE(NOW()) AND
fts.ts @@ phraseto_tsquery($1)
ORDER BY ts_rank(fts.ts, phraseto_tsquery($1)) DESC
LIMIT $2;
fts.ts @@ phraseto_tsquery(sqlc.arg(searchTerm))
ORDER BY ts_rank(fts.ts, phraseto_tsquery(sqlc.arg(searchTerm))) DESC
OFFSET sqlc.arg(page) * sqlc.arg(pageSize)
LIMIT sqlc.arg(pageSize);
-- name: GetTotal :one
SELECT COUNT(*) FROM catalog.upcoming_auctions;

9
src/catalog/internal/data/storage.go
View File

@ -8,6 +8,7 @@ import (
"git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal/data/postgres"
"git.vdhsn.com/barretthousen/barretthousen/src/catalog/internal/domain"
"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
)
type PGCatalogStorage struct {
@ -18,8 +19,9 @@ func (ps *PGCatalogStorage) GetUpcoming(ctx context.Context, q domain.UpcomingQu
var pgResults []postgres.CatalogUpcomingAuction
if pgResults, err = ps.Queries.GetUpcoming(ctx, postgres.GetUpcomingParams{
PhrasetoTsquery: q.Term,
Limit: 64,
Searchterm: q.Term,
Page: q.Page,
Pagesize: 64,
}); err != nil {
err = fmt.Errorf("could not get upcoming auctions from pg: %w", err)
return
@ -39,7 +41,7 @@ func (ps *PGCatalogStorage) GetUpcoming(ctx context.Context, q domain.UpcomingQu
Description: row.Description,
SourceSiteURL: row.Sourcesiteurl,
SourceSiteName: row.Sourcesitename,
SourceURL: row.Sourcesiteurl,
SourceURL: row.Sourceurl,
Country: row.Country,
Province: row.Province,
ItemCount: int(row.Itemcount),
@ -59,6 +61,7 @@ func (ps *PGCatalogStorage) CreateUpcoming(ctx context.Context, a domain.Auction
var auctionID int32
fingerprint = fmt.Sprintf("%x", sha512.Sum512(append([]byte(a.Title), []byte(a.Description)...)))
kernel.TraceLog.Printf("%s: %s + %s", fingerprint, a.Title, a.Description)
if auctionID, err = ps.Queries.ImportAuction(ctx, postgres.ImportAuctionParams{
Fingerprint: fingerprint,
Title: a.Title,

1
src/catalog/internal/domain/logic.go
View File

@ -23,6 +23,7 @@ type (
UpcomingQuery struct {
Term string
Page int
StartDateFilter time.Time
}

1
src/catalog/internal/handler.go
View File

@ -33,6 +33,7 @@ func (rh *catalogHandler) GetUpcoming(ctx context.Context, cmd *api.AuctionSearc
var results domain.UpcomingResults
if results, err = rh.domain.GetUpcoming(ctx, domain.UpcomingQuery{
Term: cmd.SearchTerm,
Page: int(cmd.GetPage()),
StartDateFilter: time.Unix(cmd.StartBeforeTs, 0),
}); err != nil {
err = fmt.Errorf("could not get upcoming items from domain: %w", err)

2
src/runner/internal/domain/liveauctioneers/upcoming.go
View File

@ -79,7 +79,7 @@ func LAGetUpcomingSaleIDs(ctx context.Context, in GetUpcomingSaleIDsInput) (ids
ctx,
http.MethodGet,
fmt.Sprintf(
"https://search-party-prod.liveauctioneers.com/search/catalogsearch?c=20170802&client=web&client_version=5.0.0&excludedHouses=[]&max_facet_values=0&offset=%d&sort=saleStart&pageSize=%d",
"https://search-party-prod.liveauctioneers.com/search/catalogsearch?page=%d&sort=saleStart&pageSize=%d",
in.Page,
in.Limit,
),

23
src/runner/internal/domain/logic.go
View File

@ -45,7 +45,7 @@ type (
}
CatalogService interface {
UpdateUpcomingAuction(context.Context, catalog.Auction) (string, error)
UpdateUpcomingAuction(context.Context, catalog.Auction) (catalog.AuctionCreatedEvent, error)
}
FindNewUpcomingInput struct {
@ -96,7 +96,7 @@ func (domain Domain) Status(ctx context.Context, in GetJobsInput) (out GetJobsOu
func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int) {
ctx, cancel := context.WithDeadline(context.TODO(), time.Now().Add(time.Minute))
defer cancel()
found := make(chan catalog.Auction)
found := make(chan catalog.Auction, 2048)
errGroup, innerCtx := errgroup.WithContext(ctx)
errGroup.Go(func() error {
@ -104,12 +104,23 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
})
count := 0
total := 0
errs := &strings.Builder{}
for auction := range found {
count++
if fingerprint, err := domain.CatalogService.UpdateUpcomingAuction(ctx, auction); err != nil {
total++
if !auction.Start.After(time.Now()) {
continue
}
ace, err := domain.CatalogService.UpdateUpcomingAuction(ctx, auction)
if err != nil {
kernel.TraceLog.Printf("could not import upcoming auction: %s", err.Error())
fmt.Fprintf(errs, "{ \"AuctionFingerprint\": \"%s\", \"error\": \"%s\" }\n", fingerprint, err.Error())
fmt.Fprintf(errs, "{ \"AuctionFingerprint\": \"%s\", \"error\": \"%s\" }\n", ace.Fingerprint, err.Error())
continue
}
if !ace.Duplicate {
count++
}
}
@ -127,6 +138,6 @@ func (domain *Domain) executeScrapeJob(finder UpcomingAuctionFinder, jobID int)
kernel.ErrorLog.Printf("Could not complete scrape job, failing: %w", err)
}
kernel.InfoLog.Printf("Scrape Job %d completed in %v.", jobID, completedJob.Completed.Sub(completedJob.Started))
kernel.InfoLog.Printf("Scrape Job %d completed in %v. Successfully imported %d/%d", jobID, completedJob.Completed.Sub(completedJob.Started), count, total)
return
}

4
src/web-client/src/lib/AuctionResult.svelte
View File

@ -42,8 +42,8 @@
</header>
<section>
<ul>
<li class="text-sm">Starts {saleStartTime.toLocaleString(DateTime.DATETIME_FULL)}</li>
<li class="text-sm">{province}, {country.toUpperCase()}</li>
<li class="text-sm">Starts {saleStartTime?.toLocaleString(DateTime.DATETIME_FULL)}</li>
<li class="text-sm">{province}, {country?.toUpperCase()}</li>
<li class="text-sm">{items} Items</li>
<li class="text-sm">
Available at <a href={sourceSiteURL} target="_blank">{sourceSiteName}</a>

16
src/web-client/src/routes/+layout.svelte
View File

@ -4,14 +4,14 @@
import '../app.css';
function onSubmit(evt: CustomEvent) {
const { query } = evt.detail;
goto(query ? `/?query=${query}` : '/');
const { query, page } = evt.detail;
goto(query ? `/?query=${query}&page=${page || 0}` : '/');
}
</script>
<main class="w-full">
<main class="flex flex-col w-full">
<nav
class="flex w-full bg-bh-gold text-bh-black fixed top-0 left-0 right-0 shadow-bh-black shadow-lg"
class="flex w-full bg-bh-gold text-bh-black fixed top-0 left-0 right-0"
style="height: 80px;"
>
<img
@ -31,14 +31,14 @@
</li>
</ul>
</nav>
<div class="flex flex-row pt-16">
<div id="gutter" class="h-full" style="min-width: 195px;" />
<section id="content" class="pt-6">
<div class="flex flex-row flex-grow h-full" style="padding-top: 80px;">
<div id="gutter" class="h-full" style="min-width: 195px; max-width: 195px;" />
<section id="content" class="py-6 pl-4 grow border-l border-bh-gold" style="min-height: 100vh;">
<slot />
</section>
</div>
<footer
class="flex justify-center bg-bh-gold text-bh-black w-full mt-20 py-10 shadow-inner shadnow-md shadow-bh-black"
class="flex justify-center bg-bh-gold text-bh-black w-full py-10 shadow-inner shadnow-md shadow-bh-black"
>
<p>
We find great auctions of rare and one of a kind collectibles from around the web and compile

29
src/web-client/src/routes/+page.svelte
View File

@ -3,14 +3,27 @@
import AuctionResult from '$lib/AuctionResult.svelte';
export let data: PageData;
const hasResults = (data?.results || []).length > 0;
</script>
<h1 class="pb-5 text-lg">{data?.total} Upcoming & Live Auctions</h1>
{#if !hasResults}
<section class="flex w-full flex-col justify-center text-center">
<h1 class="text-2xl">No auctions found.</h1>
{#if data?.query !== 'watch'}
<p>Try searching <em><a href="/?query=watch">Watch</a></em></p>
{/if}
</section>
{/if}
<ul class="flex flex-col m-0 p-0 justify-between pr-10">
{#each data.results as auction, i}
<li id="catalog-{auction.id}" class="pb-5">
<AuctionResult {auction} />
</li>
{/each}
</ul>
{#if hasResults}
<h1 class="pb-5 text-lg">{data?.results.length} of {data?.total} Upcoming & Live Auctions</h1>
<ul class="flex flex-col m-0 p-0 justify-between pr-10">
{#each data?.results as auction, i}
<li id="catalog-{auction.id}" class="pb-5">
<AuctionResult {auction} />
</li>
{/each}
</ul>
{/if}

19
src/web-client/src/routes/+page.ts
View File

@ -3,22 +3,33 @@ import type { PageLoad } from './$types';
const API_HOST = 'http://localhost:8000/api/v1'
export const load = (async ({ fetch, url }) => {
const searchTerm = url.searchParams.get('query') || '';
const currentPage = url.searchParams.get('page') || 1;
try {
const searchTerm = url.searchParams.get('query') || 'watch';
const response = await fetch(API_HOST + `/upcoming?searchTerm=${searchTerm}`);
const { page, total, results } = await response.json();
const response = await fetch(API_HOST + `/upcoming?searchTerm=${searchTerm}&page=${currentPage}`);
const { page, total, results } = await response.json() || {};
return {
pagination: {
page,
limit: 64,
},
query: searchTerm,
total,
results
};
} catch (e) {
console.error(e);
console.log(e);
return {
pagination: {
page: 0,
limit: 64,
},
query: searchTerm,
total: 0,
results: []
};
}
}) satisfies PageLoad;