Barretthousen
/
barretthousen
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

Merge remote-tracking branch 'origin/feat/mvp' into feat/mvp-deploy-beta 

* origin/feat/mvp:
  pagination and linking improvements
  use 1 table for catalog data + search
  re arrange some things
pull/7/head
Adam Veldhousen 2023-05-26 19:22:13 -05:00
parent ebfcccabfe e945fc2f79
commit 2e0e78ae28
Signed by: adam
GPG Key ID: 6DB29003C6DD1E4B
7 changed files with 98 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
Tiltfile
View File

@ -28,6 +28,14 @@ helm_resource(
labels=["1-ingress"]
)
k8s_resource(
workload='ingress',
labels='1-ingress',
links=[
link(url='http://admin.localhost:8000', name='Admin Panel'),
]
)
helm_resource(
'postgres',
'bitnami/postgresql',
@ -43,7 +51,9 @@ helm_resource(
resource_deps=['bitnami'],
labels=["9-data"])
k8s_yaml(
kustomize("./env/local")
)
def bh_backend_service(service="", port_forwards=[], migrateDB=False, devMode=True, labels=['2-services'], deps=['postgres']):
local_resource(
@ -133,6 +143,12 @@ k8s_resource(
resource_deps=['ingress']
)
k8s_resource(
workload='runner-sync-local',
labels='2-services'
)
# local_resource(
# 'web-client',
# dir='./src/web-client',
@ -152,22 +168,6 @@ k8s_resource(
# labels=['2-services']
# )
k8s_yaml(
kustomize("./env/local")
)
k8s_resource(
workload='runner-sync-local',
labels='2-services'
)
k8s_resource(
workload='ingress',
labels='1-ingress',
links=[
link(url='http://admin.localhost:8000', name='Admin Panel'),
]
)
# helm_resource(
# 'loki-stack',

23
src/catalog/internal/data/postgres/migrations/0001_init.sql
View File

@ -15,23 +15,17 @@ CREATE TABLE IF NOT EXISTS catalog.upcoming_auctions (
sourcesitename VARCHAR(256) NOT NULL,
sourceurl VARCHAR(1024) NOT NULL,
country VARCHAR(64) NOT NULL,
province VARCHAR(128) NOT NULL
);
CREATE TABLE IF NOT EXISTS catalog.upcoming_auctions_fts (
id SERIAL PRIMARY KEY,
auctionid INTEGER REFERENCES CATALOG.UPCOMING_AUCTIONS,
title VARCHAR(1024) NOT NULL,
sourcesitename VARCHAR(256) NOT NULL,
description TEXT NOT NULL,
province VARCHAR(128) NOT NULL,
ts tsvector GENERATED ALWAYS AS ((
setweight(to_tsvector('english', (sourcesitename)::TEXT), 'A') ||
setweight(to_tsvector('english', (title)::TEXT), 'A') ||
setweight(to_tsvector('english', (description)::TEXT), 'C')
setweight(to_tsvector('english', (description)::TEXT), 'B') ||
setweight(to_tsvector('english', (province)::TEXT), 'C') ||
setweight(to_tsvector('english', (country)::TEXT), 'C')
)) STORED
);
CREATE INDEX IF NOT EXISTS ts_idx ON catalog.upcoming_auctions_fts USING GIN(ts);
CREATE INDEX IF NOT EXISTS ts_idx ON catalog.upcoming_auctions USING GIN(ts);
-- +goose StatementBegin
CREATE OR REPLACE FUNCTION catalog.bh_import_auction(
@ -79,13 +73,6 @@ BEGIN
p_country,
p_province
) RETURNING id INTO auction_id;
INSERT INTO catalog.upcoming_auctions_fts (
auctionid,
title,
description,
sourcesitename
) VALUES ( auction_id, p_title, p_description, p_sourcesitename );
ELSE
-- 0 means there is a duplicate auction ID
auction_id = 0;

22
src/catalog/internal/data/postgres/queries.sql
View File

@ -1,18 +1,29 @@
-- name: GetUpcoming :many
SELECT ua.*
SELECT
ua.id,
ua.fingerprint,
ua.title,
ua.description,
ua.startts,
ua.endts,
ua.itemcount,
ua.sourcesiteurl,
ua.sourcesitename,
ua.sourceurl,
ua.country,
ua.province
FROM catalog.upcoming_auctions ua
LEFT JOIN catalog.upcoming_auctions_fts fts on ua.id = fts.auctionid
WHERE
ua.endts >= DATE(NOW()) AND
(case when sqlc.arg(searchTerm) = '' then
ua.id > 0
else
fts.ts @@ websearch_to_tsquery(sqlc.arg(searchTerm))
ua.ts @@ websearch_to_tsquery(sqlc.arg(searchTerm))
end)
ORDER BY
ua.endts ASC,
(case when sqlc.arg(searchTerm) != '' then
ts_rank(fts.ts, websearch_to_tsquery(sqlc.arg(searchTerm)))
ts_rank(ua.ts, websearch_to_tsquery(sqlc.arg(searchTerm)))
end) DESC
OFFSET (sqlc.arg(page)::INTEGER * sqlc.arg(pageSize)::INTEGER)
LIMIT sqlc.arg(pageSize)::INTEGER;
@ -20,13 +31,12 @@ LIMIT sqlc.arg(pageSize)::INTEGER;
-- name: GetTotals :one
SELECT COUNT(*) as total,
(SELECT COUNT(*) FROM catalog.upcoming_auctions ua
LEFT JOIN catalog.upcoming_auctions_fts fts on ua.id = fts.auctionid
WHERE
ua.endts >= DATE(NOW()) AND
(case when sqlc.arg(searchTerm) = '' then
ua.id > 0
else
fts.ts @@ websearch_to_tsquery(sqlc.arg(searchTerm))
ua.ts @@ websearch_to_tsquery(sqlc.arg(searchTerm))
end)) as found
FROM catalog.upcoming_auctions
WHERE endts >= DATE(NOW());

2
src/catalog/internal/data/storage.go
View File

@ -16,7 +16,7 @@ type PGCatalogStorage struct {
}
func (ps *PGCatalogStorage) GetUpcoming(ctx context.Context, q domain.UpcomingQuery) (results []domain.Auction, total int64, found int64, err error) {
var pgResults []postgres.CatalogUpcomingAuction
var pgResults []postgres.GetUpcomingRow
if pgResults, err = ps.Queries.GetUpcoming(ctx, postgres.GetUpcomingParams{
Searchterm: q.Term,

4
src/web-client/src/routes/+layout.svelte
View File

@ -1,6 +1,6 @@
<script lang="ts">
import type { LayoutData } from './$types';
import { goto, invalidateAll } from '$app/navigation';
import { goto } from '$app/navigation';
import SearchBox from '$lib/SearchBox.svelte';
import '../app.css';
@ -40,7 +40,7 @@
<ul class="flex w-full py-3 items-center justify-between">
<li class="flex grow justify-center">
<span class="flex grow" style="max-width: 75%;">
<SearchBox on:search={onSubmit} page={data.page} />
<SearchBox on:search={onSubmit} query={data.query} page={data.page} />
</span>
</li>
<li class="px-6">

17
src/web-client/src/routes/+page.svelte
View File

@ -5,10 +5,17 @@
export let data: PageData;
function buildQueryString({ query, page }: { query: string; page: number }): string {
const qs = [query === '' ? query : `query=${query}`, page <= 1 ? '' : `page=${page}`]
.filter((x) => x !== '')
.join('&');
return qs !== '' ? `?${qs}` : qs;
}
$: hasResults = (data.results || []).length > 0;
$: currentPage = data.page + 1;
$: pageCount = Math.max(1, Math.floor(data.found / data.limit));
$: console.log(data.found, ' / ', data.limit, ' = ', data.found / data.limit);
$: pageCount = Math.max(1, Math.ceil(data.found / data.limit));
</script>
{#if hasResults}
@ -17,7 +24,9 @@
<ol class="flex justify-between w-full center" style="padding: 0 10%;">
{#if currentPage > 1}
<li in:fade>
<a href="/?query={data.query}&page={data.page - 1}">&lt;Previous</a>
<a href="/{buildQueryString({ query: data.query, page: currentPage - 1 })}"
>&lt;Previous</a
>
</li>
{:else}
<li />
@ -25,7 +34,7 @@
<li>Page {currentPage} of {pageCount}</li>
{#if currentPage < pageCount}
<li in:fade>
<a href="/?query={data.query}&page={data.page + 1}">Next&gt;</a>
<a href="/{buildQueryString({ query: data.query, page: currentPage + 1 })}">Next&gt;</a>
</li>
{/if}
</ol>

51
src/web-client/src/routes/+page.ts
View File

@ -14,20 +14,19 @@ interface SearchPageData {
}
export const load = (async ({ fetch, url }): Promise<SearchPageData> => {
// TODO: refactor to one source of truth for all query param value fetching
const searchTerm = url.searchParams.get('query') || '';
const currentPage = Number(url.searchParams.get('page') || 0);
const currentLimit = Number(url.searchParams.get('limit') || 64);
const searchParams = new SearchParameters(url);
const limit = searchParams.getLimit();
const searchTerm = searchParams.getSearchTerm();
try {
// TODO: refactor to one source of truth for all query string building
const response = await fetch(API_HOST + `/upcoming?searchTerm=${searchTerm}&page=${currentPage}&limit=${currentLimit}`);
const response = await fetch(API_HOST + `/upcoming${searchParams.toQueryString()}`);
const { page, total, found, results } = await response.json() || {};
// TODO: return found results so we can do upperbound on pagination
return {
page,
limit: currentLimit,
limit,
query: searchTerm,
found,
total,
@ -37,7 +36,7 @@ export const load = (async ({ fetch, url }): Promise<SearchPageData> => {
console.log(e);
return {
page: 0,
limit: currentLimit,
limit,
query: searchTerm,
found: 0,
total: 0,
@ -46,3 +45,41 @@ export const load = (async ({ fetch, url }): Promise<SearchPageData> => {
}
}) satisfies PageLoad;
class SearchParameters {
page: number
limit: number
searchTerm?: string
constructor(url: URL) {
this.searchTerm = url.searchParams.get('query') || undefined;
this.page = Number(url.searchParams.get('page') || 1) - 1;
if (this.page < 0 ) {
this.page = 0;
}
this.limit = Number(url.searchParams.get('limit') || 64);
if (this.limit > 128) {
this.limit = 128;
} else if (this.limit < 32) {
this.limit = 32;
}
}
getPage(): number {
return this.page || 0;
}
getSearchTerm(): string {
return this.searchTerm || '';
}
getLimit(): number {
return this.limit || 64;
}
toQueryString(): string {
const qs = Object.entries(this).filter(t => t.length > 0 && t[1]).map(t => `${t[0]}=${t[1]}`).join('&');
return qs === '' ? '' : `?${qs}`;
}
}