Merge remote-tracking branch 'origin/feat/mvp' into feat/mvp-deploy-beta

* origin/feat/mvp:
  pagination and linking improvements
  use 1 table for catalog data + search
  re arrange some things

Tiltfile

@@ -28,6 +28,14 @@ helm_resource(
     labels=["1-ingress"]
 )
 
+k8s_resource(
+    workload='ingress',
+    labels='1-ingress',
+    links=[
+        link(url='http://admin.localhost:8000', name='Admin Panel'),
+    ]
+)
+
 helm_resource(
     'postgres',
     'bitnami/postgresql',
@@ -43,7 +51,9 @@ helm_resource(
     resource_deps=['bitnami'],
     labels=["9-data"])
 
-
+k8s_yaml(
+    kustomize("./env/local")
+)
 
 def bh_backend_service(service="", port_forwards=[], migrateDB=False, devMode=True, labels=['2-services'], deps=['postgres']):
     local_resource(
@@ -133,6 +143,12 @@ k8s_resource(
     resource_deps=['ingress']
 )
 
+
+k8s_resource(
+    workload='runner-sync-local',
+    labels='2-services'
+)
+
 # local_resource(
 #     'web-client',
 #     dir='./src/web-client',
@@ -152,22 +168,6 @@ k8s_resource(
 #     labels=['2-services']
 # )
 
-k8s_yaml(
-    kustomize("./env/local")
-)
-
-k8s_resource(
-    workload='runner-sync-local',
-    labels='2-services'
-)
-
-k8s_resource(
-    workload='ingress',
-    labels='1-ingress',
-    links=[
-        link(url='http://admin.localhost:8000', name='Admin Panel'),
-    ]
-)
 
 # helm_resource(
 #     'loki-stack',

src/catalog/internal/data/postgres/migrations/0001_init.sql

@@ -15,23 +15,17 @@ CREATE TABLE IF NOT EXISTS catalog.upcoming_auctions (
     sourcesitename VARCHAR(256) NOT NULL,
     sourceurl      VARCHAR(1024) NOT NULL,
     country        VARCHAR(64) NOT NULL,
-    province       VARCHAR(128) NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS catalog.upcoming_auctions_fts (
-    id          SERIAL PRIMARY KEY,
-    auctionid   INTEGER REFERENCES CATALOG.UPCOMING_AUCTIONS,
-    title       VARCHAR(1024) NOT NULL,
-	sourcesitename VARCHAR(256) NOT NULL,
-    description TEXT NOT NULL,
+    province       VARCHAR(128) NOT NULL,
     ts          tsvector GENERATED ALWAYS AS ((
 					setweight(to_tsvector('english', (sourcesitename)::TEXT), 'A') ||
 					setweight(to_tsvector('english', (title)::TEXT), 'A') ||
-					setweight(to_tsvector('english', (description)::TEXT), 'C')
+					setweight(to_tsvector('english', (description)::TEXT), 'B') ||
+					setweight(to_tsvector('english', (province)::TEXT), 'C') ||
+					setweight(to_tsvector('english', (country)::TEXT), 'C')
 				)) STORED
 );
 
-CREATE INDEX IF NOT EXISTS ts_idx ON catalog.upcoming_auctions_fts USING GIN(ts);
+CREATE INDEX IF NOT EXISTS ts_idx ON catalog.upcoming_auctions USING GIN(ts);
 
 -- +goose StatementBegin
 CREATE OR REPLACE FUNCTION catalog.bh_import_auction(
@@ -79,13 +73,6 @@ BEGIN
 			p_country,
 			p_province
 		) RETURNING id INTO auction_id;
-
-		INSERT INTO catalog.upcoming_auctions_fts (
-			auctionid,
-			title,
-			description,
-			sourcesitename
-		) VALUES ( auction_id, p_title, p_description, p_sourcesitename );
     ELSE
 		-- 0 means there is a duplicate auction ID
 	    auction_id = 0;

src/catalog/internal/data/postgres/queries.sql

@@ -1,18 +1,29 @@
 -- name: GetUpcoming :many
-SELECT ua.*
+SELECT
+	ua.id,
+	ua.fingerprint,
+	ua.title,
+	ua.description,
+	ua.startts,
+	ua.endts,
+	ua.itemcount,
+	ua.sourcesiteurl,
+	ua.sourcesitename,
+	ua.sourceurl,
+	ua.country,
+	ua.province
 FROM catalog.upcoming_auctions ua
-LEFT JOIN catalog.upcoming_auctions_fts fts on ua.id = fts.auctionid
 WHERE
 	ua.endts >= DATE(NOW()) AND
 	(case when sqlc.arg(searchTerm) = '' then
 		ua.id > 0
 	else
-		fts.ts @@ websearch_to_tsquery(sqlc.arg(searchTerm))
+		ua.ts @@ websearch_to_tsquery(sqlc.arg(searchTerm))
 	end)
 ORDER BY
 	ua.endts ASC,
 	(case when sqlc.arg(searchTerm) != '' then
-		ts_rank(fts.ts, websearch_to_tsquery(sqlc.arg(searchTerm)))
+		ts_rank(ua.ts, websearch_to_tsquery(sqlc.arg(searchTerm)))
 	end) DESC
 OFFSET (sqlc.arg(page)::INTEGER * sqlc.arg(pageSize)::INTEGER)
 LIMIT sqlc.arg(pageSize)::INTEGER;
@@ -20,13 +31,12 @@ LIMIT sqlc.arg(pageSize)::INTEGER;
 -- name: GetTotals :one
 SELECT COUNT(*) as total,
 	   (SELECT COUNT(*) FROM catalog.upcoming_auctions ua
-			LEFT JOIN catalog.upcoming_auctions_fts fts on ua.id = fts.auctionid
 		WHERE
 			ua.endts >= DATE(NOW()) AND
 			(case when sqlc.arg(searchTerm) = '' then
 				ua.id > 0
 			else
-				fts.ts @@ websearch_to_tsquery(sqlc.arg(searchTerm))
+				ua.ts @@ websearch_to_tsquery(sqlc.arg(searchTerm))
 		end)) as found
 FROM catalog.upcoming_auctions
 WHERE endts >= DATE(NOW());

src/catalog/internal/data/storage.go

@@ -16,7 +16,7 @@ type PGCatalogStorage struct {
 }
 
 func (ps *PGCatalogStorage) GetUpcoming(ctx context.Context, q domain.UpcomingQuery) (results []domain.Auction, total int64, found int64, err error) {
-	var pgResults []postgres.CatalogUpcomingAuction
+	var pgResults []postgres.GetUpcomingRow
 
 	if pgResults, err = ps.Queries.GetUpcoming(ctx, postgres.GetUpcomingParams{
 		Searchterm: q.Term,

src/web-client/src/routes/+layout.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
 	import type { LayoutData } from './$types';
-	import { goto, invalidateAll } from '$app/navigation';
+	import { goto } from '$app/navigation';
 	import SearchBox from '$lib/SearchBox.svelte';
 	import '../app.css';
 
@@ -40,7 +40,7 @@
 		<ul class="flex w-full py-3 items-center justify-between">
 			<li class="flex grow justify-center">
 				<span class="flex grow" style="max-width: 75%;">
-					<SearchBox on:search={onSubmit} page={data.page} />
+					<SearchBox on:search={onSubmit} query={data.query} page={data.page} />
 				</span>
 			</li>
 			<li class="px-6">

src/web-client/src/routes/+page.svelte

@@ -5,10 +5,17 @@
 
 	export let data: PageData;
 
+	function buildQueryString({ query, page }: { query: string; page: number }): string {
+		const qs = [query === '' ? query : `query=${query}`, page <= 1 ? '' : `page=${page}`]
+			.filter((x) => x !== '')
+			.join('&');
+
+		return qs !== '' ? `?${qs}` : qs;
+	}
+
 	$: hasResults = (data.results || []).length > 0;
 	$: currentPage = data.page + 1;
-	$: pageCount = Math.max(1, Math.floor(data.found / data.limit));
-	$: console.log(data.found, ' / ', data.limit, ' = ', data.found / data.limit);
+	$: pageCount = Math.max(1, Math.ceil(data.found / data.limit));
 </script>
 
 {#if hasResults}
@@ -17,7 +24,9 @@
 		<ol class="flex justify-between w-full center" style="padding: 0 10%;">
 			{#if currentPage > 1}
 				<li in:fade>
-					<a href="/?query={data.query}&page={data.page - 1}">&lt;Previous</a>
+					<a href="/{buildQueryString({ query: data.query, page: currentPage - 1 })}"
+						>&lt;Previous</a
+					>
 				</li>
 			{:else}
 				<li />
@@ -25,7 +34,7 @@
 			<li>Page {currentPage} of {pageCount}</li>
 			{#if currentPage < pageCount}
 				<li in:fade>
-					<a href="/?query={data.query}&page={data.page + 1}">Next&gt;</a>
+					<a href="/{buildQueryString({ query: data.query, page: currentPage + 1 })}">Next&gt;</a>
 				</li>
 			{/if}
 		</ol>

src/web-client/src/routes/+page.ts

@@ -14,20 +14,19 @@ interface SearchPageData {
 }
 
 export const load = (async ({ fetch, url }): Promise<SearchPageData> => {
-	// TODO: refactor to one source of truth for all query param value fetching
-	const searchTerm = url.searchParams.get('query') || '';
-	const currentPage = Number(url.searchParams.get('page') || 0);
-	const currentLimit = Number(url.searchParams.get('limit') || 64);
+	const searchParams = new SearchParameters(url);
+	const limit = searchParams.getLimit();
+	const searchTerm = searchParams.getSearchTerm();
 
 	try {
 		// TODO: refactor to one source of truth for all query string building
-		const response = await fetch(API_HOST + `/upcoming?searchTerm=${searchTerm}&page=${currentPage}&limit=${currentLimit}`);
+		const response = await fetch(API_HOST + `/upcoming${searchParams.toQueryString()}`);
 		const { page, total, found, results } = await response.json() || {};
 
 		// TODO: return found results so we can do upperbound on pagination
 		return {
 			page,
-			limit: currentLimit,
+			limit,
 			query: searchTerm,
 			found,
 			total,
@@ -37,7 +36,7 @@ export const load = (async ({ fetch, url }): Promise<SearchPageData> => {
 		console.log(e);
 		return {
 			page: 0,
-			limit: currentLimit,
+			limit,
 			query: searchTerm,
 			found: 0,
 			total: 0,
@@ -46,3 +45,41 @@ export const load = (async ({ fetch, url }): Promise<SearchPageData> => {
 	}
 
 }) satisfies PageLoad;
+
+class SearchParameters {
+	page: number
+	limit: number
+	searchTerm?: string
+
+	constructor(url: URL) {
+		this.searchTerm = url.searchParams.get('query') || undefined;
+		this.page = Number(url.searchParams.get('page') || 1) - 1;
+		if (this.page < 0 ) {
+			this.page = 0;
+		}
+
+		this.limit = Number(url.searchParams.get('limit') || 64);
+		if (this.limit > 128) {
+			this.limit = 128;
+		} else if (this.limit < 32) {
+			this.limit = 32;
+		}
+	}
+
+	getPage(): number {
+		return this.page || 0;
+	}
+
+	getSearchTerm(): string {
+		return this.searchTerm || '';
+	}
+
+	getLimit(): number {
+		return this.limit || 64;
+	}
+
+	toQueryString(): string {
+		const qs = Object.entries(this).filter(t => t.length > 0 && t[1]).map(t => `${t[0]}=${t[1]}`).join('&');
+		return qs === '' ? '' : `?${qs}`;
+	}
+}