pagination for active/completed jobs

src/admin-client/src/lib/StartScrapeForm.svelte

@@ -1,12 +1,14 @@
 <script lang="ts">
 	import { createEventDispatcher } from 'svelte';
 
+	export let disable: boolean = false;
+
 	let target: string = 'All';
 
 	const dispatch = createEventDispatcher();
 
 	function execScrape() {
-		dispatch('scrape', { target });
+		if (!disable) dispatch('scrape', { target });
 	}
 </script>
 
@@ -15,6 +17,7 @@
 		<option>All</option>
 		<option>liveauctioneers</option>
 	</select>
-
-	<button class="border-none rounded-r-md bg-bh-gold text-bh-black py-1 px-2">Start Sync</button>
+	<button class="border-none rounded-r-md bg-bh-gold text-bh-black py-1 px-2" disabled={disable}
+		>Start Sync</button
+	>
 </form>

src/admin-client/src/routes/+page.svelte

@@ -1,18 +1,27 @@
 <script lang="ts">
 	import type { PageData } from './$types';
-	import { fade } from 'svelte/transition';
 	import StartScrapeForm from '$lib/StartScrapeForm.svelte';
 	import ScrapeJobResult from '$lib/ScrapeJobResult.svelte';
 	import Pager from '$lib/Pager.svelte';
 
+	import { fade } from 'svelte/transition';
+	import { onDestroy, onMount } from 'svelte';
+	import { invalidateAll } from '$app/navigation';
+
 	export let data: PageData;
 
-	$: completedJobs = data.jobs.filter(({ completedTs }) => completedTs !== null);
-	$: activeJobs = data.jobs.filter(({ completedTs }) => completedTs === null);
-	$: activeJobCount = activeJobs.length;
-	$: completedJobCount = completedJobs.length;
+	$: activeJobCount = data.active.length;
+
+	let disableSync = false;
+
+	var intervalId: any;
+	onMount(() => {
+		intervalId = setInterval(invalidateAll, 2000);
+	});
+	onDestroy(() => clearInterval(intervalId));
 
 	async function onScrape({ detail }) {
+		disableSync = true;
 		try {
 			const response = await fetch(
 				new Request('/api/v1/sync', {
@@ -23,11 +32,10 @@
 					body: JSON.stringify({ targetSite: detail.target })
 				})
 			);
-
-			const scrapeJob = await response.json();
-			data.jobs.push(scrapeJob);
-			console.log(scrapeJob);
-		} catch (error) {}
+		} catch (error) {
+		} finally {
+			disableSync = false;
+		}
 	}
 
 	function buildQueryString({ page, pageSize }: { page: number; pageSize: number }): string {
@@ -39,52 +47,48 @@
 			.filter((x) => x !== '')
 			.join('&');
 
-		return qs !== '' ? `?${qs}` : qs;
+		return qs !== '' ? `/?${qs}` : qs;
 	}
+
+	const limit = 16;
 </script>
 
 <section class="flex w-full flex-col justify-center" in:fade out:fade>
-	<section in:fade class="pb-10">
-		<h1 class="text-2xl pb-5 mb-8">Sync Status</h1>
+	<section in:fade>
+		<h1 class="text-2xl pb-5">Sync Status: {data.total} jobs</h1>
+	</section>
+	<section class="pb-5">
+		<h2 class="text-2xl">{activeJobCount} In Progress</h2>
+		{#key data.activeTotal}
+			<ul class="flex">
+				{#each data.active as job, i}
+					<li id="job-{job.id}" in:fade={{ delay: i * 90 }}>
+						<ScrapeJobResult {job} />
+					</li>
+				{/each}
+			</ul>
+		{/key}
+	</section>
+	<StartScrapeForm on:scrape={onScrape} disable={disableSync} />
+	<section class="py-10">
+		<h2 class="text-2xl">Completed</h2>
 		<Pager
 			page={data.page}
-			itemCount={data.found}
-			pageSize={data.limit}
+			itemCount={data.completeTotal}
+			pageSize={limit}
 			createUrl={buildQueryString}
 		/>
+		{#key data.completeTotal + data.page}
+			<ul class="flex flex-wrap justify-between">
+				{#each data.complete as job, i}
+					<li id="job-{job.id}" in:fade={{ delay: i * 90 }}>
+						<ScrapeJobResult {job} />
+					</li>
+				{/each}
+			</ul>
+		{/key}
 	</section>
-	<div class="mb-8">
-		<StartScrapeForm on:scrape={onScrape} />
-	</div>
 	<section>
-		<h2>{activeJobCount} In Progress Jobs</h2>
-		<ul class="flex">
-			{#each activeJobs as j, i}
-				<li in:fade>
-					<ScrapeJobResult job={j} />
-				</li>
-				<!-- {#if i < 10}
-				{/if} -->
-			{/each}
-		</ul>
-		<h2>{completedJobCount} Complete</h2>
-		<ul class="flex flex-wrap justify-between">
-			{#each completedJobs as j, i}
-				<li in:fade>
-					<ScrapeJobResult job={j} />
-				</li>
-				<!-- {#if i < 5}
-				{/if} -->
-			{/each}
-		</ul>
-	</section>
-
-	<section class="pt-10">
-		<Pager
-			page={data.page}
-			itemCount={data.found}
-			pageSize={data.limit}
-			createUrl={buildQueryString}
-		/>
+		<Pager page={data.page} itemCount={data.total} pageSize={limit} createUrl={buildQueryString} />
 	</section>
 </section>

src/admin-client/src/routes/+page.ts

@@ -14,20 +14,42 @@ interface Job {
 }
 
 interface ScrapeStatusPageData {
-	jobs: Job[];
+	page: number;
+	total: number;
+	activeTotal: number;
+	completeTotal: number;
+	limit: number;
+	active: Job[];
+	complete: Job[];
 }
 
 export const load = (async ({ fetch, url }): Promise<ScrapeStatusPageData> => {
 	const searchParams = new SearchParameters(url);
+	const limit = searchParams.getLimit();
 	try {
 		const response = await fetch(API_HOST + `/sync${searchParams.toQueryString()}`);
-		const { results } = await response.json();
+		const { active, complete, activeTotal, completeTotal, total, page } = await response.json();
+
 		return {
-			jobs: results || []
+			active,
+			complete,
+			activeTotal,
+			completeTotal,
+			total,
+			page,
+			limit
 		};
 	} catch (e) {
 		console.log(e);
-		return { jobs: [] };
+		return {
+			activeTotal: 0,
+			completeTotal: 0,
+			active: [],
+			complete: [],
+			total: 0,
+			page: 1,
+			limit
+		};
 	}
 }) satisfies PageLoad;
 

src/runner/api/grpc/runner.proto

@@ -41,7 +41,10 @@ message StatusFilter {
 }
 
 message SyncStatusList {
-	repeated SyncStatus results = 1;
-  int32 page = 2;
-  int32 total = 3;
+	repeated SyncStatus active = 1;
+	repeated SyncStatus complete = 2;
+  int32 page = 3;
+  int32 total = 4;
+  int32 activeTotal = 5;
+  int32 completeTotal = 6;
 }

src/runner/internal/data/postgres/queries.sql

@@ -1,4 +1,4 @@
--- name: GetJobs :many
+-- name: GetCompletedJobs :many
 SELECT id,
 	   startedTs,
 	   completedTs,
@@ -6,9 +6,31 @@ SELECT id,
 	   auctionsFound,
 	   errors
 	FROM runner.scrapejob
+	WHERE completedts is not null OR NOW() >= startedts + (30 ||' minutes')::interval
 	ORDER BY startedTs DESC
 	OFFSET (sqlc.arg(page)::INTEGER * sqlc.arg(pageSize)::INTEGER)
-	LIMIT sqlc.arg(pageSize)::INTEGER;;
+	LIMIT sqlc.arg(pageSize)::INTEGER;
+
+-- name: GetActiveJobs :many
+SELECT id,
+	   startedTs,
+	   completedTs,
+	   targetSiteName,
+	   auctionsFound,
+	   errors
+	FROM runner.scrapejob
+	WHERE completedTs is null AND NOW() < startedTs + (30 ||' minutes')::interval
+	ORDER BY startedTs DESC
+	OFFSET (sqlc.arg(page)::INTEGER * sqlc.arg(pageSize)::INTEGER)
+	LIMIT sqlc.arg(pageSize)::INTEGER;
+
+-- name: GetJobCounts :one
+SELECT COUNT(*) AS total,
+       (SELECT COUNT(*) FROM runner.scrapejob
+	   		WHERE completedts is not null OR NOW() >= startedts + (30 ||' minutes')::interval) AS completed,
+       (SELECT COUNT(*) FROM runner.scrapejob
+	   		WHERE completedts is null AND NOW() < startedts + (30 ||' minutes')::interval) AS active
+	   FROM runner.scrapejob;
 
 -- name: GetJobByID :one
 SELECT id,

src/runner/internal/data/storage.go

@@ -64,9 +64,9 @@ func (db *PGRunnerStorage) CompleteScrapeJob(ctx context.Context, ID int, status
 	return
 }
 
-func (db *PGRunnerStorage) GetJobs(ctx context.Context, page int32, limit int32) (results []domain.ScrapeJob, err error) {
-	var jobs []postgres.RunnerScrapejob
-	if jobs, err = db.Queries.GetJobs(ctx, postgres.GetJobsParams{
+func (db *PGRunnerStorage) GetJobs(ctx context.Context, page int32, limit int32) (out domain.GetJobsResult, err error) {
+	var completeJobs []postgres.RunnerScrapejob
+	if completeJobs, err = db.Queries.GetCompletedJobs(ctx, postgres.GetCompletedJobsParams{
 		Page:     page,
 		Pagesize: limit,
 	}); err != nil {
@@ -74,15 +74,44 @@ func (db *PGRunnerStorage) GetJobs(ctx context.Context, page int32, limit int32)
 		return
 	}
 
-	for _, j := range jobs {
-		results = append(results, domain.ScrapeJob{
+	var activeJobs []postgres.RunnerScrapejob
+	if activeJobs, err = db.Queries.GetActiveJobs(ctx, postgres.GetActiveJobsParams{
+		Page:     0,
+		Pagesize: 64,
+	}); err != nil {
+		err = fmt.Errorf("Couldn't get jobs from DB: %w", err)
+		return
+	}
+
+	var jobCounts postgres.GetJobCountsRow
+	if jobCounts, err = db.Queries.GetJobCounts(ctx); err != nil {
+		err = fmt.Errorf("couldn't get total job count: %w", err)
+		return
+	}
+
+	out.Total = int(jobCounts.Total)
+	out.ActiveTotal = int(jobCounts.Active)
+	out.CompletedTotal = int(jobCounts.Completed)
+	out.Active = mapScrapeJob(activeJobs)
+	out.Complete = mapScrapeJob(completeJobs)
+
+	return
+}
+
+func mapScrapeJob(jobs []postgres.RunnerScrapejob) (result []domain.ScrapeJob) {
+	result = make([]domain.ScrapeJob, len(jobs))
+	for i, j := range jobs {
+		result[i] = domain.ScrapeJob{
 			ID:            int(j.ID),
 			Started:       j.Startedts,
-			Completed:     j.Completedts.Time,
 			TargetSite:    j.Targetsitename,
 			AuctionsFound: int(j.Auctionsfound),
 			Errors:        j.Errors,
-		})
+		}
+
+		if j.Completedts.Valid {
+			result[i].Completed = j.Completedts.Time
+		}
 	}
 
 	return

src/runner/internal/domain/logic.go

@@ -41,7 +41,7 @@ type (
 	Storage interface {
 		CreateScrapeJob(context.Context, string) (ScrapeJob, error)
 		CompleteScrapeJob(context.Context, int, CompleteScrapeJobStatus) (ScrapeJob, error)
-		GetJobs(context.Context, int32, int32) ([]ScrapeJob, error)
+		GetJobs(context.Context, int32, int32) (GetJobsResult, error)
 	}
 
 	CatalogService interface {
@@ -79,20 +79,21 @@ type (
 		Page  int32
 		Limit int32
 	}
-	GetJobsOutput struct {
-		Jobs []ScrapeJob
+	GetJobsResult struct {
+		Total          int
+		ActiveTotal    int
+		Active         []ScrapeJob
+		CompletedTotal int
+		Complete       []ScrapeJob
 	}
 )
 
-func (domain Domain) Status(ctx context.Context, in GetJobsInput) (out GetJobsOutput, err error) {
-	scrapeJobs, err := domain.Storage.GetJobs(ctx, in.Page, in.Limit)
-	if err != nil {
+func (domain Domain) Status(ctx context.Context, in GetJobsInput) (out GetJobsResult, err error) {
+	if out, err = domain.Storage.GetJobs(ctx, in.Page, in.Limit); err != nil {
 		err = fmt.Errorf("could not fetch jobs from storage: %w", err)
 		return
 	}
 
-	out = GetJobsOutput{Jobs: scrapeJobs}
-
 	return
 }
 

src/runner/internal/handler.go

@@ -60,26 +60,34 @@ func (rh *runnerHandler) Status(ctx context.Context, cmd *api.StatusFilter) (*ap
 	}
 
 	result := &api.SyncStatusList{
-		Results: []*api.SyncStatus{},
+		Page:          cmd.Page,
+		Total:         int32(out.Total),
+		ActiveTotal:   int32(out.ActiveTotal),
+		CompleteTotal: int32(out.CompletedTotal),
+		Active:        mapSyncStatus(out.Active),
+		Complete:      mapSyncStatus(out.Complete),
 	}
 
-	for _, j := range out.Jobs {
+	return result, nil
+}
+
+func mapSyncStatus(in []domain.ScrapeJob) (result []*api.SyncStatus) {
+	result = make([]*api.SyncStatus, len(in))
+	for i, j := range in {
 		var completedTime *timestamppb.Timestamp
 
 		if !j.Completed.IsZero() {
 			completedTime = timestamppb.New(j.Completed)
 		}
 
-		result.Results = append(result.Results, &api.SyncStatus{
+		result[i] = &api.SyncStatus{
 			Id:             int32(j.ID),
 			AuctionsFound:  int32(j.AuctionsFound),
 			CreatedTs:      timestamppb.New(j.Started),
 			CompletedTs:    completedTime,
 			TargetSiteName: j.TargetSite,
 			Errors:         j.Errors,
-		})
-
+		}
 	}
-
-	return result, nil
+	return
 }