add pagination to backend

src/admin-client/src/routes/+page.svelte

@@ -1,7 +1,6 @@
 <script lang="ts">
 	import type { PageData } from './$types';
 	import { fade } from 'svelte/transition';
-	import { invalidateAll } from '$app/navigation';
 	import StartScrapeForm from '$lib/StartScrapeForm.svelte';
 	import ScrapeJobResult from '$lib/ScrapeJobResult.svelte';
 

src/admin-client/src/routes/+page.ts

@@ -14,18 +14,54 @@ interface Job {
 }
 
 interface ScrapeStatusPageData {
-	jobs: Job[]
+	jobs: Job[];
 }
 
 export const load = (async ({ fetch, url }): Promise<ScrapeStatusPageData> => {
+	const searchParams = new SearchParameters(url);
 	try {
-		const response = await fetch(API_HOST + `/sync`);
-		const {results } = await response.json();
+		const response = await fetch(API_HOST + `/sync${searchParams.toQueryString()}`);
+		const { results } = await response.json();
 		return {
-			jobs: results    || []
+			jobs: results || []
 		};
 	} catch (e) {
 		console.log(e);
-		return { jobs:[] };
+		return { jobs: [] };
 	}
 }) satisfies PageLoad;
+
+class SearchParameters {
+	page: number;
+	limit: number;
+
+	constructor(url: URL) {
+		this.page = Number(url.searchParams.get('page') || 1);
+		if (this.page < 0) {
+			this.page = 0;
+		}
+
+		this.limit = Number(url.searchParams.get('limit') || 12);
+		if (this.limit > 32) {
+			this.limit = 32;
+		} else if (this.limit < 12) {
+			this.limit = 12;
+		}
+	}
+
+	getPage(): number {
+		return this.page || 1;
+	}
+
+	getLimit(): number {
+		return this.limit || 12;
+	}
+
+	toQueryString(): string {
+		const qs = Object.entries(this)
+			.filter((t) => t.length > 0 && t[1])
+			.map((t) => `${t[0]}=${t[1]}`)
+			.join('&');
+		return qs === '' ? '' : `?${qs}`;
+	}
+}

src/runner/acceptance-tests.yml

@@ -7,6 +7,6 @@ tests:
       statusCodes: [200]
   - description: "get list of jobs"
     request:
-      path: "/v1/sync"
+      path: "/v1/sync?page=1&limit=6"
     response:
       statusCodes: [200]

src/runner/api/grpc/runner.proto

@@ -37,7 +37,7 @@ message SyncStatus {
 
 message StatusFilter {
   int32 page = 1;
-  int32 id = 2;
+  int32 limit = 2;
 }
 
 message SyncStatusList {

src/runner/internal/data/postgres/queries.sql

@@ -6,7 +6,9 @@ SELECT id,
 	   auctionsFound,
 	   errors
 	FROM runner.scrapejob
-	ORDER BY startedTs DESC;
+	ORDER BY startedTs DESC
+	OFFSET (sqlc.arg(page)::INTEGER * sqlc.arg(pageSize)::INTEGER)
+	LIMIT sqlc.arg(pageSize)::INTEGER;;
 
 -- name: GetJobByID :one
 SELECT id,

src/runner/internal/data/storage.go

@@ -64,9 +64,12 @@ func (db *PGRunnerStorage) CompleteScrapeJob(ctx context.Context, ID int, status
 	return
 }
 
-func (db *PGRunnerStorage) GetJobs(ctx context.Context) (results []domain.ScrapeJob, err error) {
+func (db *PGRunnerStorage) GetJobs(ctx context.Context, page int32, limit int32) (results []domain.ScrapeJob, err error) {
 	var jobs []postgres.RunnerScrapejob
-	if jobs, err = db.Queries.GetJobs(ctx); err != nil {
+	if jobs, err = db.Queries.GetJobs(ctx, postgres.GetJobsParams{
+		Page:     page,
+		Pagesize: limit,
+	}); err != nil {
 		err = fmt.Errorf("Couldn't get jobs from DB: %w", err)
 		return
 	}

src/runner/internal/domain/logic.go

@@ -41,7 +41,7 @@ type (
 	Storage interface {
 		CreateScrapeJob(context.Context, string) (ScrapeJob, error)
 		CompleteScrapeJob(context.Context, int, CompleteScrapeJobStatus) (ScrapeJob, error)
-		GetJobs(context.Context) ([]ScrapeJob, error)
+		GetJobs(context.Context, int32, int32) ([]ScrapeJob, error)
 	}
 
 	CatalogService interface {
@@ -75,14 +75,17 @@ func (domain Domain) StartSync(ctx context.Context, in FindNewUpcomingInput) (ou
 }
 
 type (
-	GetJobsInput  struct{}
+	GetJobsInput struct {
+		Page  int32
+		Limit int32
+	}
 	GetJobsOutput struct {
 		Jobs []ScrapeJob
 	}
 )
 
 func (domain Domain) Status(ctx context.Context, in GetJobsInput) (out GetJobsOutput, err error) {
-	scrapeJobs, err := domain.Storage.GetJobs(ctx)
+	scrapeJobs, err := domain.Storage.GetJobs(ctx, in.Page, in.Limit)
 	if err != nil {
 		err = fmt.Errorf("could not fetch jobs from storage: %w", err)
 		return

src/runner/internal/handler.go

@@ -3,6 +3,7 @@ package internal
 import (
 	"context"
 
+	"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
 	api "git.vdhsn.com/barretthousen/barretthousen/src/runner/api/grpc"
 	"git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/domain"
 	"google.golang.org/grpc"
@@ -41,7 +42,19 @@ func (rh *runnerHandler) StartSync(ctx context.Context, cmd *api.SyncParameters)
 }
 
 func (rh *runnerHandler) Status(ctx context.Context, cmd *api.StatusFilter) (*api.SyncStatusList, error) {
-	out, err := rh.domain.Status(ctx, domain.GetJobsInput{})
+	kernel.TraceLog.Printf("paging options: page %d - limit %d", cmd.Page, cmd.Limit)
+	if cmd.Limit <= 0 {
+		cmd.Limit = 64
+	}
+
+	if cmd.Page <= 1 {
+		cmd.Page = 1
+	}
+
+	out, err := rh.domain.Status(ctx, domain.GetJobsInput{
+		Page:  cmd.Page - 1,
+		Limit: cmd.Limit,
+	})
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "method GetJobs failed: %q", err.Error())
 	}