working on scraping from LA
parent
2699f0c14d
commit
18b9523c8b
GPG Key ID:
6DB29003C6DD1E4B
|
|
@ -3,3 +3,4 @@ ctlptl 0.8.18
|
|||
kind 0.18.0
|
||||
kustomize 5.0.1
|
||||
kubectl 1.26.3
|
||||
buf 1.17.0
|
||||
|
|
|
|||
7
Makefile
7
Makefile
|
|
@ -8,12 +8,13 @@ GOBIN = $(shell go env GOPATH)/bin
|
|||
dev: .kubeconfig
|
||||
KUBECONFIG=$(KUBECONFIG) tilt up -f ./src/Tiltfile
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
.PHONY: clobber
|
||||
clobber:
|
||||
KUBECONFIG=$(KUBECONFIG) ctlptl delete registry kind-bh-registry || true
|
||||
KUBECONFIG=$(KUBECONFIG) ctlptl delete cluster kind-bh-local || true
|
||||
@rm -f $(KUBECONFIG)
|
||||
|
||||
|
||||
.PHONY: build
|
||||
build:
|
||||
docker build --build-arg "SERVICE=runner" -t bh/service-runner -f ./src/runner/Dockerfile .
|
||||
|
|
@ -29,7 +30,7 @@ acceptance-test:
|
|||
.PHONY: gen
|
||||
gen: $(GOBIN)/sqlc buf.lock
|
||||
@$(GOBIN)/sqlc generate -f ./src/sqlc.yaml
|
||||
@cd ./src && buf generate
|
||||
@cd ./src && $(GOBIN)/buf generate
|
||||
|
||||
|
||||
.PHONY: setup
|
||||
|
|
|
|||
|
|
@ -1 +1,4 @@
|
|||
go.uber.org/dig v1.0.0/go.mod h1:z+dSd2TP9Usi48jL8M3v63iSBVkiwtVyMKxMZYYauPg=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
package kernel
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type ServerBuilder func(grpc.ServiceRegistrar)
|
||||
|
||||
var (
|
||||
grpcServerInstance *grpc.Server
|
||||
listener net.Listener
|
||||
)
|
||||
|
||||
func StartGRPCServer(port int, sb ServerBuilder, opts ...grpc.ServerOption) (err error) {
|
||||
if grpcServerInstance != nil {
|
||||
return errors.New("There can only be one GRPC server running at a time")
|
||||
}
|
||||
|
||||
listener, err = net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not start tcp listener: %w", err)
|
||||
}
|
||||
|
||||
grpcServerInstance = grpc.NewServer(opts...)
|
||||
|
||||
sb(grpcServerInstance)
|
||||
|
||||
return grpcServerInstance.Serve(listener)
|
||||
}
|
||||
|
||||
func StopGRPCServer() error {
|
||||
grpcServerInstance.GracefulStop()
|
||||
return listener.Close()
|
||||
}
|
||||
|
|
@ -5,9 +5,9 @@ package main;
|
|||
import "google/protobuf/timestamp.proto";
|
||||
import "google/api/annotations.proto";
|
||||
|
||||
option go_package = "git.vdhsn.com/barretthousen/barretthousen/src/lib/services/runner";
|
||||
option go_package = "git.vdhsn.com/barretthousen/barretthousen/src/runner/api";
|
||||
|
||||
service Accounts {
|
||||
service Runner {
|
||||
rpc FindNewUpcoming(FindNewUpcomingCommand) returns (JobResult) {
|
||||
option (google.api.http) = {
|
||||
put: "/v1/findnewupcoming"
|
||||
|
|
@ -2,20 +2,37 @@ module git.vdhsn.com/barretthousen/barretthousen/src/runner
|
|||
|
||||
go 1.19
|
||||
|
||||
require git.vdhsn.com/barretthousen/barretthousen/src/lib v1.0.0
|
||||
require (
|
||||
git.vdhsn.com/barretthousen/barretthousen/src/lib v1.0.0
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2
|
||||
google.golang.org/genproto v0.0.0-20230223222841-637eb2293923
|
||||
google.golang.org/protobuf v1.28.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.1.0 // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/ilyakaznacheev/cleanenv v1.4.2 // indirect
|
||||
github.com/joho/godotenv v1.4.0 // indirect
|
||||
golang.org/x/net v0.8.0 // indirect
|
||||
golang.org/x/sys v0.6.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
|
||||
github.com/jackc/pgconn v1.14.0 // indirect
|
||||
github.com/jackc/pgconn v1.14.0
|
||||
github.com/jackc/pgio v1.0.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgproto3/v2 v2.3.2 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
|
||||
github.com/jackc/pgtype v1.14.0 // indirect
|
||||
github.com/jackc/pgx/v4 v4.18.1 // indirect
|
||||
github.com/jackc/pgx/v4 v4.18.1
|
||||
go.uber.org/automaxprocs v1.5.2 // indirect
|
||||
golang.org/x/crypto v0.6.0 // indirect
|
||||
golang.org/x/text v0.7.0 // indirect
|
||||
golang.org/x/text v0.8.0 // indirect
|
||||
google.golang.org/grpc v1.54.0
|
||||
)
|
||||
|
||||
replace git.vdhsn.com/barretthousen/barretthousen/src/lib v1.0.0 => ../lib
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I=
|
||||
github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
|
||||
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
|
||||
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
|
||||
|
|
@ -13,7 +15,17 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
|
|||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
|
||||
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAszqCJMbfTISJ7oMftp8+UGV08=
|
||||
github.com/ilyakaznacheev/cleanenv v1.4.2 h1:nRqiriLMAC7tz7GzjzUTBHfzdzw6SQ7XvTagkFqe/zU=
|
||||
github.com/ilyakaznacheev/cleanenv v1.4.2/go.mod h1:i0owW+HDxeGKE0/JPREJOdSCPIyOnmh6C0xhWAkF/xA=
|
||||
github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
|
||||
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
|
||||
|
|
@ -62,12 +74,16 @@ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0f
|
|||
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
|
||||
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
|
|
@ -146,6 +162,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL
|
|||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
@ -163,6 +181,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
|
|
@ -173,8 +193,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
|||
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
|
|
@ -189,8 +210,18 @@ golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8T
|
|||
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 h1:znp6mq/drrY+6khTAlJUDNFFcDGV2ENLYKpMq8SyCds=
|
||||
google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
|
||||
google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag=
|
||||
google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
|
||||
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
|
||||
|
|
@ -199,3 +230,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
|
|||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ=
|
||||
olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
|
||||
)
|
||||
|
||||
const LiveAuctioneers_Target = TargetSite("liveauctioneers")
|
||||
|
||||
type LA_UpcomingResults struct {
|
||||
Payload struct {
|
||||
Results struct{} `json:"results"`
|
||||
} `json:"payload"`
|
||||
}
|
||||
|
||||
// TODO: build liveauctioneers api client to put the two calls behind
|
||||
func (domain RunnerDomain) FindNewUpcoming_LiveAuctioneers(ctx context.Context, in FindNewUpcomingInput) (out FindNewUpcomingOutput, err error) {
|
||||
req, _ := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodGet,
|
||||
"https://search-party-prod.liveauctioneers.com/search/catalogsearch?c=20170802&client=web&client_version=5.0.0&excludedHouses=%5B%5D&max_facet_values=3&offset=300&pagesize=24&sort=saleStart&pageSize=24",
|
||||
nil)
|
||||
|
||||
req.Header.Set("User-Agent", "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0")
|
||||
req.Header.Set("Accept", "*/*")
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
req.Header.Set("Cache-Control", "no-cache")
|
||||
|
||||
var res *http.Response
|
||||
if res, err = http.DefaultClient.Do(req); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if res.StatusCode > 299 {
|
||||
err = fmt.Errorf("Got a bad http status: %d", res.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
var data []byte
|
||||
if data, err = ioutil.ReadAll(res.Body); err != nil {
|
||||
err = fmt.Errorf("could not read response body: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
kernel.TraceLog.Printf("%s", data)
|
||||
// curl 'https://item-api-prod.liveauctioneers.com/spa/small/catalogs?c=20170802' -X POST -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0' -H 'Accept: */*' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'Content-Type: application/json' -H 'Origin: https://www.liveauctioneers.com' -H 'DNT: 1' -H 'Alt-Used: item-api-prod.liveauctioneers.com' -H 'Connection: keep-alive' -H 'Referer: https://www.liveauctioneers.com/' -H 'Cookie: rbzid=MCMBxY4NXXt2ojSy6pRxVB6kz2RUIkGZu9j9g6FRdpdE6Ztu+RgO3PRCsEvDlJCDKpgIO+cUy/aHKCsLXjjlcUrGVF8GIo0D0Bcg8XniH2tGiuCa1MOrur3D6+lolfzerP1JI/lxbQfoCSnL30uFX4L4uqA0Qhs56Qo9wpkNDCrZA2cTbRPuCXrFyczps4bN4veZ1cOne1q4jMgxiNI75w==; rbzsessionid=a3e7079dfe6cc10e0b89f88d176b53ff; join-modal-last-seen=2023-04-26; pagination-page-size=120' -H 'Sec-Fetch-Dest: empty' -H 'Sec-Fetch-Mode: cors' -H 'Sec-Fetch-Site: same-site' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'TE: trailers' --data-raw '{"ids":[285693,284926,285574,285404,281011,285610,285838,284614,285043,285729,285148,285016,283446,280796,285031,284245,285775,284879,285554,284055,284833,284818,285920,284969,285107,285165,285924,285608,285732,284849,284768,286038,285188,285862,285375,284643,285733,285411,285074,285731,284784,281479,284758,283547,284368,284489,284723,285406,284681,285183,284751,283883,284657,283320,283130,284408,285539,285849,285583,284462,285546,285134,285867,282513,285082,284795,284642,285230,285233,285236,284801,285776,279905,284015,283432,285513,285925,284880,284207,284079,285106,285017,284243,286036,282291,285778,284747,285407,285828,284819,284295,282099,285044,285734,281896,285149,285533,284733,285032,285287,284574,285052,285555,284683,284834,284970,285929,284196,285108,285166,285737,284850,286044,284769,286039,285189]}'
|
||||
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunnerDomain_FindNewUpcoming_LiveAuctioneers(t *testing.T) {
|
||||
type fields struct {
|
||||
storage RunnerStorage
|
||||
}
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
in FindNewUpcomingInput
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
fields fields
|
||||
args args
|
||||
wantOut FindNewUpcomingOutput
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "test run",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
domain := RunnerDomain{
|
||||
storage: tt.fields.storage,
|
||||
}
|
||||
gotOut, err := domain.FindNewUpcoming_LiveAuctioneers(tt.args.ctx, tt.args.in)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("RunnerDomain.FindNewUpcoming_LiveAuctioneers() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(gotOut, tt.wantOut) {
|
||||
t.Errorf("RunnerDomain.FindNewUpcoming_LiveAuctioneers() = %v, want %v", gotOut, tt.wantOut)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.vdhsn.com/barretthousen/barretthousen/src/runner/api"
|
||||
"git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/data/postgres"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
type RunnerStorage interface {
|
||||
CreateScrapeJob(context.Context, string) (postgres.RunnerScrapejob, error)
|
||||
GetJobs(context.Context) ([]postgres.RunnerScrapejob, error)
|
||||
}
|
||||
|
||||
type RunnerDomain struct {
|
||||
storage RunnerStorage
|
||||
}
|
||||
|
||||
type (
|
||||
FindNewUpcomingInput struct {
|
||||
TargetSite string
|
||||
}
|
||||
|
||||
FindNewUpcomingOutput struct{}
|
||||
TargetSite string
|
||||
)
|
||||
|
||||
func (domain RunnerDomain) FindNewUpcoming(ctx context.Context, in FindNewUpcomingInput) (out FindNewUpcomingOutput, err error) {
|
||||
if in.TargetSite == string(LiveAuctioneers_Target) {
|
||||
out, err = domain.FindNewUpcoming_LiveAuctioneers(ctx, in)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type (
|
||||
GetJobsInput struct{}
|
||||
GetJobsOutput struct {
|
||||
Jobs []*api.JobResult
|
||||
}
|
||||
)
|
||||
|
||||
func (domain RunnerDomain) GetJobs(ctx context.Context, in GetJobsInput) (out GetJobsOutput, err error) {
|
||||
scrapeJobs, err := domain.storage.GetJobs(ctx)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("could not fetch jobs from storage: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
out = GetJobsOutput{
|
||||
Jobs: []*api.JobResult{},
|
||||
}
|
||||
|
||||
for _, sj := range scrapeJobs {
|
||||
jr := &api.JobResult{
|
||||
Id: sj.ID,
|
||||
AuctionsFound: sj.Auctionsfound,
|
||||
Errors: sj.Errors,
|
||||
TargetSiteName: sj.Targetsitename,
|
||||
CreatedTs: timestamppb.New(sj.Startedts),
|
||||
}
|
||||
|
||||
if sj.Completedts.Valid {
|
||||
jr.CompletedTs = timestamppb.New(sj.Completedts.Time)
|
||||
}
|
||||
|
||||
out.Jobs = append(out.Jobs, jr)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package internal
|
||||
|
||||
import (
|
||||
"git.vdhsn.com/barretthousen/barretthousen/src/runner/api"
|
||||
"git.vdhsn.com/barretthousen/barretthousen/src/runner/internal/data/postgres"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func NewRunnerServer(grpcServer grpc.ServiceRegistrar) {
|
||||
api.RegisterRunnerServer(grpcServer, &runnerHandler{})
|
||||
}
|
||||
|
||||
type runnerHandler struct {
|
||||
api.UnimplementedRunnerServer
|
||||
|
||||
q *postgres.Queries
|
||||
}
|
||||
|
||||
// func (rh *runnerHandler) FindNewUpcoming(ctx context.Context, cmd *runner.FindNewUpcomingCommand) (*runner.JobResult, error) {
|
||||
// return nil, status.Errorf(codes.Unimplemented, "method FindNewUpcoming not implemented")
|
||||
// }
|
||||
|
|
@ -4,11 +4,11 @@ import (
|
|||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
_ "embed"
|
||||
|
||||
"git.vdhsn.com/barretthousen/barretthousen/src/lib/kernel"
|
||||
"git.vdhsn.com/barretthousen/barretthousen/src/runner/internal"
|
||||
)
|
||||
|
||||
type (
|
||||
|
|
@ -45,21 +45,13 @@ func (app *RunnerApp) Start(ctx context.Context) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
kernel.InfoLog.Printf("%+v", app)
|
||||
t := time.NewTicker(time.Second)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-t.C:
|
||||
kernel.TraceLog.Println("waiting")
|
||||
t.Reset(time.Second)
|
||||
}
|
||||
}
|
||||
return kernel.StartGRPCServer(app.Port, internal.NewRunnerServer)
|
||||
}
|
||||
|
||||
func (app *RunnerApp) OnStop(ctx context.Context) {
|
||||
if err := kernel.StopGRPCServer(); err != nil {
|
||||
kernel.ErrorLog.Printf("could not gracefully stop GRPC server: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (app *RunnerApp) GetLogLevel() kernel.LogLevel { return app.LogLevel }
|
||||
|
|
|
|||
Loading…
Reference in New Issue