Default to SQLite rather than Postgres & Redis

.github/workflows/test.yml

@@ -27,10 +27,5 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-go-
 
-    - name: Start containers
-      run: |
-        docker-compose up -d
-        sleep 3
-
     - name: Test
-      run: go test -p 1 ./...
+      run: make test

.gitignore

@@ -1 +1,2 @@
 .idea
+dbs

Makefile

@@ -1,32 +1,3 @@
-# Determine if you have docker-compose or docker compose installed locally
-# If this does not work on your system, just set the name of the executable you have installed
-DCO_BIN := $(shell { command -v docker-compose || command -v docker compose; } 2>/dev/null)
-
-# Connect to the primary database
-.PHONY: db
-db:
-	docker exec -it pagoda_db psql postgresql://admin:admin@localhost:5432/app
-
-# Connect to the test database (you must run tests first before running this)
-.PHONY: db-test
-db-test:
-	docker exec -it pagoda_db psql postgresql://admin:admin@localhost:5432/app_test
-
-# Connect to the primary cache
-.PHONY: cache
-cache:
-	docker exec -it pagoda_cache redis-cli
-
-# Clear the primary cache
-.PHONY: cache-clear
-cache-clear:
-	docker exec -it pagoda_cache redis-cli flushall
-
- # Connect to the test cache
-.PHONY: cache-test
-cache-test:
-	docker exec -it pagoda_cache redis-cli -n 1
-
 # Install Ent code-generation module
 .PHONY: ent-install
 ent-install:
@@ -42,28 +13,6 @@ ent-gen:
 ent-new:
 	go run entgo.io/ent/cmd/ent new $(name)
 
-# Start the Docker containers
-.PHONY: up
-up:
-	$(DCO_BIN) up -d
-	sleep 3
-
-# Stop the Docker containers
-.PHONY: stop
-stop:
-	$(DCO_BIN) stop
-
-# Drop the Docker containers to wipe all data
-.PHONY: down
-down:
-	$(DCO_BIN) down
-
-# Rebuild Docker containers to wipe all data
-.PHONY: reset
-reset:
-	$(DCO_BIN) down
-	make up
-
 # Run the application
 .PHONY: run
 run:
@@ -75,12 +24,6 @@ run:
 test:
 	go test -count=1 -p 1 ./...
 
-# Run the worker
-.PHONY: worker
-worker:
-	clear
-	go run cmd/worker/main.go
-
 # Check for direct dependency updates
 .PHONY: check-updates
 check-updates:

cmd/web/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -12,6 +13,7 @@ import (
 
 	"github.com/mikestefanello/pagoda/pkg/handlers"
 	"github.com/mikestefanello/pagoda/pkg/services"
+	"github.com/mikestefanello/pagoda/pkg/tasks"
 )
 
 func main() {
@@ -49,24 +51,25 @@ func main() {
 			}
 		}
 
-		if err := c.Web.StartServer(&srv); err != http.ErrServerClosed {
+		if err := c.Web.StartServer(&srv); errors.Is(err, http.ErrServerClosed) {
 			log.Fatalf("shutting down the server: %v", err)
 		}
 	}()
 
-	// Start the scheduler service to queue periodic tasks
-	go func() {
-		if err := c.Tasks.StartScheduler(); err != nil {
-			log.Fatalf("scheduler shutdown: %v", err)
-		}
-	}()
+	// Register all task queues
+	tasks.Register(c)
+
+	// Start the task runner to execute queued tasks
+	ctx, cancel := context.WithCancel(context.Background())
+	go c.Tasks.StartRunner(ctx)
 
-	// Wait for interrupt signal to gracefully shutdown the server with a timeout of 10 seconds.
+	// Wait for interrupt signal to gracefully shut down the server with a timeout of 10 seconds.
 	quit := make(chan os.Signal, 1)
 	signal.Notify(quit, os.Interrupt)
 	signal.Notify(quit, os.Kill)
 	<-quit
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	cancel()
+	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	if err := c.Web.Shutdown(ctx); err != nil {
 		log.Fatal(err)

config/config.go

@@ -57,6 +57,7 @@ type (
 		App      AppConfig
 		Cache    CacheConfig
 		Database DatabaseConfig
+		Tasks    TasksConfig
 		Mail     MailConfig
 	}
 
@@ -89,25 +90,25 @@ type (
 
 	// CacheConfig stores the cache configuration
 	CacheConfig struct {
-		Hostname     string
-		Port         uint16
-		Password     string
-		Database     int
-		TestDatabase int
-		Expiration   struct {
+		Capacity   int
+		Expiration struct {
 			StaticFile time.Duration
 			Page       time.Duration
 		}
 	}
 
 	// DatabaseConfig stores the database configuration
 	DatabaseConfig struct {
-		Hostname     string
-		Port         uint16
-		User         string
-		Password     string
-		Database     string
-		TestDatabase string
+		Driver         string
+		Connection     string
+		TestConnection string
+	}
+
+	// TasksConfig stores the tasks configuration
+	TasksConfig struct {
+		PollInterval time.Duration
+		MaxRetries   int
+		Goroutines   int
 	}
 
 	// MailConfig stores the mail configuration

config/config.yaml

@@ -21,22 +21,20 @@ app:
   emailVerificationTokenExpiration: "12h"
 
 cache:
-  hostname: "localhost"
-  port: 6379
-  password: ""
-  database: 0
-  testDatabase: 1
+  capacity: 100000
   expiration:
     staticFile: "4380h"
     page: "24h"
 
 database:
-  hostname: "localhost"
-  port: 5432
-  user: "admin"
-  password: "admin"
-  database: "app"
-  testDatabase: "app_test"
+  driver: "sqlite3"
+  connection: "dbs/main.db?_journal=WAL&_timeout=5000&_fk=true"
+  testConnection: ":memory:?_journal=WAL&_timeout=5000&_fk=true"
+
+tasks:
+  pollInterval: "1s"
+  maxRetries: 10
+  goroutines: 1
 
 mail:
   hostname: "localhost"

go.mod

@@ -8,17 +8,15 @@ require (
 	entgo.io/ent v0.13.1
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/PuerkitoBio/goquery v1.9.1
-	github.com/eko/gocache/lib/v4 v4.1.6
-	github.com/eko/gocache/store/redis/v4 v4.2.1
 	github.com/go-playground/validator/v10 v10.19.0
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/gorilla/context v1.1.2
 	github.com/gorilla/sessions v1.2.2
-	github.com/hibiken/asynq v0.24.1
-	github.com/jackc/pgx/v4 v4.18.3
 	github.com/labstack/echo/v4 v4.12.0
 	github.com/labstack/gommon v0.4.2
-	github.com/redis/go-redis/v9 v9.5.1
+	github.com/maragudk/goqite v0.2.3
+	github.com/mattn/go-sqlite3 v1.14.22
+	github.com/maypok86/otter v1.2.1
 	github.com/spf13/viper v1.18.2
 	github.com/stretchr/testify v1.9.0
 	golang.org/x/crypto v0.22.0
@@ -31,31 +29,21 @@ require (
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/dolthub/maphash v0.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gammazero/deque v0.2.1 // indirect
 	github.com/go-openapi/inflect v0.21.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/golang/mock v1.6.0 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/hcl/v2 v2.20.1 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
-	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
-	github.com/jackc/pgconn v1.14.3 // indirect
-	github.com/jackc/pgio v1.0.0 // indirect
-	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.3.3 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
-	github.com/jackc/pgtype v1.14.3 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -66,11 +54,7 @@ require (
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.19.0 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.53.0 // indirect
-	github.com/prometheus/procfs v0.14.0 // indirect
-	github.com/robfig/cron/v3 v3.0.1 // indirect
+	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
@@ -80,8 +64,6 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
-	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
-	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/zclconf/go-cty v1.14.4 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
@@ -92,7 +74,7 @@ require (
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.20.0 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )