Accounts table, DAO sqlx implementation, seeding

Author: lzap

.editorconfig

@@ -0,0 +1,23 @@
+; https://editorconfig.org/
+
+root = true
+
+[*]
+insert_final_newline = true
+charset = utf-8
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 2
+
+[{Makefile,go.mod,go.sum,*.go,.gitmodules}]
+indent_style = tab
+indent_size = 4
+
+[*.md]
+indent_size = 4
+trim_trailing_whitespace = false
+
+eclint_indent_style = unset
+
+[Dockerfile]
+indent_size = 4

.github/workflows/gotest.yml

@@ -9,8 +9,8 @@ jobs:
     name: go fmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
         with:
           go-version: "1.16"
       - uses: Jerome1337/[email protected]
@@ -19,8 +19,8 @@ jobs:
     name: go vet
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
         with:
           go-version: "1.16"
       - run: |
@@ -30,14 +30,14 @@ jobs:
     name: golangci-lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
         with:
           go-version: "1.16"
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
-          version: latest
+          version: v1.45.2
           only-new-issues: true
           skip-go-installation: true
 
@@ -50,4 +50,4 @@ jobs:
         with:
           go-version: "1.16"
       - run: |
-          go test ./...
+          go test ./...

.golangci.yml

@@ -11,8 +11,18 @@ linters:
     - sql
     - module
     - unused
+  disable:
+    - maligned # deprecated by fieldalignment
+    - scopelint # deprecated by exportloopref
 issues:
   exclude-rules:
+    # If a Stmt is prepared on a DB, it will remain usable for the lifetime of the DB. When the Stmt needs to execute
+    # on a new underlying connection, it will prepare itself on the new connection automatically.
+    - path: 'internal/dao/sqlx/(.+)\.go'
+      linters:
+        - sqlclosecheck
+
+    # easier to write test code
     - path: '(.+)_test\.go'
       linters:
-        - forcetypeassert
+        - forcetypeassert

Dockerfile

@@ -11,4 +11,4 @@ COPY --from=build /build/pbmigrate /pbmigrate
 USER 1001
 EXPOSE 8000
 EXPOSE 5000
-ENTRYPOINT ["/pbapi"]
+ENTRYPOINT ["/pbapi"]

README.md

@@ -52,7 +52,9 @@ Migrations can be found in `internal/db/migrations` in SQL format, the only supp
 make generate-migration MIGRATION_NAME=add_new_column
 ```
 
-To execute migration either use [migrate](https://github.com/golang-migrate/migrate/tree/master/cmd/migrate) command line utility from the golang-migrate library which supports more advanced operations like down, or run `pbmigrate` to perform up migration the same way as the application does when it boots up.
+We currently do not allow down migrations, so delete the `down.sql` file and do not commit it into git (it will fail build).
+
+To apply migrations, build and run `pbmigrate` binary. This is exactly what application performs during startup. The `pbmigrate` utility also supports seeding initial data. There are files available in `internal/db/seeds` with various seed configurations. Feel free to create your own configuration which you may or may not want to commit into git. When you set `DB_SEED_SCRIPT` configuration variable, the migration tool will execute all statements from that file. By default, the variable is empty, meaning no data will be seeded.
 
 ## Building container
 
@@ -73,16 +75,22 @@ Here are few points before you start contributing:
 * All configuration is done via environment variables.
 * Database database models (structs) do belong into `internal/models`.
 * Do not put any code logic into database models.
-* All database operations (CRUD, eager loading) lives in `internal/dao` (Data Access Objects).
+* All database operations (CRUD, eager loading) lives in `internal/dao` (Data Access Objects) with a common API interface.
+* The actual implementation lives in `internal/dao/sqlx` package, all operations are passed with Context and errors are wrapped.
 * Database models must be not exposed directly into JSON API, use `internal/payloads` package to wrap them.
 * Business logic (the actual code) does belong into `internal/services` package, each API call should have a dedicated file.
 * HTTP routes go into `internal/routes` package.
 * HTTP middleware go into `internal/middleware` package.
-* Monitoring metrics are in `internal/metrics' package.`
-* Use the standard library context package for context operations. Context keys must be defined in `internal/ctxval`.
-* Database connection is at `internal/db`, cloud connections are in 'internal/clouds'.
-* Do not introduce `utils` or `tools` common packages, they tend to grow.
+* Monitoring metrics are in `internal/metrics` package.
+* Use the standard library context package for context operations. Context keys are defined in `internal/ctxval` as well as accessor functions.
+* Database connection is at `internal/db`, cloud connections are in `internal/clouds`.
+* Do not introduce `utils` or `tools` common packages.
 * Keep the line of sight (happy code path).
+* PostgreSQL version we currently use in production is v14+ so take advantage of all modern SQL features.
+* Use `BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY` for primary keys.
+* Use `TEXT` for string columns, do not apply any limits in the DB: https://wiki.postgresql.org/wiki/Don%27t_Do_This#Text_storage
+
+Keep security in mind: https://github.com/RedHatInsights/secure-coding-checklist
 
 ## License
 

build/ci/.keep

@@ -0,0 +1 @@
+

build/package/.keep

@@ -0,0 +1 @@
+

cmd/pbapi/main.go

@@ -1,6 +1,9 @@
 package main
 
 import (
+	// Performs initialization of DAO implementation, must be initialized before any database packages.
+	_ "github.com/RHEnVision/provisioning-backend/internal/dao/sqlx"
+
 	"context"
 	"errors"
 	"fmt"

internal/config/config.go

@@ -17,6 +17,7 @@ var config struct {
 		Name        string
 		User        string
 		Password    string
+		SeedScript  string
 		MaxIdleTime time.Duration
 		MaxLifetime time.Duration
 		MaxOpenConn int

internal/ctxval/ctx_getters.go

@@ -0,0 +1,31 @@
+package ctxval
+
+import (
+	"context"
+	"github.com/rs/zerolog"
+)
+
+func GetStringValue(ctx context.Context, key CommonKeyId) string {
+	return ctx.Value(key).(string)
+}
+
+func GetUInt64Value(ctx context.Context, key CommonKeyId) uint64 {
+	return ctx.Value(key).(uint64)
+}
+
+// GetLogger returns logger or nil when not in the context
+func GetLogger(ctx context.Context) *zerolog.Logger {
+	if ctx.Value(LoggerCtxKey) == nil {
+		return nil
+	}
+	logger := ctx.Value(LoggerCtxKey).(zerolog.Logger)
+	return &logger
+}
+
+// GetRequestId returns request id or an empty string when not in the context
+func GetRequestId(ctx context.Context) string {
+	if ctx.Value(RequestIdCtxKey) == nil {
+		return ""
+	}
+	return ctx.Value(RequestIdCtxKey).(string)
+}

internal/ctxval/ctx_keys.go

@@ -0,0 +1,10 @@
+package ctxval
+
+type CommonKeyId int
+
+const (
+	LoggerCtxKey     CommonKeyId = iota
+	RequestIdCtxKey  CommonKeyId = iota
+	RequestNumCtxKey CommonKeyId = iota
+	ResourceCtxKey   CommonKeyId = iota
+)

internal/ctxval/keys.go

@@ -0,0 +1,21 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+)
+
+// Error type for all DAO errors.
+type Error struct {
+	Message string
+	Context context.Context
+	Err     error
+}
+
+func (e *Error) Error() string {
+	return fmt.Sprintf("DAO error: %s: %s", e.Message, e.Err.Error())
+}
+
+func (e *Error) Unwrap() error {
+	return e.Err
+}

internal/dao/dao_errors.go

@@ -0,0 +1,15 @@
+package dao
+
+import (
+	"context"
+	"github.com/RHEnVision/provisioning-backend/internal/models"
+)
+
+var GetAccountDao func(ctx context.Context) (AccountDao, error)
+
+type AccountDao interface {
+	GetById(ctx context.Context, id uint64) (*models.Account, error)
+	GetByAccountNumber(ctx context.Context, number string) (*models.Account, error)
+	GetByOrgId(ctx context.Context, orgId string) (*models.Account, error)
+	List(ctx context.Context, limit, offset uint64) ([]*models.Account, error)
+}

internal/dao/dao_interfaces.go

@@ -0,0 +1,87 @@
+package sqlx
+
+import (
+	"context"
+	"github.com/RHEnVision/provisioning-backend/internal/dao"
+	"github.com/RHEnVision/provisioning-backend/internal/db"
+	"github.com/RHEnVision/provisioning-backend/internal/models"
+	"github.com/jmoiron/sqlx"
+)
+
+const (
+	getById            = `SELECT * FROM accounts WHERE id = $1 LIMIT 1`
+	getByAccountNumber = `SELECT * FROM accounts WHERE account_number = $1 LIMIT 1`
+	getByOrgId         = `SELECT * FROM accounts WHERE org_id = $1 LIMIT 1`
+	list               = `SELECT * FROM accounts ORDER BY id LIMIT $1 OFFSET $2`
+)
+
+type accountDaoSqlx struct {
+	getById            *sqlx.Stmt
+	getByAccountNumber *sqlx.Stmt
+	getByOrgId         *sqlx.Stmt
+	list               *sqlx.Stmt
+}
+
+func getAccountDao(ctx context.Context) (dao.AccountDao, error) {
+	var err error
+	daoImpl := accountDaoSqlx{}
+
+	daoImpl.getById, err = db.DB.PreparexContext(ctx, getById)
+	if err != nil {
+		return nil, NewPrepareStatementError(ctx, getById, err)
+	}
+	daoImpl.getByAccountNumber, err = db.DB.PreparexContext(ctx, getByAccountNumber)
+	if err != nil {
+		return nil, NewPrepareStatementError(ctx, list, err)
+	}
+	daoImpl.getByOrgId, err = db.DB.PreparexContext(ctx, getByOrgId)
+	if err != nil {
+		return nil, NewPrepareStatementError(ctx, list, err)
+	}
+	daoImpl.list, err = db.DB.PreparexContext(ctx, list)
+	if err != nil {
+		return nil, NewPrepareStatementError(ctx, list, err)
+	}
+
+	return &daoImpl, nil
+}
+
+func init() {
+	dao.GetAccountDao = getAccountDao
+}
+
+func (a *accountDaoSqlx) GetById(ctx context.Context, id uint64) (*models.Account, error) {
+	result := &models.Account{}
+	err := a.getById.GetContext(ctx, result, id)
+	if err != nil {
+		return nil, NewGetError(ctx, "get by id", err)
+	}
+	return result, nil
+}
+
+func (a *accountDaoSqlx) GetByAccountNumber(ctx context.Context, number string) (*models.Account, error) {
+	result := &models.Account{}
+	err := a.getByAccountNumber.GetContext(ctx, result, number)
+	if err != nil {
+		return nil, NewGetError(ctx, "get by id", err)
+	}
+	return result, nil
+}
+
+func (a *accountDaoSqlx) GetByOrgId(ctx context.Context, orgId string) (*models.Account, error) {
+	result := &models.Account{}
+	err := a.getByOrgId.GetContext(ctx, result, orgId)
+	if err != nil {
+		return nil, NewGetError(ctx, "get by id", err)
+	}
+	return result, nil
+}
+
+func (a *accountDaoSqlx) List(ctx context.Context, limit, offset uint64) ([]*models.Account, error) {
+	var result []*models.Account
+	err := a.list.SelectContext(ctx, &result, limit, offset)
+	if err != nil {
+		return nil, NewGetError(ctx, "list", err)
+	}
+	return result, nil
+}

internal/dao/sqlx/account_dao.go

@@ -0,0 +1,29 @@
+package sqlx
+
+import (
+	"context"
+	"github.com/RHEnVision/provisioning-backend/internal/ctxval"
+	"github.com/RHEnVision/provisioning-backend/internal/dao"
+)
+
+func NewPrepareStatementError(context context.Context, sql string, err error) *dao.Error {
+	if logger := ctxval.GetLogger(context); logger != nil {
+		logger.Error().Msgf("sqlx prepare statement error: %s: %v", sql, err)
+	}
+	return &dao.Error{
+		Message: sql,
+		Context: context,
+		Err:     err,
+	}
+}
+
+func NewGetError(context context.Context, msg string, err error) *dao.Error {
+	if logger := ctxval.GetLogger(context); logger != nil {
+		logger.Error().Msgf("sqlx get error: %s: %v", msg, err)
+	}
+	return &dao.Error{
+		Message: msg,
+		Context: context,
+		Err:     err,
+	}
+}