ci: unify local and ci docker workflows (#907)

fmartingr · web-flow · commit 647945c54e71 · 2024-05-14T08:01:52.000+02:00
* deps: go to 1.22.3

* removed ci dockerfile

* add make buildx

* updated dockerfile

* ci uses make buildx command

* commented upx for the future

* disable openbsd/arm tests

* wip

* put dist file in dist path

* removed unused make command

* build-local to speed up local tests

* don't clean when buildx

* podman workaround

* manually define source files for tests
diff --git a/.github/workflows/_buildx.yml b/.github/workflows/_buildx.yml
@@ -14,15 +14,7 @@ jobs:
       - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # 4.1.7
         with:
           name: dist
-      - name: Prep binaries
-        working-directory: .github/workflows/docker
-        run: |
-          mkdir binaries
-          cp -r ../../../shiori_linux_* binaries/
-          mv binaries/shiori_linux_arm_7 binaries/shiori_linux_arm
-          mv binaries/shiori_linux_amd64_v1 binaries/shiori_linux_amd64
-          gzip -d -S binaries/.gz__  -r .
-          chmod 755 binaries/shiori_linux_*/shiori
+          path: ./dist
 
       # Every pull request that goes into master
       - name: Prepare master push tags
@@ -55,9 +47,7 @@ jobs:
           echo "tag_flags=--tag $REPO:pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV
 
       - name: Buildx
-        working-directory: .github/workflows/docker
         run: |
           set -x
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login -u "${{ github.repository_owner }}" --password-stdin ghcr.io
-          docker buildx create --use --name builder
-          docker buildx build -f Dockerfile.ci --platform=linux/amd64,arm64,linux/arm/v7 --push ${{ env.tag_flags }} .
+          make buildx CONTAINER_BUILDX_OPTIONS="--push ${{ env.tag_flags }}"
diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml
@@ -107,9 +107,6 @@ jobs:
             architecture: x86-64
             version: '7.5'
 
-          - name: openbsd
-            architecture: arm64
-            version: '7.5'
     steps:
       - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 
diff --git a/.github/workflows/docker/Dockerfile.ci b/.github/workflows/docker/Dockerfile.ci
diff --git a/.github/workflows/docker/etc/group b/.github/workflows/docker/etc/group
diff --git a/.github/workflows/docker/etc/passwd b/.github/workflows/docker/etc/passwd
diff --git a/.gitignore b/.gitignore
@@ -22,3 +22,6 @@ dist/
 
 # frontend
 node_modules
+
+# workaround for buildx using podman
+type=docker
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
@@ -36,6 +36,15 @@ archives:
     - goos: windows
       format: zip
 
+# TODO:
+# upx:
+#   - enabled: true
+#     ids:
+#       - shiori
+#     goos: [linux, darwin]
+#     goarch: [amd64, arm, arm64]
+#     goarm: ["7"]
+
 checksum:
   name_template: 'checksums.txt'
 snapshot:
diff --git a/Dockerfile b/Dockerfile
@@ -1,19 +1,30 @@
-# build stage
-FROM ghcr.io/ghcri/golang:1.21-alpine3.19 AS builder
-WORKDIR /src
-COPY . .
-RUN go build -ldflags '-s -w'
+# Build stage
+ARG ALPINE_VERSION
+ARG GOLANG_VERSION
 
-# server image
+FROM docker.io/library/alpine:${ALPINE_VERSION} AS builder
+ARG TARGETARCH
+ARG TARGETOS
+ARG TARGETVARIANT
+COPY dist/shiori_${TARGETOS}_${TARGETARCH}${TARGETVARIANT}/shiori /usr/bin/shiori
+RUN apk add --no-cache ca-certificates tzdata && \
+    chmod +x /usr/bin/shiori
+
+# Server image
+FROM scratch
+
+ENV PORT 8080
+ENV SHIORI_DIR=/shiori
+WORKDIR ${SHIORI_DIR}
+
+LABEL org.opencontainers.image.source="https://github.com/go-shiori/shiori"
+LABEL maintainer="Felipe Martin <github@fmartingr.com>"
+
+COPY --from=builder /usr/bin/shiori /usr/bin/shiori
+COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
+EXPOSE ${PORT}
 
-FROM docker.io/alpine:3.19
-LABEL org.opencontainers.image.source https://github.com/go-shiori/shiori
-COPY --from=builder /src/shiori /usr/bin/
-RUN addgroup -g 1000 shiori \
-    && adduser -D -h /shiori -g '' -G shiori -u 1000 shiori
-USER shiori
-WORKDIR /shiori
-EXPOSE 8080
-ENV SHIORI_DIR /shiori/
 ENTRYPOINT ["/usr/bin/shiori"]
 CMD ["server"]
diff --git a/Makefile b/Makefile
@@ -1,8 +1,10 @@
 GO ?= $(shell command -v go 2> /dev/null)
 BASH ?= $(shell command -v bash 2> /dev/null)
+GOLANG_VERSION := $(shell head -n 4 go.mod | tail -n 1 | cut -d " " -f 2)
 
 # Development
 SHIORI_DIR ?= dev-data
+SOURCE_FILES ?=./internal/...
 
 # Build
 CGO_ENABLED ?= 0
@@ -11,6 +13,16 @@ BUILD_HASH := $(shell git describe --tags)
 BUILD_TAGS ?= osusergo,netgo,fts5
 LDFLAGS += -s -w -X main.version=$(BUILD_HASH) -X main.date=$(BUILD_TIME)
 
+# Build (container)
+CONTAINER_RUNTIME := docker
+CONTAINERFILE_NAME := Dockerfile
+CONTAINER_ALPINE_VERSION := 3.19
+BUILDX_PLATFORMS := linux/amd64,linux/arm64,linux/arm/v7
+
+# This is used for local development only, forcing linux to create linux only images but with the arch
+# of the running machine. Far from perfect but works.
+LOCAL_BUILD_PLATFORM = linux/$(shell go env GOARCH)
+
 # Testing
 GO_TEST_FLAGS ?= -v -race -count=1 -tags $(BUILD_TAGS) -covermode=atomic -coverprofile=coverage.out
 GOTESTFMT_FLAGS ?=
@@ -26,6 +38,15 @@ SWAGGER_DOCS_PATH ?= ./docs/swagger
 # Frontend
 CLEANCSS_OPTS ?= --with-rebase
 
+# Common exports
+export GOLANG_VERSION
+export CONTAINER_RUNTIME
+export CONTAINERFILE_NAME
+export CONTAINER_ALPINE_VERSION
+export BUILDX_PLATFORMS
+
+export SOURCE_FILES
+
 # Help documentatin à la https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
 .PHONY: help
 help:
@@ -87,12 +108,29 @@ styles-check:
 ## Build binary
 .PHONY: build
 build: clean
-	GIN_MODE=$(GIN_MODE) goreleaser build --rm-dist --snapshot
+	GIN_MODE=$(GIN_MODE) goreleaser build --clean --snapshot
+
+## Build binary for current targer
+build-local: clean
+	GIN_MODE=$(GIN_MODE) goreleaser build --clean --snapshot --single-target
+
+## Build docker image using Buildx.
+# used for multi-arch builds suing mainly the CI, that's why the task does not
+# build the binaries using a dependency task.
+.PHONY: buildx
+buildx:
+	$(info: Make: Buildx)
+	@bash scripts/buildx.sh
+
+## Build docker image for local development
+buildx-local: build-local
+	$(info: Make: Build image locally)
+	CONTAINER_BUILDX_OPTIONS="-t shiori:localdev --output type=docker" BUILDX_PLATFORMS=$(LOCAL_BUILD_PLATFORM) scripts/buildx.sh
 
 ## Creates a coverage report
 .PHONY: coverage
 coverage:
-	$(GO) test $(GO_TEST_FLAGS) -coverprofile=coverage.txt ./...
+	$(GO) test $(GO_TEST_FLAGS) -coverprofile=coverage.txt $(SOURCE_FILES)
 	$(GO) tool cover -html=coverage.txt
 
 ## Run generate accross the project
diff --git a/go.mod b/go.mod
@@ -1,7 +1,7 @@
 module github.com/go-shiori/shiori
 
 // +heroku goVersion go1.22
-go 1.22.2
+go 1.22.3
 
 require (
 	git.sr.ht/~emersion/go-sqlite3-fts5 v0.0.0-20230217131031-f2c8767594fc
diff --git a/scripts/buildx.sh b/scripts/buildx.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -ex
+
+# Check if the shiori_builder builder exists
+if [ "$CONTAINER_RUNTIME" == "docker" ]; then
+    if [ -z "$($CONTAINER_RUNTIME buildx ls | grep shiori_builder)" ]; then
+        echo "Creating shiori_builder builder"
+        $CONTAINER_RUNTIME buildx create --use --name shiori_builder
+    fi
+fi
+
+if [[ -d "dist/shiori_linux_arm_7" ]]; then
+    cp -r dist/shiori_linux_arm_7 dist/shiori_linux_armv7
+fi
+
+if [[ -d "dist/shiori_linux_amd64_v1" ]]; then
+    cp -r dist/shiori_linux_amd64_v1 dist/shiori_linux_amd64
+fi
+
+$CONTAINER_RUNTIME buildx build \
+    -f ${CONTAINERFILE_NAME} \
+    --platform=${BUILDX_PLATFORMS} \
+    --build-arg "ALPINE_VERSION=${CONTAINER_ALPINE_VERSION}" \
+    --build-arg "GOLANG_VERSION=${GOLANG_VERSION}" \
+    ${CONTAINER_BUILDX_OPTIONS} \
+    .
+
+if [ "$CONTAINER_RUNTIME" == "docker" ]; then
+    $CONTAINER_RUNTIME buildx rm shiori_builder
+fi
diff --git a/scripts/test.sh b/scripts/test.sh
@@ -8,7 +8,7 @@ fi
 # if gotestfmt is installed, run with it
 if [ -x "$(command -v gotestfmt)" ]; then
     set -o pipefail
-    go test ./... ${GO_TEST_FLAGS} -json | gotestfmt ${GOTESTFMT_FLAGS}
+    go test ${SOURCE_FILES} ${GO_TEST_FLAGS} -json | gotestfmt ${GOTESTFMT_FLAGS}
 else
-    go test ./... ${GO_TEST_FLAGS}
+    go test ${SOURCE_FILES} ${GO_TEST_FLAGS}
 fi
