Skip to content

Commit 2603cc8

Browse files
azeemshaikh38azeemshaikh38
committed
Modify tool installation
Signed-off-by: Azeem Shaikh <azeemshaikh38@gmail.com>
1 parent 0f87094 commit 2603cc8

File tree

1 file changed

+173
-98
lines changed

1 file changed

+173
-98
lines changed

‎Makefile‎

Lines changed: 173 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,11 @@
11
SHELL := /bin/bash
2-
GINKGO := ginkgo
32
GIT_HASH := $(shell git rev-parse HEAD)
43
GIT_VERSION ?= $(shell git describe --tags --always --dirty)
54
SOURCE_DATE_EPOCH=$(shell git log --date=iso8601-strict -1 --pretty=%ct)
6-
GOLANGCI_LINT := golangci-lint
7-
PROTOC_GEN_GO := protoc-gen-go
8-
MOCKGEN := mockgen
9-
PROTOC := $(shell which protoc)
10-
GORELEASER := goreleaser
115
IMAGE_NAME = scorecard
126
OUTPUT = output
137
PLATFORM="linux/amd64,linux/arm64,linux/386,linux/arm"
148
LDFLAGS=$(shell ./scripts/version-ldflags)
15-
KOCACHE_PATH=/tmp/ko
16-
17-
define create_kocache_path
18-
mkdir -p $(KOCACHE_PATH)
19-
endef
209

2110

2211

@@ -30,24 +19,72 @@ help: ## Display this help
3019

3120
###############################################################################
3221

33-
##@ Development
22+
##@ Tools
3423
################################ make install #################################
35-
.PHONY: install
36-
install: ## Installs all dependencies needed to compile Scorecard
37-
install: | $(PROTOC)
38-
@echo Installing tools from tools/tools.go
39-
cd tools; cat tools.go | grep _ | awk -F'"' '{print $$2}' | xargs -tI % go install %
24+
TOOLS_DIR := tools
25+
TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/bin)
26+
GOBIN := $(shell go env GOBIN)
27+
28+
# Golang binaries.
29+
30+
GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint
31+
$(GOLANGCI_LINT): $(TOOLS_DIR)/go.mod
32+
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint
33+
34+
KO := $(TOOLS_BIN_DIR)/ko
35+
$(KO): $(TOOLS_DIR)/go.mod
36+
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/google/ko
37+
38+
STUNNING_TRIBBLE := $(TOOLS_BIN_DIR)/stunning-tribble
39+
$(STUNNING_TRIBBLE): $(TOOLS_DIR)/go.mod
40+
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/naveensrinivasan/stunning-tribble
4041

42+
MOCKGEN := $(TOOLS_BIN_DIR)/mockgen
43+
$(MOCKGEN): $(TOOLS_DIR)/go.mod
44+
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/golang/mock/mockgen
45+
46+
GINKGO := $(TOOLS_BIN_DIR)/ginkgo
47+
$(GINKGO): $(TOOLS_DIR)/go.mod
48+
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/onsi/ginkgo/v2/ginkgo
49+
50+
GORELEASER := $(TOOLS_BIN_DIR)/goreleaser
51+
$(GORELEASER): $(TOOLS_DIR)/go.mod
52+
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install github.com/goreleaser/goreleaser
53+
54+
PROTOC_GEN_GO := $(TOOLS_BIN_DIR)/protoc-gen-go
55+
$(PROTOC_GEN_GO): $(TOOLS_DIR)/go.mod
56+
cd $(TOOLS_DIR); GOBIN=$(TOOLS_BIN_DIR) go install google.golang.org/protobuf/cmd/protoc-gen-go
57+
58+
# Non-Golang binaries.
59+
# TODO: Figure out how to install these binaries automatically.
60+
61+
PROTOC := $(shell which protoc)
4162
$(PROTOC):
4263
ifeq (,$(PROTOC))
4364
$(error download and install protobuf compiler package - https://developers.google.com/protocol-buffers/docs/downloads)
4465
endif
66+
67+
# Installs required binaries into $(TOOLS_BIN_DIR) wherever possible.
68+
# Keeping a local copy instead of a global install allows for:
69+
# i) Controlling the binary version Scorecard depends on leading to consistent
70+
# behavior across users.
71+
# ii) Avoids installing a whole bunch of otherwise unnecessary tools in the user's workspace.
72+
.PHONY: install
73+
install: ## Installs required binaries.
74+
install: $(GOLANGCI_LINT) \
75+
$(KO) \
76+
$(STUNNING_TRIBBLE) \
77+
$(PROTOC_GEN_GO) $(PROTOC) \
78+
$(MOCKGEN) \
79+
$(GINKGO) \
80+
$(GORELEASER)
81+
4582
###############################################################################
4683

4784
##@ Build
4885
################################## make all ###################################
4986
all: ## Runs build, test and verify
50-
all-targets = build check-linter check-osv validate-docs add-projects validate-projects
87+
all-targets = build check-linter check-osv validate-docs add-projects validate-projects
5188
.PHONY: all all-targets-update-dependencies $(all-targets) update-dependencies tree-status
5289
all-targets-update-dependencies: $(all-targets) | update-dependencies
5390
all: update-dependencies all-targets-update-dependencies tree-status
@@ -58,23 +95,22 @@ update-dependencies: ## Update go dependencies for all modules
5895
cd tools; go mod tidy && go mod verify; cd ../
5996
cd attestor; go mod tidy && go mod verify; cd ../
6097

61-
$(GOLANGCI_LINT): install
6298
check-linter: ## Install and run golang linter
63-
check-linter: $(GOLANGCI_LINT)
99+
check-linter: | $(GOLANGCI_LINT)
64100
# Run golangci-lint linter
65-
golangci-lint run -c .golangci.yml
101+
$(GOLANGCI_LINT) run -c .golangci.yml
66102

67103
check-osv: ## Checks osv.dev for any vulnerabilities
68-
check-osv: $(install)
104+
check-osv: | $(STUNNING_TRIBBLE)
69105
# Run stunning-tribble for checking the dependencies have any OSV
70106
go list -m -f '{{if not (or .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \
71-
| stunning-tribble
107+
| $(STUNNING_TRIBBLE)
72108
# Checking the tools which also has go.mod
73109
cd tools; go list -m -f '{{if not (or .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \
74-
| stunning-tribble ; cd ..
110+
| $(STUNNING_TRIBBLE) ; cd ..
75111
# Checking the attestor module for vulns
76112
cd attestor; go list -m -f '{{if not (or .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \
77-
| stunning-tribble ; cd ..
113+
| $(STUNNING_TRIBBLE) ; cd ..
78114

79115
add-projects: ## Adds new projects to ./cron/internal/data/projects.csv
80116
add-projects: ./cron/internal/data/projects.csv | build-add-script
@@ -99,35 +135,40 @@ build-cron: build-controller build-worker build-cii-worker \
99135
build-shuffler build-bq-transfer build-github-server \
100136
build-webhook build-add-script build-validate-script build-update-script
101137

102-
build-targets = generate-mocks generate-docs build-proto build-scorecard build-cron
138+
build-targets = generate-mocks generate-docs build-scorecard build-cron build-proto
103139
.PHONY: build $(build-targets)
104140
build: ## Build all binaries and images in the repo.
105141
build: $(build-targets)
106142

107143
build-proto: ## Compiles and generates all required protobufs
108144
build-proto: cron/internal/data/request.pb.go cron/internal/data/metadata.pb.go
109-
cron/internal/data/request.pb.go: cron/internal/data/request.proto | $(PROTOC) install
110-
protoc --go_out=../../../ cron/internal/data/request.proto
111-
cron/internal/data/metadata.pb.go: cron/internal/data/metadata.proto | $(PROTOC) install
112-
protoc --go_out=../../../ cron/internal/data/metadata.proto
145+
cron/internal/data/request.pb.go: cron/internal/data/request.proto | $(PROTOC) $(PROTOC_GEN_GO)
146+
PATH=$PATH:$(TOOLS_BIN_DIR) GOBIN=$(TOOLS_BIN_DIR) $(PROTOC) --go_out=../../../ cron/internal/data/request.proto
147+
cron/internal/data/metadata.pb.go: cron/internal/data/metadata.proto | $(PROTOC) $(PROTOC_GEN_GO)
148+
PATH=$PATH:$(TOOLS_BIN_DIR) GOBIN=$(TOOLS_BIN_DIR) $(PROTOC) --go_out=../../../ cron/internal/data/metadata.proto
113149

114150
generate-mocks: ## Compiles and generates all mocks using mockgen.
115-
generate-mocks: clients/mockclients/repo_client.go clients/mockclients/repo.go clients/mockclients/cii_client.go checks/mockclients/vulnerabilities.go cmd/packagemanager_mockclient.go
116-
clients/mockclients/repo_client.go: clients/repo_client.go
151+
generate-mocks: clients/mockclients/repo_client.go \
152+
clients/mockclients/repo.go \
153+
clients/mockclients/cii_client.go \
154+
checks/mockclients/vulnerabilities.go \
155+
cmd/packagemanager_mockclient.go
156+
clients/mockclients/repo_client.go: clients/repo_client.go | $(MOCKGEN)
117157
# Generating MockRepoClient
118158
$(MOCKGEN) -source=clients/repo_client.go -destination=clients/mockclients/repo_client.go -package=mockrepo -copyright_file=clients/mockclients/license.txt
119-
clients/mockclients/repo.go: clients/repo.go
159+
clients/mockclients/repo.go: clients/repo.go | $(MOCKGEN)
120160
# Generating MockRepo
121161
$(MOCKGEN) -source=clients/repo.go -destination=clients/mockclients/repo.go -package=mockrepo -copyright_file=clients/mockclients/license.txt
122-
clients/mockclients/cii_client.go: clients/cii_client.go
162+
clients/mockclients/cii_client.go: clients/cii_client.go | $(MOCKGEN)
123163
# Generating MockCIIClient
124164
$(MOCKGEN) -source=clients/cii_client.go -destination=clients/mockclients/cii_client.go -package=mockrepo -copyright_file=clients/mockclients/license.txt
125-
checks/mockclients/vulnerabilities.go: clients/vulnerabilities.go
165+
checks/mockclients/vulnerabilities.go: clients/vulnerabilities.go | $(MOCKGEN)
126166
# Generating MockCIIClient
127167
$(MOCKGEN) -source=clients/vulnerabilities.go -destination=clients/mockclients/vulnerabilities.go -package=mockrepo -copyright_file=clients/mockclients/license.txt
128-
cmd/packagemanager_mockclient.go: cmd/packagemanager_client.go
168+
cmd/packagemanager_mockclient.go: cmd/packagemanager_client.go | $(MOCKGEN)
129169
# Generating MockPackageManagerClient
130170
$(MOCKGEN) -source=cmd/packagemanager_client.go -destination=cmd/packagemanager_mockclient.go -package=cmd -copyright_file=clients/mockclients/license.txt
171+
131172
generate-docs: ## Generates docs
132173
generate-docs: validate-docs docs/checks.md
133174
docs/checks.md: docs/checks/internal/checks.yaml docs/checks/internal/*.go docs/checks/internal/generate/*.go
@@ -196,66 +237,6 @@ cron/internal/data/update/projects-update: cron/internal/data/update/*.go cron/
196237
# Run go build on the update script
197238
cd cron/internal/data/update && CGO_ENABLED=0 go build -trimpath -a -tags netgo -ldflags '$(LDFLAGS)' -o projects-update
198239

199-
ko-targets = scorecard-ko cron-controller-ko cron-worker-ko cron-cii-worker-ko cron-bq-transfer-ko cron-webhook-ko cron-github-server-ko
200-
.PHONY: ko-build-everything $(ko-targets)
201-
ko-build-everything: $(ko-targets)
202-
203-
scorecard-ko:
204-
$(call create_kocache_path)
205-
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) LDFLAGS="$(LDFLAGS)" \
206-
KO_CACHE=$(KOCACHE_PATH) ko build -B \
207-
--sbom=none \
208-
--platform=$(PLATFORM)\
209-
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4
210-
cron-controller-ko:
211-
$(call_create_kocache_path)
212-
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-controller LDFLAGS="$(LDFLAGS)" \
213-
KOCACHE=$(KOCACHE_PATH) ko build -B \
214-
--push=false \
215-
--sbom=none \
216-
--platform=$(PLATFORM)\
217-
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/controller
218-
cron-worker-ko:
219-
$(call_create_kocache_path)
220-
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-worker LDFLAGS="$(LDFLAGS)" \
221-
KOCACHE=$(KOCACHE_PATH) ko build -B \
222-
--push=false \
223-
--sbom=none \
224-
--platform=$(PLATFORM)\
225-
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/worker
226-
cron-cii-worker-ko:
227-
$(call_create_kocache_path)
228-
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cii-worker LDFLAGS="$(LDFLAGS)" \
229-
KOCACHE=$(KOCACHE_PATH) ko build -B \
230-
--push=false \
231-
--sbom=none \
232-
--platform=$(PLATFORM)\
233-
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/cii
234-
cron-bq-transfer-ko:
235-
$(call_create_kocache_path)
236-
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-bq-transfer LDFLAGS="$(LDFLAGS)" \
237-
KOCACHE=$(KOCACHE_PATH) ko build -B \
238-
--push=false \
239-
--sbom=none \
240-
--platform=$(PLATFORM)\
241-
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/bq
242-
cron-webhook-ko:
243-
$(call_create_kocache_path)
244-
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cron-webhook LDFLAGS="$(LDFLAGS)" \
245-
KOCACHE=$(KOCACHE_PATH) ko build -B \
246-
--push=false \
247-
--sbom=none \
248-
--platform=$(PLATFORM)\
249-
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/webhook
250-
cron-github-server-ko:
251-
$(call_create_kocache_path)
252-
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-github-server LDFLAGS="$(LDFLAGS)" \
253-
KOCACHE=$(KOCACHE_PATH) ko build -B \
254-
--push=false \
255-
--sbom=none \
256-
--platform=$(PLATFORM)\
257-
--tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper/tokens/server
258-
259240
docker-targets = scorecard-docker cron-controller-docker cron-worker-docker cron-cii-worker-docker cron-bq-transfer-docker cron-webhook-docker cron-github-server-docker
260241
.PHONY: dockerbuild $(docker-targets)
261242
dockerbuild: $(docker-targets)
@@ -290,8 +271,6 @@ unit-test: ## Runs unit test without e2e
290271
unit-test-attestor: ## Runs unit tests on scorecard-attestor
291272
cd attestor; SKIP_GINKGO=1 go test -covermode=atomic -coverprofile=unit-coverage.out `go list ./...`; cd ..;
292273

293-
$(GINKGO): install
294-
295274
check-env:
296275
ifndef GITHUB_AUTH_TOKEN
297276
$(error GITHUB_AUTH_TOKEN is undefined)
@@ -307,3 +286,99 @@ e2e-gh-token: build-scorecard check-env | $(GINKGO)
307286
# Run e2e tests. GITHUB_AUTH_TOKEN set to secrets.GITHUB_TOKEN must be used to run this.
308287
TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./...
309288
###############################################################################
289+
290+
##@ TODO(#744)
291+
################################## make ko-images #############################
292+
ko-targets = scorecard-ko cron-controller-ko cron-worker-ko cron-cii-worker-ko cron-bq-transfer-ko cron-webhook-ko cron-github-server-ko
293+
.PHONY: ko-images $(ko-targets)
294+
ko-images: $(ko-targets)
295+
296+
KOCACHE_PATH=/tmp/ko
297+
298+
$(KOCACHE_PATH):
299+
mkdir -p $(KOCACHE_PATH)
300+
301+
scorecard-ko: | $(KO) $(KOCACHE_PATH)
302+
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
303+
KO_DOCKER_REPO=${KO_PREFIX}/${IMAGE_NAME}
304+
LDFLAGS="$(LDFLAGS)" \
305+
KO_CACHE=$(KOCACHE_PATH) \
306+
$(KO) build -B \
307+
--sbom=none \
308+
--platform=$(PLATFORM) \
309+
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
310+
github.com/ossf/scorecard/v4
311+
312+
cron-controller-ko: | $(KO) $(KOCACHE_PATH)
313+
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
314+
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-controller \
315+
LDFLAGS="$(LDFLAGS)" \
316+
KOCACHE=$(KOCACHE_PATH) \
317+
$(KO) build -B \
318+
--push=false \
319+
--sbom=none \
320+
--platform=$(PLATFORM) \
321+
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
322+
github.com/ossf/scorecard/v4/cron/internal/controller
323+
324+
cron-worker-ko: | $(KO) $(KOCACHE_PATH)
325+
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
326+
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-worker \
327+
LDFLAGS="$(LDFLAGS)" \
328+
KOCACHE=$(KOCACHE_PATH) \
329+
$(KO) build -B \
330+
--push=false \
331+
--sbom=none \
332+
--platform=$(PLATFORM) \
333+
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
334+
github.com/ossf/scorecard/v4/cron/internal/worker
335+
336+
cron-cii-worker-ko: | $(KO) $(KOCACHE_PATH)
337+
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
338+
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cii-worker \
339+
LDFLAGS="$(LDFLAGS)" \
340+
KOCACHE=$(KOCACHE_PATH) \
341+
$(KO) build -B \
342+
--push=false \
343+
--sbom=none \
344+
--platform=$(PLATFORM)\
345+
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
346+
github.com/ossf/scorecard/v4/cron/internal/cii
347+
348+
cron-bq-transfer-ko: | $(KO) $(KOCACHE_PATH)
349+
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
350+
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-bq-transfer \
351+
LDFLAGS="$(LDFLAGS)" \
352+
KOCACHE=$(KOCACHE_PATH) \
353+
$(KO) build -B \
354+
--push=false \
355+
--sbom=none \
356+
--platform=$(PLATFORM) \
357+
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
358+
github.com/ossf/scorecard/v4/cron/internal/bq
359+
360+
cron-webhook-ko: | $(KO) $(KOCACHE_PATH)
361+
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
362+
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cron-webhook \
363+
LDFLAGS="$(LDFLAGS)" \
364+
KOCACHE=$(KOCACHE_PATH) \
365+
$(KO) build -B \
366+
--push=false \
367+
--sbom=none \
368+
--platform=$(PLATFORM) \
369+
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
370+
github.com/ossf/scorecard/v4/cron/internal/webhook
371+
372+
cron-github-server-ko: | $(KO) $(KOCACHE_PATH)
373+
KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
374+
KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-github-server \
375+
LDFLAGS="$(LDFLAGS)" \
376+
KOCACHE=$(KOCACHE_PATH) \
377+
$(KO) build -B \
378+
--push=false \
379+
--sbom=none \
380+
--platform=$(PLATFORM) \
381+
--tags latest,$(GIT_VERSION),$(GIT_HASH) \
382+
github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper/tokens/server
383+
384+
###############################################################################

0 commit comments

Comments
 (0)