Only run allowed checks in different modes (#1579)

Co-authored-by: Azeem Shaikh <azeems@google.com>

Author: azeemshaikh38

checker/check_request.go

@@ -30,5 +30,34 @@ type CheckRequest struct {
 	Repo                  clients.Repo
 	VulnerabilitiesClient clients.VulnerabilitiesClient
 	// UPGRADEv6: return raw results instead of scores.
-	RawResults *RawResults
+	RawResults    *RawResults
+	RequiredTypes []RequestType
+}
+
+// RequestType identifies special requirements/attributes that need to be supported by checks.
+type RequestType int
+
+const (
+	// FileBased request types require checks to run solely on file-content.
+	FileBased RequestType = iota
+)
+
+// ListUnsupported returns []RequestType not in `supported` and are `required`.
+func ListUnsupported(required, supported []RequestType) []RequestType {
+	var ret []RequestType
+	for _, t := range required {
+		if !contains(supported, t) {
+			ret = append(ret, t)
+		}
+	}
+	return ret
+}
+
+func contains(in []RequestType, exists RequestType) bool {
+	for _, r := range in {
+		if r == exists {
+			return true
+		}
+	}
+	return false
 }

checker/check_runner.go

@@ -31,16 +31,22 @@ const checkRetries = 3
 
 // Runner runs a check with retries.
 type Runner struct {
-	CheckRequest CheckRequest
 	CheckName    string
 	Repo         string
+	CheckRequest CheckRequest
 }
 
 // CheckFn defined for convenience.
 type CheckFn func(*CheckRequest) CheckResult
 
+// Check defines a Scorecard check fn and its supported request types.
+type Check struct {
+	Fn                    CheckFn
+	SupportedRequestTypes []RequestType
+}
+
 // CheckNameToFnMap defined here for convenience.
-type CheckNameToFnMap map[string]CheckFn
+type CheckNameToFnMap map[string]Check
 
 func logStats(ctx context.Context, startTime time.Time, result *CheckResult) error {
 	runTimeInSecs := time.Now().Unix() - startTime.Unix()
@@ -57,7 +63,15 @@ func logStats(ctx context.Context, startTime time.Time, result *CheckResult) err
 }
 
 // Run runs a given check.
-func (r *Runner) Run(ctx context.Context, f CheckFn) CheckResult {
+func (r *Runner) Run(ctx context.Context, c Check) CheckResult {
+	// Sanity check.
+	unsupported := ListUnsupported(r.CheckRequest.RequiredTypes, c.SupportedRequestTypes)
+	if len(unsupported) != 0 {
+		return CreateRuntimeErrorResult(r.CheckName,
+			sce.WithMessage(sce.ErrorUnsupportedCheck,
+				fmt.Sprintf("requiredType: %s not supported by check %s", fmt.Sprint(unsupported), r.CheckName)))
+	}
+
 	ctx, err := tag.New(ctx, tag.Upsert(stats.CheckName, r.CheckName))
 	if err != nil {
 		panic(err)
@@ -71,7 +85,7 @@ func (r *Runner) Run(ctx context.Context, f CheckFn) CheckResult {
 		checkRequest.Ctx = ctx
 		l = logger{}
 		checkRequest.Dlogger = &l
-		res = f(&checkRequest)
+		res = c.Fn(&checkRequest)
 		if res.Error2 != nil && errors.Is(res.Error2, sce.ErrRepoUnreachable) {
 			checkRequest.Dlogger.Warn("%v", res.Error2)
 			continue

checks/all_checks.go

@@ -22,13 +22,16 @@ import (
 // AllChecks is the list of all security checks that will be run.
 var AllChecks = checker.CheckNameToFnMap{}
 
-func registerCheck(name string, fn checker.CheckFn) error {
+func registerCheck(name string, fn checker.CheckFn, supportedRequestTypes []checker.RequestType) error {
 	if name == "" {
 		return errInternalNameCannotBeEmpty
 	}
 	if fn == nil {
 		return errInternalCheckFuncCannotBeNil
 	}
-	AllChecks[name] = fn
+	AllChecks[name] = checker.Check{
+		Fn:                    fn,
+		SupportedRequestTypes: supportedRequestTypes,
+	}
 	return nil
 }

checks/all_checks_test.go

@@ -62,7 +62,7 @@ func Test_registerCheck(t *testing.T) {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
-			if err := registerCheck(tt.args.name, tt.args.fn); (err != nil) != tt.wanterr {
+			if err := registerCheck(tt.args.name, tt.args.fn, nil /*supportedRequestTypes*/); (err != nil) != tt.wanterr {
 				t.Errorf("registerCheck() error = %v, wantErr %v", err, tt.wanterr)
 			}
 		})

checks/binary_artifact.go

@@ -26,7 +26,10 @@ const CheckBinaryArtifacts string = "Binary-Artifacts"
 
 //nolint
 func init() {
-	if err := registerCheck(CheckBinaryArtifacts, BinaryArtifacts); err != nil {
+	var supportedRequestTypes = []checker.RequestType{
+		checker.FileBased,
+	}
+	if err := registerCheck(CheckBinaryArtifacts, BinaryArtifacts, supportedRequestTypes); err != nil {
 		// this should never happen
 		panic(err)
 	}

checks/branch_protection.go

@@ -21,14 +21,12 @@ import (
 	sce "github.com/ossf/scorecard/v4/errors"
 )
 
-const (
-	// CheckBranchProtection is the exported name for Branch-Protected check.
-	CheckBranchProtection = "Branch-Protection"
-)
+// CheckBranchProtection is the exported name for Branch-Protected check.
+const CheckBranchProtection = "Branch-Protection"
 
 //nolint:gochecknoinits
 func init() {
-	if err := registerCheck(CheckBranchProtection, BranchProtection); err != nil {
+	if err := registerCheck(CheckBranchProtection, BranchProtection, nil); err != nil {
 		// this should never happen
 		panic(err)
 	}

checks/ci_tests.go

@@ -31,7 +31,7 @@ const (
 
 //nolint:gochecknoinits
 func init() {
-	if err := registerCheck(CheckCITests, CITests); err != nil {
+	if err := registerCheck(CheckCITests, CITests, nil); err != nil {
 		// this should never happen
 		panic(err)
 	}

checks/cii_best_practices.go

@@ -32,7 +32,7 @@ const (
 
 //nolint:gochecknoinits
 func init() {
-	if err := registerCheck(CheckCIIBestPractices, CIIBestPractices); err != nil {
+	if err := registerCheck(CheckCIIBestPractices, CIIBestPractices, nil); err != nil {
 		// this should never happen
 		panic(err)
 	}

checks/code_review.go

@@ -26,7 +26,7 @@ const CheckCodeReview = "Code-Review"
 
 //nolint:gochecknoinits
 func init() {
-	if err := registerCheck(CheckCodeReview, CodeReview); err != nil {
+	if err := registerCheck(CheckCodeReview, CodeReview, nil); err != nil {
 		// this should never happen
 		panic(err)
 	}

checks/contributors.go

@@ -31,7 +31,7 @@ const (
 
 //nolint:gochecknoinits
 func init() {
-	if err := registerCheck(CheckContributors, Contributors); err != nil {
+	if err := registerCheck(CheckContributors, Contributors, nil); err != nil {
 		// this should never happen
 		panic(err)
 	}