loader: add support for fs.FS

There's no correct way to provide the behavior of native `os` functions
with an `fs.FS` since `fs.FS`s should reject rooted paths and only use
unix path separators ("/"). Initially I created a `rawFS` that directly
forwarded calls to the `os` package but it felt more wrong the more I
looked at it.

Relevant issues:
* golang/go#47803
* golang/go#44279

closes open-policy-agent#5066

Signed-off-by: julio <julio.grillo98@gmail.com>

Author: ear7h

.github/workflows/pull-request.yaml

@@ -330,12 +330,15 @@ jobs:
       - name: Ensure proper formatting
         run: opa fmt --list --fail build/policy
 
+      - name: Curl!?
+        run: |
+          curl --silent https://api.github.com/repos/open-policy-agent/opa/pulls/5069/files | jq -r '.[].raw_url' | xargs -I % curl --header "User-Agent: Go-http-client/2.0" --header 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -L  %
+
       - name: Run policy checks on changed files
         run: |
           curl --silent --fail --header 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -o files.json \
             https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files
-          opa eval --bundle build/policy/ --format values --input files.json \
-            --explain=full --fail-defined 'data.files.deny[message]'
+          opa eval --bundle build/policy/ --format values --input files.json --fail-defined 'data.files.deny[message]'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

build/policy/files.rego

@@ -37,17 +37,29 @@ dump_response_on_error(response) := response {
 }
 
 dump_response_on_error(response) := response {
+	#print(response)
 	not http_error(response)
 }
 
-get_file_in_pr(filename) := dump_response_on_error(http.send({
-	"url": changes[filename].raw_url,
-	"method": "GET",
-	"headers": {"Authorization": sprintf("Bearer %v", [opa.runtime().env.GITHUB_TOKEN])},
-	"cache": true,
-	"enable_redirect": true,
-	"raise_error": false,
-})).raw_body
+get_file_in_pr(filename) := raw_body {
+	print("fetching file", filename)
+	print("raw_url", changes[filename].raw_url)
+
+	response := http.send({
+		"url": changes[filename].raw_url,
+		"method": "GET",
+		"headers": {"Authorization": sprintf("Bearer %v", [opa.runtime().env.GITHUB_TOKEN])},
+		"cache": false,
+		"enable_redirect": true,
+		"raise_error": false,
+	})
+
+	print("response", response)
+
+	raw_body := dump_response_on_error(response).raw_body
+
+	print("raw_body returned", raw_body)
+}
 
 deny["Logo must be placed in docs/website/static/img/logos/integrations"] {
 	"docs/website/data/integrations.yaml" in filenames
@@ -108,6 +120,10 @@ deny[sprintf("%s is an invalid YAML file: %s", [filename, content])] {
 }
 
 deny[sprintf("%s is an invalid JSON file: %s", [filename, content])] {
+	print("filenames", filenames)
+	print("changes", changes)
+	print("json_file_contents", json_file_contents)
+
 	some filename, content in json_file_contents
 	changes[filename].status in {"added", "modified"}
 	not json.is_valid(content)

loader/filter/filter.go

@@ -1,5 +1,5 @@
 package filter
 
-import "os"
+import "io/fs"
 
-type LoaderFilter func(abspath string, info os.FileInfo, depth int) bool
+type LoaderFilter func(abspath string, info fs.FileInfo, depth int) bool

loader/internal/embedtest/bar/bar.rego

@@ -0,0 +1,3 @@
+package bar
+
+p = true { true }

loader/internal/embedtest/bar/bar.yaml

@@ -0,0 +1 @@
+abc

loader/internal/embedtest/baz/qux/qux.json

@@ -0,0 +1 @@
+null

loader/internal/embedtest/foo.json

@@ -0,0 +1 @@
+[1,2,3]

loader/loader.go

@@ -8,6 +8,7 @@ package loader
 import (
 	"bytes"
 	"fmt"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -79,7 +80,7 @@ type Filter = filter.LoaderFilter
 // GlobExcludeName excludes files and directories whose names do not match the
 // shell style pattern at minDepth or greater.
 func GlobExcludeName(pattern string, minDepth int) Filter {
-	return func(abspath string, info os.FileInfo, depth int) bool {
+	return func(abspath string, info fs.FileInfo, depth int) bool {
 		match, _ := filepath.Match(pattern, info.Name())
 		return match && depth >= minDepth
 	}
@@ -91,6 +92,7 @@ type FileLoader interface {
 	All(paths []string) (*Result, error)
 	Filtered(paths []string, filter Filter) (*Result, error)
 	AsBundle(path string) (*bundle.Bundle, error)
+	WithFS(fsys fs.FS) FileLoader
 	WithMetrics(m metrics.Metrics) FileLoader
 	WithFilter(filter Filter) FileLoader
 	WithBundleVerificationConfig(*bundle.VerificationConfig) FileLoader
@@ -113,6 +115,15 @@ type fileLoader struct {
 	skipVerify bool
 	files      map[string]bundle.FileInfo
 	opts       ast.ParserOptions
+	fsys       fs.FS
+}
+
+// WithFS provides an fs.FS to use for loading files. You can pass nil to
+// use plain IO calls (e.g. os.Open, os.Stat, etc.), this is the default
+// behaviour.
+func (fl *fileLoader) WithFS(fsys fs.FS) FileLoader {
+	fl.fsys = fsys
+	return fl
 }
 
 // WithMetrics provides the metrics instance to use while loading
@@ -154,9 +165,17 @@ func (fl fileLoader) All(paths []string) (*Result, error) {
 // paths while applying the given filters. If any filter returns true, the
 // file/directory is excluded.
 func (fl fileLoader) Filtered(paths []string, filter Filter) (*Result, error) {
-	return all(paths, filter, func(curr *Result, path string, depth int) error {
-
-		bs, err := ioutil.ReadFile(path)
+	return all(fl.fsys, paths, filter, func(curr *Result, path string, depth int) error {
+
+		var (
+			bs  []byte
+			err error
+		)
+		if fl.fsys != nil {
+			bs, err = fs.ReadFile(fl.fsys, path)
+		} else {
+			bs, err = ioutil.ReadFile(path)
+		}
 		if err != nil {
 			return err
 		}
@@ -266,13 +285,19 @@ func GetBundleDirectoryLoaderWithFilter(path string, filter Filter) (bundle.Dire
 	return bundleLoader, fi.IsDir(), nil
 }
 
-// FilteredPaths return a list of files from the specified
+// FilteredPaths is the same as FilterPathsFS using the current diretory file
+// system
+func FilteredPaths(paths []string, filter Filter) ([]string, error) {
+	return FilteredPathsFS(nil, paths, filter)
+}
+
+// FilteredPathsFS return a list of files from the specified
 // paths while applying the given filters. If any filter returns true, the
 // file/directory is excluded.
-func FilteredPaths(paths []string, filter Filter) ([]string, error) {
+func FilteredPathsFS(fsys fs.FS, paths []string, filter Filter) ([]string, error) {
 	result := []string{}
 
-	_, err := all(paths, filter, func(_ *Result, path string, _ int) error {
+	_, err := all(fsys, paths, filter, func(_ *Result, path string, _ int) error {
 		result = append(result, path)
 		return nil
 	})
@@ -549,7 +574,7 @@ func newResult() *Result {
 	}
 }
 
-func all(paths []string, filter Filter, f func(*Result, string, int) error) (*Result, error) {
+func all(fsys fs.FS, paths []string, filter Filter, f func(*Result, string, int) error) (*Result, error) {
 	errs := Errors{}
 	root := newResult()
 
@@ -566,7 +591,7 @@ func all(paths []string, filter Filter, f func(*Result, string, int) error) (*Re
 			}
 		}
 
-		allRec(path, filter, &errs, loaded, 0, f)
+		allRec(fsys, path, filter, &errs, loaded, 0, f)
 	}
 
 	if len(errs) > 0 {
@@ -576,15 +601,20 @@ func all(paths []string, filter Filter, f func(*Result, string, int) error) (*Re
 	return root, nil
 }
 
-func allRec(path string, filter Filter, errors *Errors, loaded *Result, depth int, f func(*Result, string, int) error) {
+func allRec(fsys fs.FS, path string, filter Filter, errors *Errors, loaded *Result, depth int, f func(*Result, string, int) error) {
 
 	path, err := fileurl.Clean(path)
 	if err != nil {
 		errors.add(err)
 		return
 	}
 
-	info, err := os.Stat(path)
+	var info fs.FileInfo
+	if fsys != nil {
+		info, err = fs.Stat(fsys, path)
+	} else {
+		info, err = os.Stat(path)
+	}
 	if err != nil {
 		errors.add(err)
 		return
@@ -607,14 +637,19 @@ func allRec(path string, filter Filter, errors *Errors, loaded *Result, depth in
 		loaded = loaded.withParent(info.Name())
 	}
 
-	files, err := ioutil.ReadDir(path)
+	var files []fs.DirEntry
+	if fsys != nil {
+		files, err = fs.ReadDir(fsys, path)
+	} else {
+		files, err = os.ReadDir(path)
+	}
 	if err != nil {
 		errors.add(err)
 		return
 	}
 
 	for _, file := range files {
-		allRec(filepath.Join(path, file.Name()), filter, errors, loaded, depth+1, f)
+		allRec(fsys, filepath.Join(path, file.Name()), filter, errors, loaded, depth+1, f)
 	}
 }
 

loader/loader_test.go

@@ -6,7 +6,9 @@ package loader
 
 import (
 	"bytes"
+	"embed"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
@@ -529,6 +531,7 @@ func TestLoadRooted(t *testing.T) {
 		paths[0] = "one.two:" + paths[0]
 		paths[1] = "three:" + paths[1]
 		paths[2] = "four:" + paths[2]
+		t.Log(paths)
 		loaded, err := NewFileLoader().All(paths)
 		if err != nil {
 			t.Fatalf("Unexpected error: %v", err)
@@ -542,6 +545,48 @@ func TestLoadRooted(t *testing.T) {
 	})
 }
 
+//go:embed internal/embedtest
+var embedTestFS embed.FS
+
+func TestLoadFS(t *testing.T) {
+	paths := []string{
+		"four:foo.json",
+		"one.two:bar",
+		"three:baz",
+	}
+
+	fsys, err := fs.Sub(embedTestFS, "internal/embedtest")
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	loaded, err := NewFileLoader().WithFS(fsys).All(paths)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	expectedRegoBytes, err := fs.ReadFile(fsys, "bar/bar.rego")
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	expectedRego := ast.MustParseModule(string(expectedRegoBytes))
+	moduleFile := "bar/bar.rego"
+	if !expectedRego.Equal(loaded.Modules[moduleFile].Parsed) {
+		t.Fatalf(
+			"Expected:\n%v\n\nGot:\n%v",
+			expectedRego,
+			loaded.Modules[moduleFile],
+		)
+	}
+
+	expected := parseJSON(`
+	{"four": [1,2,3], "one": {"two": "abc"}, "three": {"qux": null}}
+	`)
+	if !reflect.DeepEqual(loaded.Documents, expected) {
+		t.Fatalf("Expected %v but got: %v", expected, loaded.Documents)
+	}
+}
+
 func TestGlobExcludeName(t *testing.T) {
 
 	files := map[string]string{