cue/load: move towards unified syntax cache

In order to to fix issue #3177, we need a cache that's shared between
the module code in `internal/mod/...` and the `cue/load` package, which
means that the syntax cache needs to be pushed down to a lower level in
`cue/load` because currently it's based around cue/load-specific
abstractions.

This CL is an incremental progression towards that goal: it adds a
`getCUESyntax` method to the internal filesystem abstraction in
`cue/load`, which will become a cache in a subsequent CL.

In order to do this, some refactoring was required, namely removal of
the `matchFile` function, because that function read data without
parsing the CUE, but the parsing and reading operations now need to be
implemented by the same operation (the new `getCUESyntax` method). In
any case, the `matchFile` function did not really fulfil its original
purpose or fit the name any more.

Also remove the return value from `fileProcessor.add` while we're
refactoring it, because nothing uses it.

For #3177.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: I1876f22b99a5c611e3d8650d3b7afb8f8e43f202
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1197525
Unity-Result: CUE porcuepine <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>
Reviewed-by: Paul Jolly <[email protected]>

Author: rogpeppe

cue/load/fs.go

@@ -27,8 +27,11 @@ import (
 	"time"
 
 	"cuelang.org/go/cue/ast"
+	"cuelang.org/go/cue/build"
 	"cuelang.org/go/cue/errors"
+	"cuelang.org/go/cue/parser"
 	"cuelang.org/go/cue/token"
+	"cuelang.org/go/internal/cueimports"
 	"cuelang.org/go/mod/module"
 )
 
@@ -179,6 +182,58 @@ func (fs *fileSystem) getOverlay(path string) *overlayFile {
 	return nil
 }
 
+func (fs *fileSystem) getCUESyntax(fi *build.File) (*ast.File, error) {
+	switch src := fi.Source.(type) {
+	case []byte, string:
+		return parser.ParseFile(fi.Filename, src, parser.ImportsOnly)
+	case *ast.File:
+		return src, nil
+	case nil:
+		if fi.Filename == "-" {
+			panic("source unexpectedly not provided for stdin")
+		}
+		return fs.readCUE(fi.Filename, true)
+	}
+	return nil, errors.Newf(token.NoPos, "unsupported source type %T", fi.Source)
+}
+
+func (fs *fileSystem) readCUE(path string, importsOnly bool) (*ast.File, error) {
+	var parseMode parser.Option
+	if importsOnly {
+		parseMode = parser.ImportsOnly
+	}
+	var data []byte
+	if f := fs.getOverlay(path); f != nil {
+		if f.file != nil {
+			return f.file, nil
+		}
+		if f.isDir {
+			return nil, fmt.Errorf("%q is a directory", path)
+		}
+		data = f.contents
+	} else {
+		var err error
+		f, err := fs.openFile(path)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		if importsOnly {
+			data, err = cueimports.Read(f)
+		} else {
+			data, err = io.ReadAll(f)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read %s: %v", path, err)
+		}
+	}
+	pf, err := parser.ParseFile(path, data, parseMode)
+	if err != nil {
+		return nil, err
+	}
+	return pf, nil
+}
+
 func (fs *fileSystem) stat(path string) (iofs.FileInfo, errors.Error) {
 	path = fs.makeAbs(path)
 	if fi := fs.getOverlay(path); fi != nil {

cue/load/loader.go

@@ -186,6 +186,9 @@ func (c *syntaxCache) getSyntax(bf *build.File) ([]*ast.File, error) {
 	if ok {
 		return syntax.files, syntax.err
 	}
+	if bf.Encoding != build.CUE {
+		panic("syntax for non-CUE file")
+	}
 	d := encoding.NewDecoder(c.ctx, bf, &c.config)
 	for ; !d.Done(); d.Next() {
 		syntax.files = append(syntax.files, d.File())

cue/load/loader_common.go

@@ -25,7 +25,6 @@ import (
 
 	"cuelang.org/go/cue/build"
 	"cuelang.org/go/cue/errors"
-	"cuelang.org/go/cue/parser"
 	"cuelang.org/go/cue/token"
 )
 
@@ -37,6 +36,15 @@ const (
 	allowExcludedFiles
 )
 
+var errExclude = errors.New("file rejected")
+
+type cueError = errors.Error
+type excludeError struct {
+	cueError
+}
+
+func (e excludeError) Is(err error) bool { return err == errExclude }
+
 func rewriteFiles(p *build.Instance, root string, isLocal bool) {
 	p.Root = root
 
@@ -145,7 +153,7 @@ func (fp *fileProcessor) finalize(p *build.Instance) errors.Error {
 }
 
 // add adds the given file to the appropriate package in fp.
-func (fp *fileProcessor) add(root string, file *build.File, mode importMode) (added bool) {
+func (fp *fileProcessor) add(root string, file *build.File, mode importMode) {
 	fullPath := file.Filename
 	if fullPath != "-" {
 		if !filepath.IsAbs(fullPath) {
@@ -160,42 +168,46 @@ func (fp *fileProcessor) add(root string, file *build.File, mode importMode) (ad
 	p := fp.pkg // default package
 
 	// badFile := func(p *build.Instance, err errors.Error) bool {
-	badFile := func(err errors.Error) bool {
+	badFile := func(err errors.Error) {
 		fp.err = errors.Append(fp.err, err)
 		file.ExcludeReason = fp.err
 		p.InvalidFiles = append(p.InvalidFiles, file)
-		return true
+		return
 	}
 	if err := setFileSource(fp.c, file); err != nil {
-		return badFile(errors.Promote(err, ""))
+		badFile(errors.Promote(err, ""))
+		return
 	}
 
-	match, data, err := matchFile(fp.c, file, true, mode)
-	switch {
-	case match:
-
-	case err == nil:
+	if file.Encoding != build.CUE {
 		// Not a CUE file.
 		p.OrphanedFiles = append(p.OrphanedFiles, file)
-		return false
-
-	case !errors.Is(err, errExclude):
-		return badFile(err)
-
-	default:
-		file.ExcludeReason = err
-		if file.Interpretation == "" {
-			p.IgnoredFiles = append(p.IgnoredFiles, file)
-		} else {
-			p.OrphanedFiles = append(p.OrphanedFiles, file)
+		return
+	}
+	if (mode & allowExcludedFiles) == 0 {
+		var badPrefix string
+		for _, prefix := range []string{".", "_"} {
+			if strings.HasPrefix(base, prefix) {
+				badPrefix = prefix
+			}
+		}
+		if badPrefix != "" {
+			file.ExcludeReason = errors.Newf(token.NoPos, "filename starts with a '%s'", badPrefix)
+			if file.Interpretation == "" {
+				p.IgnoredFiles = append(p.IgnoredFiles, file)
+			} else {
+				p.OrphanedFiles = append(p.OrphanedFiles, file)
+			}
+			return
 		}
-		return false
 	}
-
-	pf, perr := parser.ParseFile(fullPath, data, parser.ImportsOnly)
+	// Note: when path is "-" (stdin), it will already have
+	// been read and file.Source set to the resulting data
+	// by setFileSource.
+	pf, perr := fp.c.fileSystem.getCUESyntax(file)
 	if perr != nil {
 		badFile(errors.Promote(perr, "add failed"))
-		return true
+		return
 	}
 
 	pkg := pf.PackageName()
@@ -227,7 +239,7 @@ func (fp *fileProcessor) add(root string, file *build.File, mode importMode) (ad
 	default:
 		file.ExcludeReason = excludeError{errors.Newf(pos, "no package name")}
 		p.IgnoredFiles = append(p.IgnoredFiles, file)
-		return false // don't mark as added
+		return
 	}
 
 	if !fp.c.AllCUEFiles {
@@ -245,7 +257,7 @@ func (fp *fileProcessor) add(root string, file *build.File, mode importMode) (ad
 			}
 			file.ExcludeReason = err
 			p.IgnoredFiles = append(p.IgnoredFiles, file)
-			return false
+			return
 		}
 	}
 
@@ -258,14 +270,15 @@ func (fp *fileProcessor) add(root string, file *build.File, mode importMode) (ad
 				file.ExcludeReason = excludeError{errors.Newf(pos,
 					"package is %s, want %s", pkg, p.PkgName)}
 				p.IgnoredFiles = append(p.IgnoredFiles, file)
-				return false
+				return
 			}
 			if !fp.allPackages {
-				return badFile(&MultiplePackageError{
+				badFile(&MultiplePackageError{
 					Dir:      p.Dir,
 					Packages: []string{p.PkgName, pkg},
 					Files:    []string{fp.firstFile, base},
 				})
+				return
 			}
 		}
 	}
@@ -306,7 +319,6 @@ func (fp *fileProcessor) add(root string, file *build.File, mode importMode) (ad
 	default:
 		p.BuildFiles = append(p.BuildFiles, file)
 	}
-	return true
 }
 
 func cleanImports(m map[string][]token.Pos) ([]string, map[string][]token.Pos) {

cue/load/match.go

@@ -28,6 +28,14 @@ import (
 	"cuelang.org/go/mod/module"
 )
 
+// A match represents the result of matching a single package pattern.
+type match struct {
+	Pattern string // the pattern itself
+	Literal bool   // whether it is a literal (no wildcards)
+	Pkgs    []*build.Instance
+	Err     errors.Error
+}
+
 // TODO: should be matched from module file only.
 // The pattern is either "all" (all packages), "std" (standard packages),
 // "cmd" (standard commands), or a path including "...".