encoding/jsonschema: implement ExtractCRDs

This adds a CRD-specific entry-point that extracts all the
CRDs from within a particular CRD spec and makes it
easy for a caller to filter them at will.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: Ieb6bda596ed57b5e933e5d35b62127a59531a9cc
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1217735
TryBot-Result: CUEcueckoo <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>
Reviewed-by: Daniel Martí <[email protected]>

Author: rogpeppe

encoding/jsonschema/crd.cue

@@ -0,0 +1,61 @@
+package jsonschema
+
+// input holds the parsed YAML document, which may contain multiple
+// Kubernetes resources, in which case it will be an array.
+input!: _
+
+// specs holds the resulting CRD specs: when there was only
+// a single resource in the input, we'll get an array with one
+// element.
+// TODO(rog) replace comparisons to bottom with whatever the
+// correct replacement will be.
+specs: {
+	[...]
+	if (input & [...]) != _|_ {
+		// It's an array: include only elements that look like CRDs.
+		[
+			for doc in input
+			if (doc & {#crdlike, ...}) != _|_ {
+				// Note: don't check for unification with #CRDSpec above because
+				// we want it to fail if it doesn't unify with the entirety of #CRDSpec,
+				// not just exclude the document.
+				doc
+				#CRDSpec
+			},
+		]
+	}
+	if (input & [...]) == _|_ {
+		// It's a single document. Include it if it looks like a CRD.
+		if (input & {#crdlike, ...}) != _|_ {
+			[{input, #CRDSpec}]
+		}
+	}
+}
+
+#crdlike: {
+	apiVersion!: "apiextensions.k8s.io/v1"
+	kind!:       "CustomResourceDefinition"
+} @go(crdLike)
+
+// CRDSpec defines a subset of the CRD schema, suitable for filtering
+// CRDs based on common criteria like group and name.
+#CRDSpec: {
+	#crdlike
+	apiVersion!: "apiextensions.k8s.io/v1"
+	kind!:       "CustomResourceDefinition"
+	spec!: {
+		group!: string
+		names!: {
+			kind!:     string
+			plural!:   string
+			singular!: string
+		}
+		scope!: "Namespaced" | "Cluster"
+		versions!: [... {
+			name!: string
+			schema!: {
+				openAPIV3Schema!: _ @go(,type="cuelang.org/go/cue".Value)
+			}
+		}]
+	}
+}

encoding/jsonschema/crd.go

@@ -0,0 +1,141 @@
+package jsonschema
+
+import (
+	_ "embed"
+	"fmt"
+
+	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/ast"
+	"cuelang.org/go/cue/token"
+)
+
+//go:generate go run cuelang.org/go/cmd/cue exp gengotypes .
+
+//go:embed crd.cue
+var crdCUE []byte
+
+// CRDConfig holds configuration for [ExtractCRDs].
+// Although this empty currently, it allows configuration
+// to be added in the future without breaking the API.
+type CRDConfig struct{}
+
+// ExtractedCRD holds an extracted Kubernetes CRD and the data it was derived from.
+type ExtractedCRD struct {
+	// Versions holds the CUE schemas extracted from the CRD: one per
+	// version.
+	Versions map[string]*ast.File
+
+	// Data holds chosen fields extracted from the source CRD document.
+	Data *CRDSpec
+
+	// Source holds the raw CRD document from which Data is derived.
+	Source cue.Value
+}
+
+// ExtractCRDs extracts Kubernetes custom resource definitions
+// (CRDs) from the given data. If data holds an array, each element
+// of the array might itself be a Kubernetes resource.
+//
+// While the data must hold Kubernetes resources, those resources
+// need not all be CRDs: resources with a kind that's not "CustomResourceDefinition"
+// will be ignored.
+//
+// If cfg is nil, it's equivalent to passing a pointer to the zero-valued [CRDConfig].
+func ExtractCRDs(data cue.Value, cfg *CRDConfig) ([]*ExtractedCRD, error) {
+	crdInfos, crdValues, err := decodeCRDSpecs(data)
+	if err != nil {
+		return nil, fmt.Errorf("cannot decode CRD: %v", err)
+	}
+	crds := make([]*ExtractedCRD, len(crdInfos))
+	for crdIndex, crd := range crdInfos {
+		versions := make(map[string]*ast.File)
+		for i, version := range crd.Spec.Versions {
+			f, err := Extract(crdValues[crdIndex], &Config{
+				PkgName: version.Name,
+				// There are several kubernetes-related keywords that aren't implemented yet
+				StrictFeatures: false,
+				StrictKeywords: true,
+				Root:           fmt.Sprintf("#/spec/versions/%d/schema/openAPIV3Schema", i),
+				SingleRoot:     true,
+				DefaultVersion: VersionKubernetesCRD,
+			})
+			if err != nil {
+				return nil, err
+			}
+			namespaceConstraint := token.OPTION
+			if crd.Spec.Scope == "Namespaced" {
+				namespaceConstraint = token.NOT
+			}
+			// TODO provide a way to let this refer to a shared definition
+			// in another package as the canonical definition for the schema.
+			f.Decls = append(f.Decls,
+				&ast.Field{
+					Label: ast.NewIdent("apiVersion"),
+					Value: ast.NewString(crd.Spec.Group + "/" + version.Name),
+				},
+				&ast.Field{
+					Label: ast.NewIdent("kind"),
+					Value: ast.NewString(crd.Spec.Names.Kind),
+				},
+				&ast.Field{
+					Label:      ast.NewIdent("metadata"),
+					Constraint: token.NOT,
+					Value: ast.NewStruct(
+						"name", token.NOT, ast.NewIdent("string"),
+						"namespace", namespaceConstraint, ast.NewIdent("string"),
+						// TODO inline struct lit
+						"labels", token.OPTION, ast.NewStruct(
+							ast.NewList(ast.NewIdent("string")), ast.NewIdent("string"),
+						),
+						"annotations", token.OPTION, ast.NewStruct(
+							ast.NewList(ast.NewIdent("string")), ast.NewIdent("string"),
+						),
+						// The above fields aren't exhaustive.
+						// TODO it would be nicer to refer to the actual #ObjectMeta
+						// definition instead (and for that to be the case for embedded
+						// resources in general) but that needs a deeper fix inside
+						// encoding/jsonschema.
+						&ast.Ellipsis{},
+					),
+				},
+			)
+			versions[version.Name] = f
+		}
+		crds[crdIndex] = &ExtractedCRD{
+			Versions: versions,
+			Data:     crdInfos[crdIndex],
+			Source:   crdValues[crdIndex],
+		}
+	}
+	return crds, nil
+}
+
+// decodeCRDSpecs decodes the CRD(s) held in the given value.
+// It returns both the (partially) decoded CRDs and the values
+// they were decoded from.
+func decodeCRDSpecs(v cue.Value) ([]*CRDSpec, []cue.Value, error) {
+	ctx := v.Context()
+
+	// Check against the CUE version of the schema which can
+	// do more detailed checks, including checking required fields,
+	// before decoding into the Go struct.
+
+	// TODO it would be nice to avoid compiling crdCUE every time, but
+	// that's not possible given the restrictions on combining cue.Values
+	// derived from different contexts.
+	crdSchema := ctx.CompileBytes(crdCUE)
+	crdSchema = crdSchema.FillPath(cue.MakePath(cue.Str("input")), v)
+	specsv := crdSchema.LookupPath(cue.MakePath(cue.Str("specs")))
+	if err := specsv.Err(); err != nil {
+		return nil, nil, err
+	}
+	var specs []*CRDSpec
+	var specsValues []cue.Value
+	if err := specsv.Decode(&specs); err != nil {
+		return nil, nil, err
+	}
+	if err := specsv.Decode(&specsValues); err != nil {
+		return nil, nil, err
+	}
+	return specs, specsValues, nil
+}

encoding/jsonschema/cue_types_gen.go

@@ -904,6 +904,8 @@ func (s0 *state) schemaState(n cue.Value, types cue.Kind, init func(*state)) (ex
 		if s.schemaVersion == VersionKubernetesCRD && s.isRoot {
 			// The root of a CRD is always a resource, so treat it as if it contained
 			// the x-kubernetes-embedded-resource keyword
+			// TODO remove this behavior now that we have an explicit
+			// ExtractCRDs function which does a better job at doing this.
 			c := constraintMap["x-kubernetes-embedded-resource"]
 			if c.phase != pass {
 				continue

encoding/jsonschema/decode.go

@@ -18,8 +18,10 @@ import (
 	"bytes"
 	"fmt"
 	"io/fs"
+	"maps"
 	"net/url"
 	"path"
+	"slices"
 	"strings"
 	"testing"
 
@@ -71,6 +73,17 @@ func TestDecode(t *testing.T) {
 		Matrix: cuetdtest.FullMatrix,
 	}
 	test.Run(t, func(t *cuetxtar.Test) {
+		ctx := t.CueContext()
+
+		fsys, err := txtar.FS(t.Archive)
+		if err != nil {
+			t.Fatal(err)
+		}
+		v, err := readSchema(ctx, fsys)
+		if err != nil {
+			t.Fatal(err)
+		}
+
 		cfg := &jsonschema.Config{}
 
 		if t.HasTag("brokenInV2") && t.M.Name() == "v2" {
@@ -96,6 +109,9 @@ func TestDecode(t *testing.T) {
 				cfg.Root = "#/components/schemas/"
 				cfg.StrictKeywords = true
 			case "k8sCRD":
+				if v.IncompleteKind() == cue.ListKind {
+					t.Skip("regular Extract does not cope with extracting CRDs from arrays")
+				}
 				cfg.DefaultVersion = jsonschema.VersionKubernetesCRD
 				// Default to the first version; can be overridden with #root.
 				cfg.Root = "#/spec/versions/0/schema/openAPIV3Schema"
@@ -119,20 +135,6 @@ func TestDecode(t *testing.T) {
 		}
 		cfg.PkgName, _ = t.Value("pkgName")
 
-		ctx := t.CueContext()
-
-		fsys, err := txtar.FS(t.Archive)
-		if err != nil {
-			t.Fatal(err)
-		}
-		v, err := readSchema(ctx, fsys)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if err := v.Err(); err != nil {
-			t.Fatal(err)
-		}
-
 		w := t.Writer("extract")
 		expr, err := jsonschema.Extract(v, cfg)
 		if err != nil {
@@ -234,9 +236,53 @@ func TestDecode(t *testing.T) {
 	})
 }
 
+func TestDecodeCRD(t *testing.T) {
+	test := cuetxtar.TxTarTest{
+		Root:   "./testdata/txtar",
+		Name:   "decodeCRD",
+		Matrix: cuetdtest.FullMatrix,
+	}
+	test.Run(t, func(t *cuetxtar.Test) {
+		if versStr, ok := t.Value("version"); !ok || versStr != "k8sCRD" {
+			t.Skip("test not relevant to CRDs")
+		}
+
+		ctx := t.CueContext()
+
+		fsys, err := txtar.FS(t.Archive)
+		if err != nil {
+			t.Fatal(err)
+		}
+		v, err := readSchema(ctx, fsys)
+		if err != nil {
+			t.Fatal(err)
+		}
+		crds, err := jsonschema.ExtractCRDs(v, &jsonschema.CRDConfig{})
+		if err != nil {
+			w := t.Writer("extractCRD/error")
+			fmt.Fprintf(w, "%v\n", err)
+			return
+		}
+		for i, crd := range crds {
+			for _, version := range slices.Sorted(maps.Keys(crd.Versions)) {
+				w := t.Writer(fmt.Sprintf("extractCRD/%d/%s", i, version))
+				f := crd.Versions[version]
+				b, err := format.Node(f, format.Simplify())
+				if err != nil {
+					t.Fatal(errors.Details(err, nil))
+				}
+				b = append(bytes.TrimSpace(b), '\n')
+				w.Write(b)
+				// TODO test that schema actually works.
+			}
+		}
+	})
+}
+
 func readSchema(ctx *cue.Context, fsys fs.FS) (cue.Value, error) {
 	jsonData, jsonErr := fs.ReadFile(fsys, "schema.json")
 	yamlData, yamlErr := fs.ReadFile(fsys, "schema.yaml")
+	var v cue.Value
 	switch {
 	case jsonErr == nil && yamlErr == nil:
 		return cue.Value{}, fmt.Errorf("cannot define both schema.json and schema.yaml")
@@ -245,15 +291,20 @@ func readSchema(ctx *cue.Context, fsys fs.FS) (cue.Value, error) {
 		if err != nil {
 			return cue.Value{}, err
 		}
-		return ctx.BuildExpr(expr), nil
+		v = ctx.BuildExpr(expr)
 	case yamlErr == nil:
 		file, err := yaml.Extract("schema.yaml", yamlData)
 		if err != nil {
 			return cue.Value{}, err
 		}
-		return ctx.BuildFile(file), nil
+		v = ctx.BuildFile(file)
+	default:
+		return cue.Value{}, fmt.Errorf("no schema.yaml or schema.json file found for test")
+	}
+	if err := v.Err(); err != nil {
+		return cue.Value{}, err
 	}
-	return cue.Value{}, fmt.Errorf("no schema.yaml or schema.json file found for test")
+	return v, nil
 }
 
 func TestMapURL(t *testing.T) {