cmd/cue/cmd: hook up protobuf encodings types

Closes #606

Change-Id: I712ed6f9fc37d53b633a44ed87e4c71c1e018b0f
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/9372
Reviewed-by: CUE cueckoo <[email protected]>
Reviewed-by: Marcel van Lohuizen <[email protected]>

Author: mpvl

cmd/cue/cmd/common.go

@@ -296,7 +296,7 @@ func (i *streamingIterator) scan() bool {
 	if schema := i.b.encConfig.Schema; schema.Exists() {
 		i.e = schema.Err()
 		if i.e == nil {
-			i.v = i.v.Unify(schema)
+			i.v = i.v.Unify(schema) // TODO(required fields): don't merge in schema
 			i.e = i.v.Err()
 		}
 		i.f = nil
@@ -409,6 +409,19 @@ func (p *buildPlan) getDecoders(b *build.Instance) (schemas, values []*decoderIn
 	for _, f := range b.OrphanedFiles {
 		switch f.Encoding {
 		case build.Protobuf, build.YAML, build.JSON, build.JSONL, build.Text:
+			if f.Interpretation == build.ProtobufJSON {
+				// Need a schema.
+				values = append(values, &decoderInfo{f, nil})
+				continue
+			}
+		case build.TextProto:
+			if p.importing {
+				return schemas, values, errors.Newf(token.NoPos,
+					"cannot import textproto files")
+			}
+			// Needs to be decoded after any schema.
+			values = append(values, &decoderInfo{f, nil})
+			continue
 		default:
 			return schemas, values, errors.Newf(token.NoPos,
 				"unsupported encoding %q", f.Encoding)
@@ -569,13 +582,17 @@ func parseArgs(cmd *Command, args []string, cfg *config) (p *buildPlan, err erro
 		}
 
 		switch {
+		default:
+			fallthrough
+
+		case p.schema != nil:
+			p.orphaned = values
+
 		case p.mergeData, p.usePlacement(), p.importing:
 			if err = p.placeOrphans(b, values); err != nil {
 				return nil, err
 			}
 
-		default:
-			p.orphaned = values
 		}
 
 		if len(b.Files) > 0 {

cmd/cue/cmd/help.go

@@ -185,9 +185,11 @@ var filetypeHelp = &cobra.Command{
     jsonl       .jsonl/.ldjson  Line-separated JSON values.
     jsonschema                  JSON Schema.
     openapi                     OpenAPI schema.
+	pb                          Use Protobuf mappings (e.g. json+pb)
+    textproto    .textproto     Text-based protocol buffers.
     proto        .proto         Protocol Buffer definitions.
-    go          .go             Go source files.
-    text        .txt            Raw text file; the evaluated
+    go           .go            Go source files.
+    text         .txt           Raw text file; the evaluated
                                 value must be of type string.
 
 OpenAPI, JSON Schema and Protocol Buffer definitions are
@@ -495,7 +497,6 @@ cuelang.org/go/pkg/tool/tool.cue.
 // - id=<url>
 
 // TODO: filetypes:
-// - textpb
 // - binpb
 
 // TODO: cue.mod help topic

cmd/cue/cmd/orphans.go

@@ -27,6 +27,7 @@ import (
 	"cuelang.org/go/cue/errors"
 	"cuelang.org/go/cue/parser"
 	"cuelang.org/go/cue/token"
+	"cuelang.org/go/encoding/protobuf/jsonpb"
 	"cuelang.org/go/internal"
 	"cuelang.org/go/internal/astinternal"
 	"cuelang.org/go/internal/encoding"
@@ -172,6 +173,7 @@ func placeOrphans(b *buildPlan, d *encoding.Decoder, pkg string, objs ...*ast.Fi
 		expr := internal.ToExpr(file)
 		p, _, _ := internal.PackageInfo(file)
 
+		var path cue.Path
 		var labels []ast.Label
 
 		switch {
@@ -227,6 +229,22 @@ func placeOrphans(b *buildPlan, d *encoding.Decoder, pkg string, objs ...*ast.Fi
 				a = append(a, cue.Label(label))
 				labels = append(labels, label)
 			}
+
+			path = cue.MakePath(a...)
+		}
+
+		switch d.Interpretation() {
+		case build.ProtobufJSON:
+			v := b.instance.Value().LookupPath(path)
+			if b.useList {
+				v, _ = v.Elem()
+			}
+			if !v.Exists() {
+				break
+			}
+			if err := jsonpb.NewDecoder(v).RewriteFile(file); err != nil {
+				return nil, err
+			}
 		}
 
 		if b.useList {

cmd/cue/cmd/testdata/script/cmd_jsonpb.txt

@@ -0,0 +1,81 @@
+cue eval schema.cue json+pb: data.json
+cmp stdout out/data1
+
+cue eval schemaflag.cue -d '#X' json+pb: data.json
+cmp stdout out/data1
+
+! cue eval schema.cue json+pb: data-err.json
+cmp stderr out/data-err
+
+cue eval .:nested yaml+pb: stream.yaml -l kind
+cmp stdout out/stream
+
+-- schema.cue --
+a: int @protobuf(1,int64) // to string
+b: int @protobuf(2,int32) // also allow string
+c: int // also allow
+d: float
+s: string
+t: bytes
+
+-- nested.cue --
+package nested
+
+A: {
+    a: int @protobuf(1,int64) // to string
+    b: bytes
+}
+
+B: {
+    a: int @protobuf(1,int64) // to string
+    s: string
+}
+
+-- schemaflag.cue --
+#X: {
+    a: int @protobuf(1,int64) // to string
+    b: int @protobuf(2,int32) // also allow string
+    c: int // also allow
+    d: float
+    s: string
+    t: bytes
+}
+
+-- data.json --
+{"a": "10", "b": "20", "c": 30, "d": "1.2",
+"s":"SGVsbG8sIOS4lueVjA==",
+"t": "SGVsbG8sIOS4lueVjA=="}
+
+-- data-err.json --
+{"a": "10", "b": "20", "c": "30", "t": "SGVsbG8sIOS4lue???VjA==" }
+
+-- stream.yaml --
+kind: "A"
+a: "10"
+b: "SGVsbG8sIOS4lueVjA=="
+---
+kind: "B"
+a: "10"
+s: "SGVsbG8sIOS4lueVjA=="
+
+-- out/data1 --
+a: 10
+b: 20
+c: 30
+d: 1.2
+s: "SGVsbG8sIOS4lueVjA=="
+t: 'Hello, 世界'
+-- out/stream --
+A: {
+    kind: "A"
+    a:    10
+    b:    'Hello, 世界'
+}
+B: {
+    kind: "B"
+    a:    10
+    s:    "SGVsbG8sIOS4lueVjA=="
+}
+-- out/data-err --
+t: failed to decode base64: illegal base64 data at input byte 15:
+    ./data-err.json:1:40

cmd/cue/cmd/testdata/script/cmd_textproto.txt

@@ -0,0 +1,37 @@
+cue eval topschema.cue foo.textproto
+cmp stdout out/topfoo.textproto
+
+cue eval -d '#X' schema.cue foo.textproto
+cmp stdout out/foo.textproto
+
+! cue eval -d '#X' schema.cue foo.textproto -l c
+cmp stderr out/stderr3
+
+-- topschema.cue --
+a: int
+b: [...int]
+c: string
+-- schema.cue --
+#X: {
+        a: int
+        b: [...int]
+        c: string
+}
+-- foo.textproto --
+a: 4
+b: 1
+b: 2
+b: 3
+b: 4
+b: 5
+c: "foo"
+-- out/topfoo.textproto --
+a: 4
+b: [1, 2, 3, 4, 5]
+c: "foo"
+-- out/foo.textproto --
+a: 4
+b: [1, 2, 3, 4, 5]
+c: "foo"
+-- out/stderr3 --
+cannot combine --schema flag with flag "path", "list", or "files"

cue/build/file.go

@@ -30,17 +30,17 @@ type File struct {
 type Encoding string
 
 const (
-	CUE      Encoding = "cue"
-	JSON     Encoding = "json"
-	YAML     Encoding = "yaml"
-	JSONL    Encoding = "jsonl"
-	Text     Encoding = "text"
-	Protobuf Encoding = "proto"
+	CUE         Encoding = "cue"
+	JSON        Encoding = "json"
+	YAML        Encoding = "yaml"
+	JSONL       Encoding = "jsonl"
+	Text        Encoding = "text"
+	Protobuf    Encoding = "proto"
+	TextProto   Encoding = "textproto"
+	BinaryProto Encoding = "pb"
 
 	// TODO:
 	// TOML
-	// TextProto
-	// BinProto
 
 	Code Encoding = "code" // Programming languages
 )
@@ -62,9 +62,10 @@ const (
 	// the info.title and info.version fields.
 	//
 	// In all other cases, the underlying data is interpreted as is.
-	Auto       Interpretation = "auto"
-	JSONSchema Interpretation = "jsonschema"
-	OpenAPI    Interpretation = "openapi"
+	Auto         Interpretation = "auto"
+	JSONSchema   Interpretation = "jsonschema"
+	OpenAPI      Interpretation = "openapi"
+	ProtobufJSON Interpretation = "pb"
 )
 
 // A Form specifies the form in which a program should be represented.

internal/encoding/encoder.go

@@ -30,6 +30,8 @@ import (
 	"cuelang.org/go/cue/format"
 	"cuelang.org/go/cue/token"
 	"cuelang.org/go/encoding/openapi"
+	"cuelang.org/go/encoding/protobuf/jsonpb"
+	"cuelang.org/go/encoding/protobuf/textproto"
 	"cuelang.org/go/internal"
 	"cuelang.org/go/internal/filetypes"
 	"cuelang.org/go/pkg/encoding/yaml"
@@ -87,6 +89,12 @@ func NewEncoder(f *build.File, cfg *Config) (*Encoder, error) {
 			}
 			return openapi.Generate(i, cfg)
 		}
+	case build.ProtobufJSON:
+		e.interpret = func(v cue.Value) (*ast.File, error) {
+			f := valueToFile(v)
+			return f, jsonpb.NewEncoder(v).RewriteFile(f)
+		}
+
 	// case build.JSONSchema:
 	// 	// TODO: get encoding options
 	// 	cfg := openapi.Config{}
@@ -182,6 +190,20 @@ func NewEncoder(f *build.File, cfg *Config) (*Encoder, error) {
 			return err
 		}
 
+	case build.TextProto:
+		// TODO: verify that the schema is given. Otherwise err out.
+		e.concrete = true
+		e.encValue = func(v cue.Value) error {
+			v = v.Unify(cfg.Schema)
+			b, err := textproto.NewEncoder().Encode(v)
+			if err != nil {
+				return err
+			}
+
+			_, err = w.Write(b)
+			return err
+		}
+
 	case build.Text:
 		e.concrete = true
 		e.encValue = func(v cue.Value) error {

internal/encoding/encoding.go

@@ -37,6 +37,8 @@ import (
 	"cuelang.org/go/encoding/jsonschema"
 	"cuelang.org/go/encoding/openapi"
 	"cuelang.org/go/encoding/protobuf"
+	"cuelang.org/go/encoding/protobuf/jsonpb"
+	"cuelang.org/go/encoding/protobuf/textproto"
 	"cuelang.org/go/internal"
 	"cuelang.org/go/internal/filetypes"
 	"cuelang.org/go/internal/third_party/yaml"
@@ -48,6 +50,7 @@ type Decoder struct {
 	cfg            *Config
 	closer         io.Closer
 	next           func() (ast.Expr, error)
+	rewriteFunc    rewriteFunc
 	interpretFunc  interpretFunc
 	interpretation build.Interpretation
 	expr           ast.Expr
@@ -59,6 +62,7 @@ type Decoder struct {
 }
 
 type interpretFunc func(*cue.Instance) (file *ast.File, id string, err error)
+type rewriteFunc func(*ast.File) (file *ast.File, err error)
 
 // ID returns a canonical identifier for the decoded object or "" if no such
 // identifier could be found.
@@ -89,7 +93,15 @@ func (i *Decoder) Next() {
 }
 
 func (i *Decoder) doInterpret() {
-	// Interpretations
+	if i.rewriteFunc != nil {
+		i.file = i.File()
+		var err error
+		i.file, err = i.rewriteFunc(i.file)
+		if err != nil {
+			i.err = err
+			return
+		}
+	}
 	if i.interpretFunc != nil {
 		var r cue.Runtime
 		i.file = i.File()
@@ -208,6 +220,9 @@ func NewDecoder(f *build.File, cfg *Config) *Decoder {
 	case build.JSONSchema:
 		i.interpretation = build.JSONSchema
 		i.interpretFunc = jsonSchemaFunc(cfg, f)
+	case build.ProtobufJSON:
+		i.interpretation = build.ProtobufJSON
+		i.rewriteFunc = protobufJSONFunc(cfg, f)
 	default:
 		i.err = fmt.Errorf("unsupported interpretation %q", f.Interpretation)
 	}
@@ -242,6 +257,13 @@ func NewDecoder(f *build.File, cfg *Config) *Decoder {
 			PkgName: cfg.PkgName,
 		}
 		i.file, i.err = protobuf.Extract(path, r, paths)
+	case build.TextProto:
+		b, err := ioutil.ReadAll(r)
+		i.err = err
+		if err == nil {
+			d := textproto.NewDecoder()
+			i.expr, i.err = d.Parse(cfg.Schema, path, b)
+		}
 	default:
 		i.err = fmt.Errorf("unsupported encoding %q", f.Encoding)
 	}
@@ -286,6 +308,16 @@ func openAPIFunc(c *Config, f *build.File) interpretFunc {
 	}
 }
 
+func protobufJSONFunc(cfg *Config, file *build.File) rewriteFunc {
+	return func(f *ast.File) (*ast.File, error) {
+		if !cfg.Schema.Exists() {
+			return f, errors.Newf(token.NoPos,
+				"no schema specified for protobuf interpretation.")
+		}
+		return f, jsonpb.NewDecoder(cfg.Schema).RewriteFile(f)
+	}
+}
+
 func reader(f *build.File, stdin io.Reader) (io.ReadCloser, error) {
 	switch s := f.Source.(type) {
 	case nil: