Generate marshal JSON without gogo proto jsonpb (#13450)

This PR is part of an effort to remove usage of gogo proto in pdata and
switch to a model where we enhance the pdatagen tool to generate compact
(reflection free to avoid DCE disabling) and faster code.

The list of the PRs will do:
1. Auto generate the JSON marshaling logic (this PR).
1. Auto generate the JSON unmarshaling logic and replace manually
written code.
1. Auto generate the proto marshaling logic.
1. Auto generate the proto unmarshaling logic.
1. Remove the usage of auto-generate gogo proto code completely and
generate field members in the pdata objects.
1. Add support for lazy unmarshaling. 

Performance improvements are more than 30x:

**Before:**
```
BenchmarkLogsMarshalJSON
BenchmarkLogsMarshalJSON-12    	      94	  12541041 ns/op	 8458944 B/op	  196458 allocs/op
BenchmarkMetricsMarshalJSON
BenchmarkMetricsMarshalJSON-12    	       3	 452309069 ns/op	350229024 B/op	 8131400 allocs/op
BenchmarkProfilesMarshalJSON
BenchmarkProfilesMarshalJSON-12    	      30	  36731071 ns/op	20317088 B/op	  531652 allocs/op
BenchmarkTracesMarshalJSON
BenchmarkTracesMarshalJSON-12    	       8	 132521260 ns/op	95920588 B/op	 2208765 allocs/op
```

**After:**
```
BenchmarkLogsMarshalJSON
BenchmarkLogsMarshalJSON-12    	    3000	    403587 ns/op	  282465 B/op	    1487 allocs/op
BenchmarkMetricsMarshalJSON
BenchmarkMetricsMarshalJSON-12    	      72	  16772373 ns/op	11856529 B/op	   72833 allocs/op
BenchmarkProfilesMarshalJSON
BenchmarkProfilesMarshalJSON-12    	    1460	    849419 ns/op	  674545 B/op	    1145 allocs/op
BenchmarkTracesMarshalJSON
BenchmarkTracesMarshalJSON-12    	     271	   4344647 ns/op	 3010259 B/op	   16237 allocs/op
```

Signed-off-by: Bogdan Drutu <[email protected]>

Author: bogdandrutu

.chloggen/gen-marshal-json-stream.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: pdata
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Use pdatagen to generate marshalJSON without using gogo proto jsonpb.
+
+# One or more tracking issues or pull requests related to the change
+issues: [13450]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

internal/cmd/pdatagen/internal/message_field.go

@@ -39,6 +39,15 @@ const messageSetTestTemplate = `{{ if .isCommon -}}
 const messageCopyOrigTemplate = `{{ if .isCommon }}{{ if not .isBaseStructCommon }}internal.{{ end }}CopyOrig{{ else }}copyOrig{{ end }}
 {{- .returnType }}(&dest.{{ .originFieldName }}, &src.{{ .originFieldName }})`
 
+const messageMarshalJSONTemplate = `{{- if eq .returnType "TraceState" }} if ms.orig.{{ .originFieldName }} != "" { {{ end -}}
+	dest.WriteObjectField("{{ lowerFirst .originFieldName }}")
+	{{- if .isCommon }}
+	{{ if not .isBaseStructCommon }}internal.{{ end }}MarshalJSONStream{{ .returnType }}(
+	{{- if not .isBaseStructCommon }}internal.{{ end }}New{{ .returnType }}(&ms.orig.{{ .originFieldName }}, ms.state), dest)
+	{{- else }}
+	ms.{{ .fieldName }}().marshalJSONStream(dest)
+	{{- end }}{{ if eq .returnType "TraceState" -}} } {{- end }}`
+
 type MessageField struct {
 	fieldName     string
 	returnMessage *messageStruct
@@ -64,6 +73,11 @@ func (mf *MessageField) GenerateCopyOrig(ms *messageStruct) string {
 	return executeTemplate(t, mf.templateFields(ms))
 }
 
+func (mf *MessageField) GenerateMarshalJSON(ms *messageStruct) string {
+	t := template.Must(templateNew("messageMarshalJSONTemplate").Parse(messageMarshalJSONTemplate))
+	return executeTemplate(t, mf.templateFields(ms))
+}
+
 func (mf *MessageField) templateFields(ms *messageStruct) map[string]any {
 	return map[string]any{
 		"isCommon":        usedByOtherDataTypes(mf.returnMessage.packageName),

internal/cmd/pdatagen/internal/one_of_field.go

@@ -39,6 +39,12 @@ const oneOfCopyOrigTemplate = `switch t := src.{{ .originFieldName }}.(type) {
 {{- end }}
 }`
 
+const oneOfMarshalJSONTemplate = `switch ov := ms.{{ .origAccessor }}.{{ .originFieldName }}.(type) {
+		{{- range .values }}
+		{{ .GenerateMarshalJSON $.baseStruct $.OneOfField }}
+		{{- end }}
+	}`
+
 type OneOfField struct {
 	originFieldName            string
 	typeName                   string
@@ -74,6 +80,11 @@ func (of *OneOfField) GenerateCopyOrig(ms *messageStruct) string {
 	return executeTemplate(t, of.templateFields(ms))
 }
 
+func (of *OneOfField) GenerateMarshalJSON(ms *messageStruct) string {
+	t := template.Must(templateNew("oneOfMarshalJSONTemplate").Parse(oneOfMarshalJSONTemplate))
+	return executeTemplate(t, of.templateFields(ms))
+}
+
 func (of *OneOfField) templateFields(ms *messageStruct) map[string]any {
 	return map[string]any{
 		"baseStruct":           ms,
@@ -99,4 +110,5 @@ type oneOfValue interface {
 	GenerateSetWithTestValue(ms *messageStruct, of *OneOfField) string
 	GenerateCopyOrig(ms *messageStruct, of *OneOfField) string
 	GenerateType(ms *messageStruct, of *OneOfField) string
+	GenerateMarshalJSON(ms *messageStruct, of *OneOfField) string
 }

internal/cmd/pdatagen/internal/one_of_message_value.go

@@ -68,6 +68,10 @@ const oneOfMessageCopyOrigTemplate = `	case *{{ .originStructType }}:
 const oneOfMessageTypeTemplate = `case *{{ .originStructName }}_{{ .originFieldName }}:
 	return {{ .typeName }}`
 
+const oneOfMessageMarshalJSONTemplate = `case *{{ .originStructName }}_{{ .originFieldName }}:
+	dest.WriteObjectField("{{ lowerFirst .originFieldName }}")
+	new{{ .returnType }}(ov.{{ .fieldName }}, ms.state).marshalJSONStream(dest)`
+
 type OneOfMessageValue struct {
 	fieldName              string
 	originFieldPackageName string
@@ -99,6 +103,11 @@ func (omv *OneOfMessageValue) GenerateType(ms *messageStruct, of *OneOfField) st
 	return executeTemplate(t, omv.templateFields(ms, of))
 }
 
+func (omv *OneOfMessageValue) GenerateMarshalJSON(ms *messageStruct, of *OneOfField) string {
+	t := template.Must(templateNew("oneOfMessageMarshalJSONTemplate").Parse(oneOfMessageMarshalJSONTemplate))
+	return executeTemplate(t, omv.templateFields(ms, of))
+}
+
 func (omv *OneOfMessageValue) templateFields(ms *messageStruct, of *OneOfField) map[string]any {
 	return map[string]any{
 		"fieldName":               omv.fieldName,

internal/cmd/pdatagen/internal/one_of_primitive_value.go

@@ -54,6 +54,10 @@ const oneOfPrimitiveCopyOrigTemplate = `case *{{ .originStructName }}_{{ .origin
 const oneOfPrimitiveTypeTemplate = `case *{{ .originStructName }}_{{ .originFieldName }}:
 	return {{ .typeName }}`
 
+const oneOfPrimitiveMarshalJSONTemplate = `case *{{ .originStructName }}_{{ .originFieldName }}:
+	dest.WriteObjectField("{{ lowerFirst .originFieldName }}")
+	dest.Write{{ upperFirst .returnType }}(ov.{{ .originFieldName }})`
+
 type OneOfPrimitiveValue struct {
 	fieldName       string
 	defaultVal      string
@@ -87,6 +91,11 @@ func (opv *OneOfPrimitiveValue) GenerateType(ms *messageStruct, of *OneOfField)
 	return executeTemplate(t, opv.templateFields(ms, of))
 }
 
+func (opv *OneOfPrimitiveValue) GenerateMarshalJSON(ms *messageStruct, of *OneOfField) string {
+	t := template.Must(templateNew("oneOfPrimitiveMarshalJSONTemplate").Parse(oneOfPrimitiveMarshalJSONTemplate))
+	return executeTemplate(t, opv.templateFields(ms, of))
+}
+
 func (opv *OneOfPrimitiveValue) templateFields(ms *messageStruct, of *OneOfField) map[string]any {
 	return map[string]any{
 		"structName":  ms.getName(),

internal/cmd/pdatagen/internal/optional_primitive_field.go

@@ -67,6 +67,11 @@ const optionalPrimitiveCopyOrigTemplate = `if src{{ .fieldName }}, ok := src.{{
 	dest.{{ .fieldName }}_ = nil
 }`
 
+const optionalPrimitiveMarshalJSONTemplate = `if ms.Has{{ .fieldName }}() {
+		dest.WriteObjectField("{{ lowerFirst .fieldName }}")
+		dest.Write{{ upperFirst .returnType }}(ms.{{ .fieldName }}())
+	}`
+
 type OptionalPrimitiveField struct {
 	fieldName  string
 	defaultVal string
@@ -94,6 +99,11 @@ func (opv *OptionalPrimitiveField) GenerateCopyOrig(ms *messageStruct) string {
 	return executeTemplate(t, opv.templateFields(ms))
 }
 
+func (opv *OptionalPrimitiveField) GenerateMarshalJSON(ms *messageStruct) string {
+	t := template.Must(templateNew("optionalPrimitiveMarshalJSONTemplate").Parse(optionalPrimitiveMarshalJSONTemplate))
+	return executeTemplate(t, opv.templateFields(ms))
+}
+
 func (opv *OptionalPrimitiveField) templateFields(ms *messageStruct) map[string]any {
 	return map[string]any{
 		"structName":       ms.getName(),

internal/cmd/pdatagen/internal/primitive_field.go

@@ -48,6 +48,11 @@ const primitiveSetTestTemplate = `tv.orig.{{ .originFieldName }} = {{ .testValue
 
 const primitiveCopyOrigTemplate = `dest.{{ .originFieldName }} = src.{{ .originFieldName }}`
 
+const primitiveMarshalJSONTemplate = `if ms.orig.{{ .originFieldName }} != {{ .defaultVal }} {
+		dest.WriteObjectField("{{ lowerFirst .originFieldName }}")
+		dest.Write{{ upperFirst .returnType }}(ms.orig.{{ .originFieldName }})
+	}`
+
 type PrimitiveField struct {
 	fieldName       string
 	originFieldName string
@@ -76,6 +81,11 @@ func (pf *PrimitiveField) GenerateCopyOrig(ms *messageStruct) string {
 	return executeTemplate(t, pf.templateFields(ms))
 }
 
+func (pf *PrimitiveField) GenerateMarshalJSON(ms *messageStruct) string {
+	t := template.Must(templateNew("primitiveMarshalJSONTemplate").Parse(primitiveMarshalJSONTemplate))
+	return executeTemplate(t, pf.templateFields(ms))
+}
+
 func (pf *PrimitiveField) templateFields(ms *messageStruct) map[string]any {
 	return map[string]any{
 		"structName":       ms.getName(),

internal/cmd/pdatagen/internal/primitive_slice_field.go

@@ -26,6 +26,14 @@ const primitiveSliceCopyOrigTemplate = `dest.{{ .originFieldName }} =
 {{- if .isCommon }}{{ if not .isBaseStructCommon }}internal.{{ end }}CopyOrig{{ else }}copyOrig{{ end }}
 {{- .returnType }}(dest.{{ .originFieldName }}, src.{{ .originFieldName }})`
 
+const primitiveSliceMarshalJSONTemplate = `dest.WriteObjectField("{{ lowerFirst .originFieldName }}")
+	{{- if .isCommon }}
+	{{ if not .isBaseStructCommon }}internal.{{ end }}MarshalJSONStream{{ .returnType }}(
+	{{- if not .isBaseStructCommon }}internal.{{ end }}New{{ .returnType }}(&ms.orig.{{ .originFieldName }}, ms.state), dest)
+	{{- else }}
+	ms.{{ .fieldName }}().marshalJSONStream(dest)
+	{{- end }}`
+
 // PrimitiveSliceField is used to generate fields for slice of primitive types
 type PrimitiveSliceField struct {
 	fieldName         string
@@ -56,6 +64,11 @@ func (psf *PrimitiveSliceField) GenerateCopyOrig(ms *messageStruct) string {
 	return executeTemplate(t, psf.templateFields(ms))
 }
 
+func (psf *PrimitiveSliceField) GenerateMarshalJSON(ms *messageStruct) string {
+	t := template.Must(templateNew("primitiveSliceMarshalJSONTemplate").Parse(primitiveSliceMarshalJSONTemplate))
+	return executeTemplate(t, psf.templateFields(ms))
+}
+
 func (psf *PrimitiveSliceField) templateFields(ms *messageStruct) map[string]any {
 	return map[string]any{
 		"structName": ms.getName(),

internal/cmd/pdatagen/internal/slice_field.go

@@ -46,6 +46,16 @@ const sliceCopyOrigTemplate = `dest.{{ .originFieldName }} =
 {{- if .isCommon }}{{ if not .isBaseStructCommon }}internal.{{ end }}CopyOrig{{ else }}copyOrig{{ end }}
 {{- .returnType }}(dest.{{ .originFieldName }}, src.{{ .originFieldName }})`
 
+const sliceMarshalJSONTemplate = `if len(ms.orig.{{ .originFieldName }}) > 0 {
+		dest.WriteObjectField("{{ lowerFirst .originFieldName }}")
+		{{- if .isCommon }}
+		{{ if not .isBaseStructCommon }}internal.{{ end }}MarshalJSONStream{{ .returnType }}(
+		{{- if not .isBaseStructCommon }}internal.{{ end }}New{{ .returnType }}(&ms.orig.{{ .originFieldName }}, ms.state), dest)
+		{{- else }}
+		ms.{{ .fieldName }}().marshalJSONStream(dest)
+		{{- end }}
+	}`
+
 type SliceField struct {
 	fieldName       string
 	originFieldName string
@@ -79,6 +89,11 @@ func (sf *SliceField) GenerateCopyOrig(ms *messageStruct) string {
 	return executeTemplate(t, sf.templateFields(ms))
 }
 
+func (sf *SliceField) GenerateMarshalJSON(ms *messageStruct) string {
+	t := template.Must(templateNew("sliceMarshalJSONTemplate").Parse(sliceMarshalJSONTemplate))
+	return executeTemplate(t, sf.templateFields(ms))
+}
+
 func (sf *SliceField) templateFields(ms *messageStruct) map[string]any {
 	return map[string]any{
 		"structName": ms.getName(),

internal/cmd/pdatagen/internal/templates/message.go.tmpl

@@ -7,9 +7,9 @@
 package {{ .packageName }}
 
 import (
-{{ range $index, $element := .imports -}}
-{{ $element }}
-{{ end }}
+	{{ range $index, $element := .imports -}}
+	{{ $element }}
+	{{ end }}
 )
 
 {{ .description }}
@@ -81,6 +81,15 @@ func (ms {{ .structName }}) getState() *internal.State {
 	return internal.Get{{ .structName }}State(internal.{{ .structName }}(ms))
 }
 {{- else }}
+// marshalJSONStream marshals all properties from the current struct to the destination stream.
+func (ms {{ .structName }}) marshalJSONStream(dest *json.Stream) {
+	dest.WriteObjectStart()
+	{{ range .fields -}}
+	{{ .GenerateMarshalJSON $.messageStruct }}
+	{{ end -}}
+	dest.WriteObjectEnd()
+}
+
 func copyOrig{{ .structName }}(dest, src *{{ .originName }}) {
 	{{- range .fields }}
 	{{ .GenerateCopyOrig $.messageStruct }}