Generate Logs/Traces/Metrics and ExportResponse (#13597)

This removes lots of hand written JSON encoding code, and will help when
start using generated proto size/marshal/unmarshal.

This also adds tests for all previously hand written funcs.

Signed-off-by: Bogdan Drutu <[email protected]>

Author: bogdandrutu

.chloggen/generate-more.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: pdata
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Generate Logs/Traces/Metrics/Profiles and p[log|trace|metric|profile]ExportResponse with pdatagen.
+
+# One or more tracking issues or pull requests related to the change
+issues: [13597]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: This change brings consistency on how these structs are written and remove JSON marshaling/unmarshaling hand written logic.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api, user]

internal/cmd/pdatagen/internal/base_slices.go

@@ -6,6 +6,7 @@ package internal // import "go.opentelemetry.io/collector/internal/cmd/pdatagen/
 type baseSlice interface {
 	getName() string
 	getOriginName() string
+	getHasWrapper() bool
 	getElementProtoType() ProtoType
 	getElementOriginName() string
 	getPackageName() string
@@ -43,11 +44,12 @@ func (ss *sliceOfPtrs) generateInternalTests(packageInfo *PackageInfo) []byte {
 }
 
 func (ss *sliceOfPtrs) templateFields(packageInfo *PackageInfo) map[string]any {
-	orig := origAccessor(ss.packageName)
-	state := stateAccessor(ss.packageName)
+	hasWrapper := usedByOtherDataTypes(ss.packageName)
+	orig := origAccessor(hasWrapper)
+	state := stateAccessor(hasWrapper)
 	return map[string]any{
 		"type":                  "sliceOfPtrs",
-		"isCommon":              usedByOtherDataTypes(ss.packageName),
+		"hasWrapper":            usedByOtherDataTypes(ss.packageName),
 		"structName":            ss.structName,
 		"elementName":           ss.element.getName(),
 		"elementOriginFullName": ss.element.originFullName,
@@ -56,8 +58,8 @@ func (ss *sliceOfPtrs) templateFields(packageInfo *PackageInfo) map[string]any {
 		"originElementPtr":      "",
 		"emptyOriginElement":    "&" + ss.element.originFullName + "{}",
 		"newElement":            "new" + ss.element.getName() + "((*es." + orig + ")[i], es." + state + ")",
-		"origAccessor":          orig,
-		"stateAccessor":         state,
+		"origAccessor":          origAccessor(hasWrapper),
+		"stateAccessor":         stateAccessor(hasWrapper),
 		"packageName":           packageInfo.name,
 		"imports":               packageInfo.imports,
 		"testImports":           packageInfo.testImports,
@@ -72,6 +74,10 @@ func (ss *sliceOfPtrs) getElementProtoType() ProtoType {
 	return ProtoTypeMessage
 }
 
+func (ss *sliceOfPtrs) getHasWrapper() bool {
+	return usedByOtherDataTypes(ss.packageName)
+}
+
 func (ss *sliceOfPtrs) getElementOriginName() string {
 	return ss.element.getOriginName()
 }
@@ -110,11 +116,12 @@ func (ss *sliceOfValues) generateInternalTests(packageInfo *PackageInfo) []byte
 }
 
 func (ss *sliceOfValues) templateFields(packageInfo *PackageInfo) map[string]any {
-	orig := origAccessor(ss.packageName)
-	state := stateAccessor(ss.packageName)
+	hasWrapper := usedByOtherDataTypes(ss.packageName)
+	orig := origAccessor(hasWrapper)
+	state := stateAccessor(hasWrapper)
 	return map[string]any{
 		"type":                  "sliceOfValues",
-		"isCommon":              usedByOtherDataTypes(ss.packageName),
+		"hasWrapper":            usedByOtherDataTypes(ss.packageName),
 		"structName":            ss.getName(),
 		"elementName":           ss.element.getName(),
 		"elementOriginFullName": ss.element.originFullName,
@@ -139,6 +146,10 @@ func (ss *sliceOfValues) getElementProtoType() ProtoType {
 	return ProtoTypeMessage
 }
 
+func (ss *sliceOfValues) getHasWrapper() bool {
+	return usedByOtherDataTypes(ss.packageName)
+}
+
 func (ss *sliceOfValues) getElementOriginName() string {
 	return ss.element.getOriginName()
 }

internal/cmd/pdatagen/internal/base_struct.go

@@ -9,6 +9,7 @@ import (
 type baseStruct interface {
 	getName() string
 	getOriginName() string
+	getHasWrapper() bool
 	generate(packageInfo *PackageInfo) []byte
 	generateTests(packageInfo *PackageInfo) []byte
 	generateInternal(packageInfo *PackageInfo) []byte
@@ -23,6 +24,7 @@ type messageStruct struct {
 	description    string
 	originFullName string
 	fields         []Field
+	hasWrapper     bool
 }
 
 func (ms *messageStruct) getName() string {
@@ -46,22 +48,33 @@ func (ms *messageStruct) generateInternalTests(packageInfo *PackageInfo) []byte
 }
 
 func (ms *messageStruct) templateFields(packageInfo *PackageInfo) map[string]any {
+	hasWrapper := ms.hasWrapper
+	if !hasWrapper {
+		hasWrapper = usedByOtherDataTypes(ms.packageName)
+	}
 	return map[string]any{
 		"messageStruct":  ms,
 		"fields":         ms.fields,
 		"structName":     ms.getName(),
 		"originFullName": ms.originFullName,
 		"originName":     ms.getOriginName(),
 		"description":    ms.description,
-		"isCommon":       usedByOtherDataTypes(ms.packageName),
-		"origAccessor":   origAccessor(ms.packageName),
-		"stateAccessor":  stateAccessor(ms.packageName),
+		"hasWrapper":     hasWrapper,
+		"origAccessor":   origAccessor(hasWrapper),
+		"stateAccessor":  stateAccessor(hasWrapper),
 		"packageName":    packageInfo.name,
 		"imports":        packageInfo.imports,
 		"testImports":    packageInfo.testImports,
 	}
 }
 
+func (ms *messageStruct) getHasWrapper() bool {
+	if ms.hasWrapper {
+		return true
+	}
+	return usedByOtherDataTypes(ms.packageName)
+}
+
 func (ms *messageStruct) getOriginName() string {
 	_, after, _ := strings.Cut(ms.originFullName, ".")
 	return after

internal/cmd/pdatagen/internal/field.go

@@ -21,15 +21,15 @@ type Field interface {
 	GenerateMarshalProto(ms *messageStruct) string
 }
 
-func origAccessor(packageName string) string {
-	if usedByOtherDataTypes(packageName) {
+func origAccessor(hasWrapper bool) string {
+	if hasWrapper {
 		return "getOrig()"
 	}
 	return "orig"
 }
 
-func stateAccessor(packageName string) string {
-	if usedByOtherDataTypes(packageName) {
+func stateAccessor(hasWrapper bool) string {
+	if hasWrapper {
 		return "getState()"
 	}
 	return "state"

internal/cmd/pdatagen/internal/message_field.go

@@ -10,7 +10,7 @@ import (
 
 const messageAccessorsTemplate = `// {{ .fieldName }} returns the {{ .lowerFieldName }} associated with this {{ .structName }}.
 func (ms {{ .structName }}) {{ .fieldName }}() {{ .packageName }}{{ .returnType }} {
-	{{- if .isCommon }}
+	{{- if .messageHasWrapper }}
 	return {{ .packageName }}{{ .returnType }}(internal.New{{ .returnType }}(&ms.{{ .origAccessor }}.{{ .fieldOriginFullName }}, ms.{{ .stateAccessor }}))
 	{{- else }}
 	return new{{ .returnType }}(&ms.{{ .origAccessor }}.{{ .fieldOriginFullName }}, ms.{{ .stateAccessor }})
@@ -21,7 +21,7 @@ const messageAccessorsTestTemplate = `func Test{{ .structName }}_{{ .fieldName }
 	ms := New{{ .structName }}()
 	assert.Equal(t, {{ .packageName }}New{{ .returnType }}{{- if eq .returnType "Value" }}Empty{{- end }}(), ms.{{ .fieldName }}())
 	internal.FillOrigTest{{ .fieldOriginName }}(&ms.{{ .origAccessor }}.{{ .fieldOriginFullName }})
-	{{- if .isCommon }}
+	{{- if .messageHasWrapper }}
 	assert.Equal(t, {{ .packageName }}{{ .returnType }}(internal.GenerateTest{{ .returnType }}()), ms.{{ .fieldName }}())
 	{{- else }}
 	assert.Equal(t, generateTest{{ .returnType }}(), ms.{{ .fieldName }}())
@@ -41,9 +41,10 @@ const messageUnmarshalJSONTemplate = `case "{{ lowerFirst .fieldOriginFullName }
 	UnmarshalJSONOrig{{ .fieldOriginName }}(&orig.{{ .fieldOriginFullName }}, iter)`
 
 type MessageField struct {
-	fieldName     string
-	protoID       uint32
-	returnMessage *messageStruct
+	fieldName           string
+	fieldOriginFullName string
+	protoID             uint32
+	returnMessage       *messageStruct
 }
 
 func (mf *MessageField) GenerateAccessors(ms *messageStruct) string {
@@ -92,17 +93,17 @@ func (mf *MessageField) toProtoField() *ProtoField {
 	return &ProtoField{
 		Type:        pt,
 		ID:          mf.protoID,
-		Name:        mf.fieldName,
+		Name:        mf.getFieldOriginFullName(),
 		MessageName: mf.returnMessage.getOriginName(),
 	}
 }
 
 func (mf *MessageField) templateFields(ms *messageStruct) map[string]any {
 	return map[string]any{
-		"isCommon":            usedByOtherDataTypes(mf.returnMessage.packageName),
+		"messageHasWrapper":   usedByOtherDataTypes(mf.returnMessage.packageName),
 		"structName":          ms.getName(),
 		"fieldName":           mf.fieldName,
-		"fieldOriginFullName": mf.fieldName,
+		"fieldOriginFullName": mf.getFieldOriginFullName(),
 		"fieldOriginName":     mf.returnMessage.getOriginName(),
 		"lowerFieldName":      strings.ToLower(mf.fieldName),
 		"returnType":          mf.returnMessage.getName(),
@@ -112,9 +113,16 @@ func (mf *MessageField) templateFields(ms *messageStruct) map[string]any {
 			}
 			return ""
 		}(),
-		"origAccessor":  origAccessor(ms.packageName),
-		"stateAccessor": stateAccessor(ms.packageName),
+		"origAccessor":  origAccessor(ms.getHasWrapper()),
+		"stateAccessor": stateAccessor(ms.getHasWrapper()),
 	}
 }
 
+func (mf *MessageField) getFieldOriginFullName() string {
+	if mf.fieldOriginFullName != "" {
+		return mf.fieldOriginFullName
+	}
+	return mf.fieldName
+}
+
 var _ Field = (*MessageField)(nil)

internal/cmd/pdatagen/internal/one_of_field.go

@@ -129,8 +129,8 @@ func (of *OneOfField) templateFields(ms *messageStruct) map[string]any {
 		"typeName":             of.typeName,
 		"originFieldName":      of.originFieldName,
 		"lowerOriginFieldName": strings.ToLower(of.originFieldName),
-		"origAccessor":         origAccessor(ms.packageName),
-		"stateAccessor":        stateAccessor(ms.packageName),
+		"origAccessor":         origAccessor(ms.getHasWrapper()),
+		"stateAccessor":        stateAccessor(ms.getHasWrapper()),
 		"values":               of.values,
 		"originTypePrefix":     ms.originFullName + "_",
 	}

internal/cmd/pdatagen/internal/plog_package.go

@@ -17,6 +17,7 @@ var plog = &Package{
 			`"go.opentelemetry.io/collector/pdata/internal/data"`,
 			`"go.opentelemetry.io/collector/pdata/internal/json"`,
 			`"go.opentelemetry.io/collector/pdata/internal/proto"`,
+			`otlpcollectorlog "go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/logs/v1"`,
 			`otlplogs "go.opentelemetry.io/collector/pdata/internal/data/protogen/logs/v1"`,
 			`"go.opentelemetry.io/collector/pdata/pcommon"`,
 		},
@@ -30,11 +31,13 @@ var plog = &Package{
 			`"go.opentelemetry.io/collector/pdata/internal"`,
 			`"go.opentelemetry.io/collector/pdata/internal/data"`,
 			`"go.opentelemetry.io/collector/pdata/internal/json"`,
+			`otlpcollectorlog "go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/logs/v1"`,
 			`otlplogs "go.opentelemetry.io/collector/pdata/internal/data/protogen/logs/v1"`,
 			`"go.opentelemetry.io/collector/pdata/pcommon"`,
 		},
 	},
 	structs: []baseStruct{
+		logs,
 		resourceLogsSlice,
 		resourceLogs,
 		scopeLogsSlice,
@@ -44,6 +47,21 @@ var plog = &Package{
 	},
 }
 
+var logs = &messageStruct{
+	structName:     "Logs",
+	description:    "// Logs is the top-level struct that is propagated through the logs pipeline.\n// Use NewLogs to create new instance, zero-initialized instance is not valid for use.",
+	originFullName: "otlpcollectorlog.ExportLogsServiceRequest",
+	fields: []Field{
+		&SliceField{
+			fieldName:   "ResourceLogs",
+			protoID:     1,
+			protoType:   ProtoTypeMessage,
+			returnSlice: resourceLogsSlice,
+		},
+	},
+	hasWrapper: true,
+}
+
 var resourceLogsSlice = &sliceOfPtrs{
 	structName: "ResourceLogsSlice",
 	element:    resourceLogs,

internal/cmd/pdatagen/internal/plogotlp_package.go

@@ -27,10 +27,24 @@ var plogotlp = &Package{
 		},
 	},
 	structs: []baseStruct{
+		exportLogsResponse,
 		exportLogsPartialSuccess,
 	},
 }
 
+var exportLogsResponse = &messageStruct{
+	structName:     "ExportResponse",
+	description:    "// ExportResponse represents the response for gRPC/HTTP client/server.",
+	originFullName: "otlpcollectorlog.ExportLogsServiceResponse",
+	fields: []Field{
+		&MessageField{
+			fieldName:     "PartialSuccess",
+			protoID:       1,
+			returnMessage: exportLogsPartialSuccess,
+		},
+	},
+}
+
 var exportLogsPartialSuccess = &messageStruct{
 	structName:     "ExportPartialSuccess",
 	description:    "// ExportPartialSuccess represents the details of a partially successful export request.",

internal/cmd/pdatagen/internal/pmetric_package.go

@@ -17,6 +17,7 @@ var pmetric = &Package{
 			`"go.opentelemetry.io/collector/pdata/internal/data"`,
 			`"go.opentelemetry.io/collector/pdata/internal/json"`,
 			`"go.opentelemetry.io/collector/pdata/internal/proto"`,
+			`otlpcollectormetrics "go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/metrics/v1"`,
 			`otlpmetrics "go.opentelemetry.io/collector/pdata/internal/data/protogen/metrics/v1"`,
 			`"go.opentelemetry.io/collector/pdata/pcommon"`,
 		},
@@ -30,11 +31,13 @@ var pmetric = &Package{
 			`"go.opentelemetry.io/collector/pdata/internal"`,
 			`"go.opentelemetry.io/collector/pdata/internal/data"`,
 			`"go.opentelemetry.io/collector/pdata/internal/json"`,
+			`otlpcollectormetrics "go.opentelemetry.io/collector/pdata/internal/data/protogen/collector/metrics/v1"`,
 			`otlpmetrics "go.opentelemetry.io/collector/pdata/internal/data/protogen/metrics/v1"`,
 			`"go.opentelemetry.io/collector/pdata/pcommon"`,
 		},
 	},
 	structs: []baseStruct{
+		metrics,
 		resourceMetricsSlice,
 		resourceMetrics,
 		scopeMetricsSlice,
@@ -62,6 +65,21 @@ var pmetric = &Package{
 	},
 }
 
+var metrics = &messageStruct{
+	structName:     "Metrics",
+	description:    "// Metrics is the top-level struct that is propagated through the metrics pipeline.\n// Use NewMetrics to create new instance, zero-initialized instance is not valid for use.",
+	originFullName: "otlpcollectormetrics.ExportMetricsServiceRequest",
+	fields: []Field{
+		&SliceField{
+			fieldName:   "ResourceMetrics",
+			protoID:     1,
+			protoType:   ProtoTypeMessage,
+			returnSlice: resourceMetricsSlice,
+		},
+	},
+	hasWrapper: true,
+}
+
 var resourceMetricsSlice = &sliceOfPtrs{
 	structName: "ResourceMetricsSlice",
 	element:    resourceMetrics,

internal/cmd/pdatagen/internal/pmetricotlp_package.go

@@ -28,10 +28,24 @@ var pmetricotlp = &Package{
 		},
 	},
 	structs: []baseStruct{
+		exportMetricsResponse,
 		exportMetricsPartialSuccess,
 	},
 }
 
+var exportMetricsResponse = &messageStruct{
+	structName:     "ExportResponse",
+	description:    "// ExportResponse represents the response for gRPC/HTTP client/server.",
+	originFullName: "otlpcollectormetrics.ExportMetricsServiceResponse",
+	fields: []Field{
+		&MessageField{
+			fieldName:     "PartialSuccess",
+			protoID:       1,
+			returnMessage: exportMetricsPartialSuccess,
+		},
+	},
+}
+
 var exportMetricsPartialSuccess = &messageStruct{
 	structName:     "ExportPartialSuccess",
 	description:    "// ExportPartialSuccess represents the details of a partially successful export request.",