Skip to content

Commit f8c550f

Browse files
pmcollinspmcollins
authored
dotnet receiver: convert stream data to metrics (#2608)
This is the fifth PR in a series for the dotnet core diagnostics receiver. This PR converts the raw metric data extracted from the dotnet stream into Otel metrics and sends them down the pipeline. For the full implementation please see #2200
1 parent 9fd7b91 commit f8c550f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+997
-95
lines changed

‎receiver/dotnetdiagnosticsreceiver/config.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,4 +36,16 @@ type Config struct {
3636
// be displayed by the `dotnet-counters` tool:
3737
// https://docs.microsoft.com/en-us/dotnet/core/diagnostics/dotnet-counters
3838
Counters []string `mapstructure:"counters"`
39+
40+
// LocalDebugDir takes an optional directory name where stream data can be written for
41+
// offline analysis and troubleshooting. If LocalDebugDir is empty, no stream data is
42+
// written. If it has a value, MaxLocalDebugFiles also needs to be set, and stream
43+
// data will be written to disk at the specified location using the naming
44+
// convention `msg.%d.bin` as each message is received, where %d is the current
45+
// message number.
46+
LocalDebugDir string `mapstructure:"local_debug_dir"`
47+
// MaxLocalDebugFiles indicates the maximum number of files kept in LocalDebugDir. When a
48+
// file is written, the oldest one will be deleted if necessary to keep the
49+
// number of files in LocalDebugDir at the specified maximum.
50+
MaxLocalDebugFiles int `mapstructure:"max_local_debug_files"`
3951
}

‎receiver/dotnetdiagnosticsreceiver/dotnet/event_header_test.go

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ import (
2424

2525
func TestEventHeader(t *testing.T) {
2626
h := eventHeader{}
27-
err := parseEventHeader(network.NewMultiReader(network.NewDefaultFakeRW("", "", "")), &h)
27+
rw := network.NewDefaultFakeRW("", "", "")
28+
mr := network.NewMultiReader(rw, &network.NopBlobWriter{})
29+
err := parseEventHeader(mr, &h)
2830
require.NoError(t, err)
2931
}
3032

@@ -49,6 +51,7 @@ func testEventHeaderError(t *testing.T, flags headerFlags, errPos int) {
4951
0: {byte(flags)},
5052
},
5153
}
52-
err := parseEventHeader(network.NewMultiReader(pr), &eventHeader{})
54+
mr := network.NewMultiReader(pr, &network.NopBlobWriter{})
55+
err := parseEventHeader(mr, &eventHeader{})
5356
require.Error(t, err)
5457
}

‎receiver/dotnetdiagnosticsreceiver/dotnet/event_parser.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,9 @@ func parseEventBlock(r network.MultiReader, fm fieldMetadataMap) (metrics []Metr
7171
if err != nil {
7272
return
7373
}
74-
metrics = append(metrics, m)
74+
if len(m) > 0 {
75+
metrics = append(metrics, m)
76+
}
7577
}
7678

7779
return

‎receiver/dotnetdiagnosticsreceiver/dotnet/event_parser_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func TestEventParser(t *testing.T) {
2828
data, err := network.ReadBlobData(path.Join("..", "testdata"), 4)
2929
require.NoError(t, err)
3030
rw := network.NewBlobReader(data)
31-
reader := network.NewMultiReader(rw)
31+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
3232
err = reader.Seek(1131)
3333
require.NoError(t, err)
3434
metrics, err := parseEventBlock(reader, fms())
@@ -73,9 +73,9 @@ func TestEventParserErrors(t *testing.T) {
7373

7474
func testEventParserError(t *testing.T, data [][]byte, i int) {
7575
rw := network.NewBlobReader(data)
76-
reader := network.NewMultiReader(rw)
76+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
7777
err := reader.Seek(1131)
78-
rw.SetReadError(i)
78+
rw.ErrOnRead(i)
7979
require.NoError(t, err)
8080
_, err = parseEventBlock(reader, fms())
8181
require.Error(t, err)

‎receiver/dotnetdiagnosticsreceiver/dotnet/ipc_parser_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func TestIPCParser_NoErrors(t *testing.T) {
3131
0: []byte(magicTerminated),
3232
},
3333
}
34-
r := network.NewMultiReader(rw)
34+
r := network.NewMultiReader(rw, &network.NopBlobWriter{})
3535
err := parseIPC(r)
3636
require.NoError(t, err)
3737
}
@@ -43,7 +43,7 @@ func TestIPCParser_BadMagic(t *testing.T) {
4343
0: []byte("DOTNET_IPC_V2"),
4444
},
4545
}
46-
err := parseIPC(network.NewMultiReader(rw))
46+
err := parseIPC(network.NewMultiReader(rw, &network.NopBlobWriter{}))
4747
require.EqualError(t, err, `ipc header: expected magic "DOTNET_IPC_V1" got "DOTNET_IPC_V2"`)
4848
}
4949

@@ -55,7 +55,7 @@ func TestIPCParser_BadResponseID(t *testing.T) {
5555
3: {responseError},
5656
},
5757
}
58-
r := network.NewMultiReader(rw)
58+
r := network.NewMultiReader(rw, &network.NopBlobWriter{})
5959
err := parseIPC(r)
6060
assert.EqualError(t, err, "ipc header: got error response")
6161
}
@@ -70,7 +70,7 @@ func testIPCError(t *testing.T, idx int) {
7070
rw := &network.FakeRW{
7171
ReadErrIdx: idx,
7272
}
73-
r := network.NewMultiReader(rw)
73+
r := network.NewMultiReader(rw, &network.NopBlobWriter{})
7474
err := parseIPC(r)
7575
require.EqualError(t, err, fmt.Sprintf("deliberate error on read %d", idx))
7676
}

‎receiver/dotnetdiagnosticsreceiver/dotnet/metadata_parser_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func TestParseMetadata(t *testing.T) {
2828
data, err := network.ReadBlobData(path.Join("..", "testdata"), 4)
2929
require.NoError(t, err)
3030
rw := network.NewBlobReader(data)
31-
reader := network.NewMultiReader(rw)
31+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
3232
err = reader.Seek(159)
3333
require.NoError(t, err)
3434
msgs := fieldMetadataMap{}
@@ -62,11 +62,11 @@ func TestParseMetadataErrors(t *testing.T) {
6262

6363
func testParseMetadataError(t *testing.T, data [][]byte, i int) {
6464
rw := network.NewBlobReader(data)
65-
reader := network.NewMultiReader(rw)
65+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
6666
err := reader.Seek(159)
6767
require.NoError(t, err)
6868

69-
rw.SetReadError(i)
69+
rw.ErrOnRead(i)
7070

7171
msgs := fieldMetadataMap{}
7272
err = parseMetadataBlock(reader, msgs)

‎receiver/dotnetdiagnosticsreceiver/dotnet/nettrace_parser_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func TestParseNettrace(t *testing.T) {
3131
2: []byte(nettraceSerialization),
3232
},
3333
}
34-
r := network.NewMultiReader(rw)
34+
r := network.NewMultiReader(rw, &network.NopBlobWriter{})
3535
err := parseNettrace(r)
3636
require.NoError(t, err)
3737
}
@@ -43,7 +43,7 @@ func TestParseNettrace_BadHeaderName(t *testing.T) {
4343
0: []byte("nettrace"),
4444
},
4545
}
46-
r := network.NewMultiReader(rw)
46+
r := network.NewMultiReader(rw, &network.NopBlobWriter{})
4747
err := parseNettrace(r)
4848
require.EqualError(t, err, `header name: expected "Nettrace" got "nettrace"`)
4949
}
@@ -58,7 +58,7 @@ func TestParseNettrace_BadSerializationName(t *testing.T) {
5858
2: []byte(serType),
5959
},
6060
}
61-
r := network.NewMultiReader(rw)
61+
r := network.NewMultiReader(rw, &network.NopBlobWriter{})
6262
err := parseNettrace(r)
6363
require.EqualError(t, err, `serialization type: expected "!FastSerialization.1" got "foo"`)
6464
}
@@ -74,6 +74,6 @@ func testParseNettraceReadErr(i int) error {
7474
rw := &network.FakeRW{
7575
ReadErrIdx: i,
7676
}
77-
r := network.NewMultiReader(rw)
77+
r := network.NewMultiReader(rw, &network.NopBlobWriter{})
7878
return parseNettrace(r)
7979
}

‎receiver/dotnetdiagnosticsreceiver/dotnet/parser.go

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,20 @@ import (
2626

2727
// Parser encapsulates all of the functionality to parse an IPC stream.
2828
type Parser struct {
29-
r network.MultiReader
30-
logger *zap.Logger
29+
r network.MultiReader
30+
consume func([]Metric)
31+
logger *zap.Logger
3132
}
3233

33-
// NewParser accepts an io.Reader and logger returns a Parser for processing an
34-
// IPC stream.
35-
func NewParser(rdr io.Reader, logger *zap.Logger) *Parser {
36-
r := network.NewMultiReader(rdr)
37-
return &Parser{r: r, logger: logger}
34+
// MetricsConsumer is a function that accepts a slice of Metrics. Parser has a
35+
// member consumer function, used to send Metrics as they are created.
36+
type MetricsConsumer func([]Metric)
37+
38+
// NewParser accepts an io.Reader, a MetricsConsumer, and logger, and returns a
39+
// Parser for processing an IPC stream.
40+
func NewParser(rdr io.Reader, mc MetricsConsumer, bw network.BlobWriter, logger *zap.Logger) *Parser {
41+
r := network.NewMultiReader(rdr, bw)
42+
return &Parser{r: r, consume: mc, logger: logger}
3843
}
3944

4045
// ParseIPC parses the IPC response from the initial request to a dotnet process.
@@ -58,6 +63,8 @@ func (p *Parser) ParseAll(ctx context.Context) error {
5863
return nil
5964
default:
6065
err = p.parseBlock(fms)
66+
// flush regardless of error
67+
p.r.Flush()
6168
if err != nil {
6269
return err
6370
}
@@ -97,10 +104,12 @@ func (p *Parser) parseBlock(fms fieldMetadataMap) error {
97104
return err
98105
}
99106
case "EventBlock":
100-
_, err = parseEventBlock(p.r, fms)
107+
var metrics []Metric
108+
metrics, err = parseEventBlock(p.r, fms)
101109
if err != nil {
102110
return err
103111
}
112+
p.consume(metrics)
104113
case "SPBlock":
105114
err = parseSPBlock(p.r)
106115
if err != nil {

‎receiver/dotnetdiagnosticsreceiver/dotnet/parser_test.go

Lines changed: 53 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package dotnet
1616

1717
import (
18+
"context"
1819
"path"
1920
"testing"
2021

@@ -36,7 +37,12 @@ func TestParser(t *testing.T) {
3637
8: []byte(fastSerialization),
3738
},
3839
}
39-
p := NewParser(rw, zap.NewNop())
40+
p := NewParser(
41+
rw,
42+
func(metrics []Metric) {},
43+
&network.NopBlobWriter{},
44+
zap.NewNop(),
45+
)
4046
err := p.ParseIPC()
4147
require.NoError(t, err)
4248
err = p.ParseNettrace()
@@ -47,7 +53,7 @@ func TestParser_TestData(t *testing.T) {
4753
data, err := network.ReadBlobData(path.Join("..", "testdata"), 16)
4854
require.NoError(t, err)
4955
rw := network.NewBlobReader(data)
50-
parser := NewParser(rw, zap.NewNop())
56+
parser := NewParser(rw, func([]Metric) {}, &network.NopBlobWriter{}, zap.NewNop())
5157
err = parser.ParseIPC()
5258
require.NoError(t, err)
5359
err = parser.ParseNettrace()
@@ -85,10 +91,10 @@ func testParseBlockError(t *testing.T, data [][]byte, offset, errIdx int) {
8591

8692
func testParseBlock(t *testing.T, data [][]byte, offset, errIdx int) error {
8793
rw := network.NewBlobReader(data)
88-
reader := network.NewMultiReader(rw)
94+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
8995
err := reader.Seek(offset)
9096
require.NoError(t, err)
91-
rw.SetReadError(errIdx)
97+
rw.ErrOnRead(errIdx)
9298
msgs := fieldMetadataMap{}
9399
parser := &Parser{r: reader, logger: zap.NewNop()}
94100
for i := 0; i < 16; i++ {
@@ -99,3 +105,46 @@ func testParseBlock(t *testing.T, data [][]byte, offset, errIdx int) error {
99105
}
100106
return nil
101107
}
108+
109+
func TestParser_ParseAll_Error(t *testing.T) {
110+
data, err := network.ReadBlobData(path.Join("..", "testdata"), 16)
111+
require.NoError(t, err)
112+
rw := network.NewBlobReader(data)
113+
parser := NewParser(rw, func([]Metric) {}, &network.NopBlobWriter{}, zap.NewNop())
114+
err = parser.ParseIPC()
115+
require.NoError(t, err)
116+
err = parser.ParseNettrace()
117+
require.NoError(t, err)
118+
rw.StopOnRead(0)
119+
errCh := make(chan error)
120+
go func() {
121+
err = parser.ParseAll(context.Background())
122+
errCh <- err
123+
}()
124+
<-rw.Gate()
125+
rw.ErrOnRead(0)
126+
rw.Gate() <- struct{}{}
127+
require.Error(t, <-errCh)
128+
}
129+
130+
func TestParser_ParseAll_Cancel(t *testing.T) {
131+
data, err := network.ReadBlobData(path.Join("..", "testdata"), 16)
132+
require.NoError(t, err)
133+
rw := network.NewBlobReader(data)
134+
parser := NewParser(rw, func([]Metric) {}, &network.NopBlobWriter{}, zap.NewNop())
135+
err = parser.ParseIPC()
136+
require.NoError(t, err)
137+
err = parser.ParseNettrace()
138+
require.NoError(t, err)
139+
rw.StopOnRead(0)
140+
errCh := make(chan error)
141+
ctx, cancel := context.WithCancel(context.Background())
142+
go func() {
143+
err = parser.ParseAll(ctx)
144+
errCh <- err
145+
}()
146+
<-rw.Gate()
147+
cancel()
148+
rw.Gate() <- struct{}{}
149+
require.NoError(t, <-errCh)
150+
}

‎receiver/dotnetdiagnosticsreceiver/dotnet/sequence_parser_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func TestParseSPBlock(t *testing.T) {
2727
data, err := network.ReadBlobData(path.Join("..", "testdata"), 16)
2828
require.NoError(t, err)
2929
rw := network.NewBlobReader(data)
30-
reader := network.NewMultiReader(rw)
30+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
3131
err = reader.Seek(36870)
3232
require.NoError(t, err)
3333
err = parseSPBlock(reader)
@@ -44,10 +44,10 @@ func TestParseSPBlock_Errors(t *testing.T) {
4444

4545
func testParseSPBlockError(t *testing.T, data [][]byte, i int) {
4646
rw := network.NewBlobReader(data)
47-
reader := network.NewMultiReader(rw)
47+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
4848
err := reader.Seek(36870)
4949
require.NoError(t, err)
50-
rw.SetReadError(i)
50+
rw.ErrOnRead(i)
5151
err = parseSPBlock(reader)
5252
require.Error(t, err)
5353
}

‎receiver/dotnetdiagnosticsreceiver/dotnet/serialization_type_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func TestParseSerializationType(t *testing.T) {
2828
data, err := network.ReadBlobData(path.Join("..", "testdata"), 1)
2929
require.NoError(t, err)
3030
rw := network.NewBlobReader(data)
31-
reader := network.NewMultiReader(rw)
31+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
3232
err = reader.Seek(61)
3333
require.NoError(t, err)
3434
st, err := parseSerializationType(reader)
@@ -48,11 +48,11 @@ func TestParseSerializationType_Error(t *testing.T) {
4848

4949
func testParseSerializationTypeErr(t *testing.T, data [][]byte, i int) {
5050
rw := network.NewBlobReader(data)
51-
reader := network.NewMultiReader(rw)
51+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
5252
// metadata block is 61 bytes in
5353
err := reader.Seek(61)
5454
require.NoError(t, err)
55-
rw.SetReadError(i)
55+
rw.ErrOnRead(i)
5656
_, err = parseSerializationType(reader)
5757
require.Error(t, err)
5858
}

‎receiver/dotnetdiagnosticsreceiver/dotnet/stack_parser_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func TestParseStackBlock(t *testing.T) {
2727
data, err := network.ReadBlobData(path.Join("..", "testdata"), 4)
2828
require.NoError(t, err)
2929
rw := network.NewBlobReader(data)
30-
reader := network.NewMultiReader(rw)
30+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
3131
err = reader.Seek(972)
3232
require.NoError(t, err)
3333
err = parseStackBlock(reader)
@@ -44,10 +44,10 @@ func TestParseStackBlockErrors(t *testing.T) {
4444

4545
func testParseStackBlockError(t *testing.T, data [][]byte, i int) {
4646
rw := network.NewBlobReader(data)
47-
reader := network.NewMultiReader(rw)
47+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
4848
err := reader.Seek(972)
4949
require.NoError(t, err)
50-
rw.SetReadError(i)
50+
rw.ErrOnRead(i)
5151
err = parseStackBlock(reader)
5252
require.Error(t, err)
5353
}

‎receiver/dotnetdiagnosticsreceiver/dotnet/trace_parser_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func TestTraceParser(t *testing.T) {
2727
data, err := network.ReadBlobData(path.Join("..", "testdata"), 2)
2828
require.NoError(t, err)
2929
rw := network.NewBlobReader(data)
30-
reader := network.NewMultiReader(rw)
30+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
3131
err = reader.Seek(81)
3232
require.NoError(t, err)
3333
err = parseTraceMessage(reader)
@@ -45,10 +45,10 @@ func TestTraceParser_Errors(t *testing.T) {
4545

4646
func testTraceParserReadError(t *testing.T, data [][]byte, i int) {
4747
rw := network.NewBlobReader(data)
48-
reader := network.NewMultiReader(rw)
48+
reader := network.NewMultiReader(rw, &network.NopBlobWriter{})
4949
err := reader.Seek(81)
5050
require.NoError(t, err)
51-
rw.SetReadError(i)
51+
rw.ErrOnRead(i)
5252
err = parseTraceMessage(reader)
5353
require.Error(t, err)
5454
}

0 commit comments

Comments
 (0)