Skip to content

Commit 4013aed

Browse files
BinaryHexerBinaryHexer
authored
loadbalancingexporter: add support for logs - 4/5 + 5/5 (#2721)
* add(exporter/loadbalancingexporter): add log exporter to exporter factory * add(exporter/loadbalancingexporter): docker demo for logging * fix(exporter/loadbalancingexporter): missing newline for testdata yaml
1 parent 539932c commit 4013aed

File tree

10 files changed

+251
-2
lines changed

10 files changed

+251
-2
lines changed

‎exporter/loadbalancingexporter/README.md

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# Trace ID aware load-balancing exporter
22

3-
This is an exporter that will consistently export spans belonging to the same trace to the same backend.
3+
Supported pipeline types: traces, logs
4+
5+
This is an exporter that will consistently export spans and logs belonging to the same trace to the same backend.
46

57
It requires a source of backend information to be provided: static, with a fixed list of backends, or DNS, with a hostname that will resolve to all IP addresses to use. The DNS resolver will periodically check for updates.
68

@@ -54,6 +56,12 @@ service:
5456
processors: []
5557
exporters:
5658
- loadbalancing
59+
logs:
60+
receivers:
61+
- otlp
62+
processors: []
63+
exporters:
64+
- loadbalancing
5765
```
5866
5967
For testing purposes, the following configuration can be used, where both the load balancer and all backends are running locally:
@@ -133,6 +141,37 @@ service:
133141
processors: []
134142
exporters:
135143
- logging
144+
145+
logs/loadbalancer:
146+
receivers:
147+
- otlp/loadbalancer
148+
processors: []
149+
exporters:
150+
- loadbalancing
151+
logs/backend-1:
152+
receivers:
153+
- otlp/backend-1
154+
processors: []
155+
exporters:
156+
- logging
157+
logs/backend-2:
158+
receivers:
159+
- otlp/backend-2
160+
processors: []
161+
exporters:
162+
- logging
163+
logs/backend-3:
164+
receivers:
165+
- otlp/backend-3
166+
processors: []
167+
exporters:
168+
- logging
169+
logs/backend-4:
170+
receivers:
171+
- otlp/backend-4
172+
processors: []
173+
exporters:
174+
- logging
136175
```
137176
138177
## Metrics

‎exporter/loadbalancingexporter/example/Dockerfile

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
FROM golang:1.14 AS build
2+
3+
WORKDIR /src
4+
ADD . /src
5+
6+
RUN make otelcontribcol
7+
8+
FROM alpine:latest as certs
9+
RUN apk --update add ca-certificates
10+
11+
FROM scratch
12+
COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
13+
COPY --from=build /src/bin/otelcontribcol_linux_amd64 /otelcontribcol
14+
ENTRYPOINT ["/otelcontribcol"]
15+
EXPOSE 55680 55679

‎exporter/loadbalancingexporter/example/README.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Trace ID aware load-balancing exporter demo
2+
3+
Supported pipeline types: logs
4+
5+
## How to run
6+
7+
1. From the root of the project, run:
8+
```shell
9+
docker build -t otelcontribcol .
10+
```
11+
12+
2. Then from this directory (exporter/loadbalacingexporter/example), run:
13+
```shell
14+
docker-compose up
15+
```
16+
17+
## How does it work
18+
19+
- The `mingrammer/flog` container is used as a log generator.
20+
- The `otel-agent` container is used as the otel-collector that receives log using `fluentforward`.
21+
- The `otel-collector-{1,3}` containers are used to represent multiple otel-collector backends.
22+
- Since none of the logs generated contain `traceID`, the load-balancing is random.

‎exporter/loadbalancingexporter/example/docker-compose.yaml

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
version: "3"
2+
3+
services:
4+
otel-collector-1:
5+
image: otelcontribcol:latest
6+
command: ["--config=/etc/otel-collector-config.yaml"]
7+
volumes:
8+
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
9+
ports:
10+
- "1888" # pprof extension
11+
- "8888" # Prometheus metrics exposed by the collector
12+
- "8889" # Prometheus exporter metrics
13+
- "13133" # health_check extension
14+
- "4317" # OTLP gRPC receiver
15+
- "55670" # zpages extension
16+
17+
otel-collector-2:
18+
image: otelcontribcol:latest
19+
command: ["--config=/etc/otel-collector-config.yaml"]
20+
volumes:
21+
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
22+
ports:
23+
- "1888" # pprof extension
24+
- "8888" # Prometheus metrics exposed by the collector
25+
- "8889" # Prometheus exporter metrics
26+
- "13133" # health_check extension
27+
- "4317" # OTLP gRPC receiver
28+
- "55670" # zpages extension
29+
30+
otel-collector-3:
31+
image: otelcontribcol:latest
32+
command: ["--config=/etc/otel-collector-config.yaml"]
33+
volumes:
34+
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
35+
ports:
36+
- "1888" # pprof extension
37+
- "8888" # Prometheus metrics exposed by the collector
38+
- "8889" # Prometheus exporter metrics
39+
- "13133" # health_check extension
40+
- "4317" # OTLP gRPC receiver
41+
- "55670" # zpages extension
42+
43+
# Otel agent (running loadbalacing exporter)
44+
otel-agent:
45+
image: otelcontribcol:latest
46+
command: ["--config=/etc/otel-agent-config.yaml"]
47+
volumes:
48+
- ./otel-agent-config.yaml:/etc/otel-agent-config.yaml
49+
ports:
50+
- "1888" # pprof extension
51+
- "8888" # Prometheus metrics exposed by the collector
52+
- "8889" # Prometheus exporter metrics
53+
- "13133" # health_check extension
54+
- "4317" # OTLP gRPC receiver
55+
- "55670" # zpages extension
56+
- "24224:24224" # fluentforwarder
57+
- "24224:24224/udp" # fluentforwarder
58+
depends_on:
59+
- otel-collector-1
60+
- otel-collector-2
61+
- otel-collector-3
62+
63+
# Log generator
64+
flog:
65+
image: mingrammer/flog:0.4.3
66+
# Output 1 log per second in JSON format
67+
command: ["--format=json", "--loop", "--delay=1s", "--number=1"]
68+
depends_on:
69+
- otel-agent
70+
logging:
71+
driver: fluentd
72+
options:
73+
# Allow time for otel-agent to spin up, then forward fluentd logs to the fluentforwarder receiver.
74+
fluentd-async-connect: "true"
75+
# Use nanosecond precision
76+
fluentd-sub-second-precision: "true"

‎exporter/loadbalancingexporter/example/otel-agent-config.yaml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
receivers:
2+
fluentforward:
3+
endpoint: 0.0.0.0:24224
4+
5+
processors:
6+
batch:
7+
8+
exporters:
9+
logging:
10+
loglevel: debug
11+
loadbalancing:
12+
protocol:
13+
otlp:
14+
timeout: 1s
15+
insecure: true
16+
resolver:
17+
static:
18+
hostnames:
19+
- otel-collector-1:4317
20+
- otel-collector-2:4317
21+
- otel-collector-3:4317
22+
23+
extensions:
24+
health_check:
25+
pprof:
26+
zpages:
27+
28+
service:
29+
extensions: [pprof, zpages, health_check]
30+
pipelines:
31+
logs:
32+
receivers: [fluentforward]
33+
processors: [batch]
34+
exporters: [loadbalancing, logging]

‎exporter/loadbalancingexporter/example/otel-collector-config.yaml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
receivers:
2+
otlp:
3+
protocols:
4+
grpc:
5+
6+
processors:
7+
batch:
8+
9+
exporters:
10+
logging:
11+
loglevel: debug
12+
13+
extensions:
14+
health_check:
15+
pprof:
16+
zpages:
17+
18+
service:
19+
extensions: [pprof, zpages, health_check]
20+
pipelines:
21+
logs:
22+
receivers: [otlp]
23+
processors: [batch]
24+
exporters: [logging]

‎exporter/loadbalancingexporter/factory.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ func NewFactory() component.ExporterFactory {
3737
typeStr,
3838
createDefaultConfig,
3939
exporterhelper.WithTraces(createTraceExporter),
40+
exporterhelper.WithLogs(createLogExporter),
4041
)
4142
}
4243

@@ -58,3 +59,7 @@ func createDefaultConfig() configmodels.Exporter {
5859
func createTraceExporter(_ context.Context, params component.ExporterCreateParams, cfg configmodels.Exporter) (component.TracesExporter, error) {
5960
return newTracesExporter(params, cfg)
6061
}
62+
63+
func createLogExporter(_ context.Context, params component.ExporterCreateParams, cfg configmodels.Exporter) (component.LogsExporter, error) {
64+
return newLogsExporter(params, cfg)
65+
}

‎exporter/loadbalancingexporter/factory_test.go

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import (
2424
"go.uber.org/zap"
2525
)
2626

27-
func TestProcessorGetsCreatedWithValidConfiguration(t *testing.T) {
27+
func TestTraceExporterGetsCreatedWithValidConfiguration(t *testing.T) {
2828
// prepare
2929
factory := NewFactory()
3030
creationParams := component.ExporterCreateParams{Logger: zap.NewNop()}
@@ -45,3 +45,25 @@ func TestProcessorGetsCreatedWithValidConfiguration(t *testing.T) {
4545
assert.Nil(t, err)
4646
assert.NotNil(t, exp)
4747
}
48+
49+
func TestLogExporterGetsCreatedWithValidConfiguration(t *testing.T) {
50+
// prepare
51+
factory := NewFactory()
52+
creationParams := component.ExporterCreateParams{Logger: zap.NewNop()}
53+
cfg := &Config{
54+
ExporterSettings: configmodels.ExporterSettings{
55+
NameVal: "loadbalancing",
56+
TypeVal: "loadbalancing",
57+
},
58+
Resolver: ResolverSettings{
59+
Static: &StaticResolver{Hostnames: []string{"endpoint-1"}},
60+
},
61+
}
62+
63+
// test
64+
exp, err := factory.CreateLogsExporter(context.Background(), creationParams, cfg)
65+
66+
// verify
67+
assert.Nil(t, err)
68+
assert.NotNil(t, exp)
69+
}

‎exporter/loadbalancingexporter/testdata/config.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,3 +42,9 @@ service:
4242
processors: []
4343
exporters:
4444
- loadbalancing
45+
logs:
46+
receivers:
47+
- nop
48+
processors: []
49+
exporters:
50+
- loadbalancing

‎exporter/loadbalancingexporter/testdata/test-build-exporter-config.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,9 @@ service:
2121
processors: []
2222
exporters:
2323
- loadbalancing
24+
logs:
25+
receivers:
26+
- nop
27+
processors: []
28+
exporters:
29+
- loadbalancing

0 commit comments

Comments
 (0)