Skip to content

Commit f4a8ec5

Browse files
tracee-bench: tool to track performance information (#1985)
tracee-bench is a cli tool to track performance information of tracee based apps exported through prometheus. Will be used as a performance probe as part of EPIC #1986.
1 parent f35e039 commit f4a8ec5

File tree

8 files changed

+279
-0
lines changed

8 files changed

+279
-0
lines changed

Makefile

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ help:
217217
@echo " $$ make bpf-nocore # build ./dist/tracee.bpf.XXX.o"
218218
@echo " $$ make tracee-ebpf # build ./dist/tracee-ebpf"
219219
@echo " $$ make tracee-rules # build ./dist/tracee-rules"
220+
@echo " $$ make tracee-bench # build ./dist/tracee-bench"
220221
@echo " $$ make rules # build ./dist/rules"
221222
@echo ""
222223
@echo "# install"
@@ -231,6 +232,7 @@ help:
231232
@echo " $$ make clean-bpf-nocore # wipe ./dist/tracee.bpf.XXX.o"
232233
@echo " $$ make clean-tracee-ebpf # wipe ./dist/tracee-ebpf"
233234
@echo " $$ make clean-tracee-rules # wipe ./dist/tracee-rules"
235+
@echo " $$ make clean-tracee-bench # wipe ./dist/tracee-bench"
234236
@echo " $$ make clean-rules # wipe ./dist/rules"
235237
@echo ""
236238
@echo "# test"
@@ -694,6 +696,27 @@ check-staticcheck: \
694696
-tags $(GO_TAGS_EBPF) \
695697
./...
696698

699+
#
700+
# tracee-bench
701+
#
702+
703+
TRACEE_BENCH_SRC_DIRS = ./cmd/tracee-bench/
704+
TRACEE_BENCH_SRC = $(shell find $(TRACEE_BENCH_SRC_DIRS) -type f -name '*.go')
705+
.PHONY: tracee-bench
706+
tracee-bench: \
707+
.checkver_$(CMD_GO) \
708+
$(TRACEE_BENCH_SRC) \
709+
| $(OUTPUT_DIR)
710+
#
711+
$(CMD_GO) build \
712+
-v -o $(OUTPUT_DIR)/tracee-bench \
713+
./cmd/tracee-bench
714+
715+
.PHONY: clean-tracee-bench
716+
clean-tracee-bench:
717+
#
718+
$(CMD_RM) -rf $(OUTPUT_DIR)/tracee-bench
719+
697720
#
698721
# clean
699722
#

cmd/tracee-bench/main.go

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"log"
8+
"os"
9+
"os/signal"
10+
"strconv"
11+
"strings"
12+
"sync"
13+
"syscall"
14+
"time"
15+
16+
"github.com/prometheus/client_golang/api"
17+
v1 "github.com/prometheus/client_golang/api/prometheus/v1"
18+
"github.com/urfave/cli/v2"
19+
)
20+
21+
const (
22+
prometheusAddressFlag = "prometheus"
23+
periodFlag = "period"
24+
outputFlag = "output"
25+
)
26+
27+
type measurement struct {
28+
AvgEbpfRate float64 `json:"avgEbpfRate"`
29+
AvgLostEventsRate float64 `json:"avgLostEventsRate"`
30+
LostEvents int `json:"lostEvents"`
31+
}
32+
33+
func (m measurement) Print() {
34+
log.Printf("\n")
35+
fmt.Printf("Events/Sec: %f\n", m.AvgEbpfRate)
36+
fmt.Printf("EventsLost/Sec: %f\n", m.AvgLostEventsRate)
37+
fmt.Printf("Events Lost: %d\n", m.LostEvents)
38+
fmt.Println("===============================================")
39+
}
40+
func (m measurement) PrintJson() {
41+
res, _ := json.Marshal(m)
42+
fmt.Println(string(res))
43+
}
44+
45+
type OutputMode string
46+
47+
const (
48+
jsonOutput OutputMode = "json"
49+
prettyOutput OutputMode = "pretty"
50+
)
51+
52+
func main() {
53+
app := cli.App{
54+
Name: "tracee-bench",
55+
Usage: "A prometheus based performance probe for tracee",
56+
Flags: []cli.Flag{
57+
&cli.StringFlag{
58+
Name: prometheusAddressFlag,
59+
Usage: "address of a prometheus instance tracking tracee",
60+
Value: "http://localhost:9090",
61+
},
62+
&cli.IntFlag{
63+
Name: periodFlag,
64+
Usage: "period of scraping in seconds",
65+
Value: 5,
66+
},
67+
&cli.StringFlag{
68+
Name: outputFlag,
69+
Usage: "set output format (options: pretty, json)",
70+
Value: "pretty",
71+
},
72+
},
73+
Action: func(ctx *cli.Context) error {
74+
address := ctx.String(prometheusAddressFlag)
75+
76+
if address == "" {
77+
return fmt.Errorf("prometheus address required for tracee-er")
78+
}
79+
80+
client, err := api.NewClient(api.Config{
81+
Address: address,
82+
})
83+
84+
if err != nil {
85+
return err
86+
}
87+
88+
done := sigHandler()
89+
prom := v1.NewAPI(client)
90+
ticker := time.NewTicker(time.Duration(ctx.Int(periodFlag)) * time.Second)
91+
92+
// promql queries
93+
const (
94+
eventspersec = "events/sec"
95+
lostpersec = "lost/sec"
96+
rulespersec = "rules/sec"
97+
lostoverall = "lost_events"
98+
)
99+
queries := map[string]struct {
100+
queryName string
101+
query string
102+
}{
103+
eventspersec: {queryName: "average ebpf_events/sec", query: "rate(tracee_ebpf_events_total[1m])"},
104+
lostpersec: {queryName: "average ebpf_lostevents/sec", query: "rate(tracee_ebpf_lostevents_total[1m])"},
105+
lostoverall: {queryName: "lost events", query: "tracee_ebpf_lostevents_total"},
106+
}
107+
108+
outputMode := OutputMode(ctx.String(outputFlag))
109+
if outputMode == prettyOutput {
110+
fmt.Println("===================TRACEE-ER===================")
111+
}
112+
go func() {
113+
for {
114+
select {
115+
case <-done:
116+
{
117+
return
118+
}
119+
case now := <-ticker.C:
120+
{
121+
122+
measurement := measurement{}
123+
wg := sync.WaitGroup{}
124+
wg.Add((len(queries)))
125+
for field, query := range queries {
126+
go func(queryField string, queryName string, query string) {
127+
defer wg.Done()
128+
res, _, err := prom.Query(context.Background(), query, now)
129+
if err != nil {
130+
log.Printf("failed to fetch %s: %v\n", queryName, err)
131+
return
132+
}
133+
134+
queryResString := res.String()
135+
if queryResString == "" {
136+
log.Printf("failed to fetch %s: empty\n", queryName)
137+
return
138+
}
139+
val, _ := parseQueryResString(queryResString)
140+
switch queryField {
141+
case eventspersec:
142+
measurement.AvgEbpfRate = val
143+
case lostpersec:
144+
measurement.AvgLostEventsRate = val
145+
case lostoverall:
146+
measurement.LostEvents = int(val)
147+
}
148+
}(field, query.queryName, query.query)
149+
}
150+
wg.Wait()
151+
switch outputMode {
152+
case prettyOutput:
153+
measurement.Print()
154+
case jsonOutput:
155+
measurement.PrintJson()
156+
}
157+
}
158+
}
159+
}
160+
}()
161+
<-done
162+
return nil
163+
},
164+
}
165+
err := app.Run(os.Args)
166+
if err != nil {
167+
log.Fatal(err)
168+
}
169+
}
170+
171+
func parseQueryResString(queryRes string) (float64, error) {
172+
startIndex := strings.LastIndex(queryRes, "=> ") + 3
173+
lastIndex := strings.LastIndex(queryRes, "@[") - 1
174+
return strconv.ParseFloat(queryRes[startIndex:lastIndex], 64)
175+
}
176+
177+
func sigHandler() chan bool {
178+
sigs := make(chan os.Signal, 1)
179+
done := make(chan bool, 1)
180+
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
181+
go func() {
182+
<-sigs
183+
done <- true
184+
}()
185+
return done
186+
}

cmd/tracee-bench/main_test.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package main
2+
3+
import (
4+
"testing"
5+
6+
"github.com/stretchr/testify/assert"
7+
)
8+
9+
func TestParseQueryResString(t *testing.T) {
10+
testCases := []struct {
11+
resultString string
12+
expectedResult float64
13+
}{
14+
{
15+
resultString: `{instance="localhost:3366", job="prometheus"} => 4485.236363636363 @[1648131101.208]`,
16+
expectedResult: 4485.236363636363,
17+
},
18+
{
19+
resultString: `{instance="localhost:3366", job="prometheus"} => 4544.054545454545 @[1648131106.208]`,
20+
expectedResult: 4544.054545454545,
21+
},
22+
}
23+
24+
for _, tc := range testCases {
25+
res, err := parseQueryResString(tc.resultString)
26+
assert.NoError(t, err)
27+
assert.Equal(t, tc.expectedResult, res)
28+
}
29+
}

cmd/tracee-bench/prometheus.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
docker run -d --rm \
4+
-p 9090:9090 \
5+
--net=host \
6+
--mount type=bind,source="$(pwd)"/prometheus.yml,target=/etc/prometheus/prometheus.yml \
7+
prom/prometheus

cmd/tracee-bench/prometheus.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# A scrape configuration containing exactly one endpoint to scrape:
2+
scrape_configs:
3+
- job_name: 'tracee'
4+
scrape_interval: 5s
5+
static_configs:
6+
- targets: ['localhost:3366', 'localhost:4466']

cmd/tracee-bench/readme.md

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Documentation
2+
3+
`tracee-bench` is a promQL based tool to query `tracee-ebpf` runtime performance metrics.
4+
It can be used to benchmark tracee's event pipeline's (see [here](https://aquasecurity.github.io/tracee/dev/architecture/)) performance on your environment.
5+
6+
## Enabling Prometheus
7+
8+
In order to use prometheus with tracee see [this](https://aquasecurity.github.io/tracee/dev/integrating/prometheus/) documentation.
9+
A simple script for running a prometheus container scraping tracee is available in this repository in `prometheus.sh`.
10+
11+
## Metrics tracked
12+
13+
`tracee-bench` tracks three important stats about tracee's performance in your environment:1
14+
1. Avg rate of events emitted per second
15+
2. Avg rate of events lost per second
16+
3. Overall events lost
17+
18+
Ideal performance of tracee should have a stable throughput of events emitted with minimal event loss. If heavy event loss occurs, consider tuning tracee either through [filtering](https://aquasecurity.github.io/tracee/dev/tracing/event-filtering/) or allocating additional CPU (if running on kubernetes).

go.mod

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ require (
6060
github.com/google/uuid v1.3.0 // indirect
6161
github.com/huandu/xstrings v1.3.1 // indirect
6262
github.com/imdario/mergo v0.3.12 // indirect
63+
github.com/json-iterator/go v1.1.12 // indirect
6364
github.com/klauspost/compress v1.13.6 // indirect
6465
github.com/magiconair/properties v1.8.5 // indirect
6566
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
@@ -70,6 +71,8 @@ require (
7071
github.com/moby/sys/mountinfo v0.5.0 // indirect
7172
github.com/moby/sys/signal v0.6.0 // indirect
7273
github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
74+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
75+
github.com/modern-go/reflect2 v1.0.2 // indirect
7376
github.com/morikuni/aec v1.0.0 // indirect
7477
github.com/opencontainers/go-digest v1.0.0 // indirect
7578
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect

go.sum

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,11 +638,13 @@ github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52Cu
638638
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
639639
github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
640640
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
641+
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
641642
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
642643
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
643644
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
644645
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
645646
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
647+
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
646648
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
647649
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
648650
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -732,9 +734,11 @@ github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXy
732734
github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0=
733735
github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
734736
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
737+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
735738
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
736739
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
737740
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
741+
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
738742
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
739743
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
740744
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
@@ -743,6 +747,7 @@ github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2
743747
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
744748
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
745749
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
750+
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
746751
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
747752
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
748753
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
@@ -1203,6 +1208,7 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ
12031208
golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
12041209
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
12051210
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
1211+
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg=
12061212
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
12071213
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
12081214
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1446,6 +1452,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
14461452
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
14471453
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
14481454
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
1455+
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
14491456
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
14501457
google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=
14511458
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=

0 commit comments

Comments
 (0)