Skip to content

Commit 53e8f30

Browse files
Asphalttbrb
authored andcommitted
Accelerate attaching/detaching kprobes
In order to accelerate attaching/detaching kprobes, do attach/detach kprobes concurrently. By concurrent way, it is a little faster than original way. On my 6 CPU cores VM, run by concurrent way: ``` 2023/10/25 14:16:03 Attaching kprobes (via kprobe)... 1462 / 1462 [----------------------------------------------------------------------------------------------------] 100.00% 342 p/s 2023/10/25 14:16:07 Attached (ignored 0) 2023/10/25 14:16:07 Listening for events.. SKB CPU PROCESS FUNC ^C2023/10/25 14:16:08 Received signal, exiting program.. 2023/10/25 14:16:08 Detaching kprobes... 1462 / 1462 [-----------------------------------------------------------------------------------------------------] 100.00% 35 p/s ``` run by original way: ``` 2023/10/25 14:17:27 Attaching kprobes (via kprobe)... 1462 / 1462 [----------------------------------------------------------------------------------------------------] 100.00% 282 p/s 2023/10/25 14:17:32 Attached (ignored 0) 2023/10/25 14:17:32 Listening for events.. SKB CPU PROCESS FUNC ^C2023/10/25 14:17:33 Received signal, exiting program.. 2023/10/25 14:17:33 Detaching kprobes... 1462 / 1462 [-----------------------------------------------------------------------------------------------------] 100.00% 21 p/s ``` Signed-off-by: Leon Hwang <[email protected]>
1 parent fa55033 commit 53e8f30

File tree

4 files changed

+235
-73
lines changed

4 files changed

+235
-73
lines changed

README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ Usage: pwru [options] [pcap-filter]
5656
--backend string Tracing backend('kprobe', 'kprobe-multi'). Will auto-detect if not specified.
5757
--filter-func string filter kernel functions to be probed by name (exact match, supports RE2 regular expression)
5858
--filter-ifname string filter skb ifname in --filter-netns (if not specified, use current netns)
59+
--filter-kprobe-batch uint batch size for kprobe attaching/detaching (default 10)
5960
--filter-mark uint32 filter skb mark
6061
--filter-netns string filter netns ("/proc/<pid>/ns/net", "inode:<inode>")
6162
--filter-trace-tc trace TC bpf progs
@@ -138,7 +139,7 @@ See [docs/vagrant.md](docs/vagrant.md)
138139

139140
* Go >= 1.16
140141
* LLVM/clang >= 1.12
141-
* Bison
142+
* Bison
142143
* Lex/Flex >= 2.5.31
143144

144145
### Building

internal/pwru/kprobe.go

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
/* Copyright 2024 Authors of Cilium */
3+
4+
package pwru
5+
6+
import (
7+
"context"
8+
"errors"
9+
"fmt"
10+
"log"
11+
"os"
12+
"sync"
13+
"syscall"
14+
15+
"github.com/cheggaaa/pb/v3"
16+
"github.com/cilium/ebpf"
17+
"github.com/cilium/ebpf/link"
18+
"golang.org/x/sync/errgroup"
19+
)
20+
21+
type Kprobe struct {
22+
hookFunc string // internal use
23+
HookFuncs []string
24+
Prog *ebpf.Program
25+
}
26+
27+
func attachKprobes(ctx context.Context, bar *pb.ProgressBar, kprobes []Kprobe) (links []link.Link, ignored int, err error) {
28+
links = make([]link.Link, 0, len(kprobes))
29+
for _, kprobe := range kprobes {
30+
select {
31+
case <-ctx.Done():
32+
return
33+
34+
default:
35+
}
36+
37+
var kp link.Link
38+
kp, err = link.Kprobe(kprobe.hookFunc, kprobe.Prog, nil)
39+
if err != nil {
40+
if !errors.Is(err, os.ErrNotExist) && !errors.Is(err, syscall.EADDRNOTAVAIL) {
41+
err = fmt.Errorf("opening kprobe %s: %w", kprobe.hookFunc, err)
42+
return
43+
} else {
44+
err = nil
45+
ignored++
46+
}
47+
} else {
48+
links = append(links, kp)
49+
}
50+
51+
bar.Increment()
52+
}
53+
54+
return
55+
}
56+
57+
// AttachKprobes attaches kprobes concurrently.
58+
func AttachKprobes(ctx context.Context, bar *pb.ProgressBar, kps []Kprobe, batch uint) (links []link.Link, ignored int) {
59+
if batch == 0 {
60+
log.Fatal("--filter-kprobe-batch must be greater than 0")
61+
}
62+
63+
var kprobes []Kprobe
64+
for _, kp := range kps {
65+
for _, fn := range kp.HookFuncs {
66+
kprobes = append(kprobes, Kprobe{
67+
hookFunc: fn,
68+
Prog: kp.Prog,
69+
})
70+
}
71+
}
72+
73+
if len(kprobes) == 0 {
74+
return
75+
}
76+
77+
errg, ctx := errgroup.WithContext(ctx)
78+
79+
var mu sync.Mutex
80+
links = make([]link.Link, 0, len(kprobes))
81+
82+
attaching := func(kprobes []Kprobe) error {
83+
l, i, e := attachKprobes(ctx, bar, kprobes)
84+
if e != nil {
85+
return e
86+
}
87+
88+
mu.Lock()
89+
links = append(links, l...)
90+
ignored += i
91+
mu.Unlock()
92+
93+
return nil
94+
}
95+
96+
var i uint
97+
for i = 0; i+batch < uint(len(kprobes)); i += batch {
98+
kps := kprobes[i : i+batch]
99+
errg.Go(func() error {
100+
return attaching(kps)
101+
})
102+
}
103+
if i < uint(len(kprobes)) {
104+
kps := kprobes[i:]
105+
errg.Go(func() error {
106+
return attaching(kps)
107+
})
108+
}
109+
110+
if err := errg.Wait(); err != nil {
111+
log.Fatalf("Attaching kprobes: %v\n", err)
112+
}
113+
114+
return
115+
}
116+
117+
// DetachKprobes detaches kprobes concurrently.
118+
func DetachKprobes(links []link.Link, showProgressBar bool, batch uint) {
119+
log.Println("Detaching kprobes...")
120+
121+
if batch < 2 {
122+
for _, l := range links {
123+
_ = l.Close()
124+
}
125+
126+
return
127+
}
128+
129+
var errg errgroup.Group
130+
var bar *pb.ProgressBar
131+
132+
if showProgressBar {
133+
bar = pb.StartNew(len(links))
134+
defer bar.Finish()
135+
}
136+
increment := func() {
137+
if showProgressBar {
138+
bar.Increment()
139+
}
140+
}
141+
142+
var i uint
143+
for i = 0; i+batch < uint(len(links)); i += batch {
144+
l := links[i : i+batch]
145+
errg.Go(func() error {
146+
for _, l := range l {
147+
_ = l.Close()
148+
increment()
149+
}
150+
return nil
151+
})
152+
}
153+
for ; i < uint(len(links)); i++ {
154+
_ = links[i].Close()
155+
increment()
156+
}
157+
158+
_ = errg.Wait()
159+
}
160+
161+
// AttachKprobeMulti attaches kprobe-multi serially.
162+
func AttachKprobeMulti(ctx context.Context, bar *pb.ProgressBar, kprobes []Kprobe, a2n Addr2Name) (links []link.Link, ignored int) {
163+
links = make([]link.Link, 0, len(kprobes))
164+
165+
for _, kp := range kprobes {
166+
select {
167+
case <-ctx.Done():
168+
return
169+
default:
170+
}
171+
172+
addrs := make([]uintptr, 0, len(kp.HookFuncs))
173+
for _, fn := range kp.HookFuncs {
174+
if addr, ok := a2n.Name2AddrMap[fn]; ok {
175+
addrs = append(addrs, addr...)
176+
} else {
177+
ignored += 1
178+
bar.Increment()
179+
continue
180+
}
181+
}
182+
183+
if len(addrs) == 0 {
184+
continue
185+
}
186+
187+
opts := link.KprobeMultiOptions{Addresses: addrs}
188+
l, err := link.KprobeMulti(kp.Prog, opts)
189+
bar.Add(len(kp.HookFuncs))
190+
if err != nil {
191+
log.Fatalf("Opening kprobe-multi: %s\n", err)
192+
}
193+
194+
links = append(links, l)
195+
}
196+
197+
return
198+
}

internal/pwru/types.go

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,14 @@ type Flags struct {
2525

2626
KernelBTF string
2727

28-
FilterNetns string
29-
FilterMark uint32
30-
FilterFunc string
31-
FilterTrackSkb bool
32-
FilterTraceTc bool
33-
FilterIfname string
34-
FilterPcap string
28+
FilterNetns string
29+
FilterMark uint32
30+
FilterFunc string
31+
FilterTrackSkb bool
32+
FilterTraceTc bool
33+
FilterIfname string
34+
FilterPcap string
35+
FilterKprobeBatch uint
3536

3637
OutputTS string
3738
OutputMeta bool
@@ -61,6 +62,7 @@ func (f *Flags) SetFlags() {
6162
flag.BoolVar(&f.FilterTrackSkb, "filter-track-skb", false, "trace a packet even if it does not match given filters (e.g., after NAT or tunnel decapsulation)")
6263
flag.BoolVar(&f.FilterTraceTc, "filter-trace-tc", false, "trace TC bpf progs")
6364
flag.StringVar(&f.FilterIfname, "filter-ifname", "", "filter skb ifname in --filter-netns (if not specified, use current netns)")
65+
flag.UintVar(&f.FilterKprobeBatch, "filter-kprobe-batch", 10, "batch size for kprobe attaching/detaching")
6466
flag.StringVar(&f.OutputTS, "timestamp", "none", "print timestamp per skb (\"current\", \"relative\", \"absolute\", \"none\")")
6567
flag.BoolVar(&f.OutputMeta, "output-meta", false, "print skb metadata")
6668
flag.BoolVar(&f.OutputTuple, "output-tuple", false, "print L4 tuple")

main.go

Lines changed: 26 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -195,21 +195,19 @@ func main() {
195195

196196
var kprobes []link.Link
197197
defer func() {
198+
var showProgressBar bool
198199
select {
199200
case <-ctx.Done():
200-
log.Println("Detaching kprobes...")
201-
bar := pb.StartNew(len(kprobes))
202-
for _, kp := range kprobes {
203-
_ = kp.Close()
204-
bar.Increment()
205-
}
206-
bar.Finish()
201+
showProgressBar = true
207202

208203
default:
209-
for _, kp := range kprobes {
210-
_ = kp.Close()
211-
}
212204
}
205+
206+
batch := uint(1)
207+
if !useKprobeMulti {
208+
batch = flags.FilterKprobeBatch
209+
}
210+
pwru.DetachKprobes(kprobes, showProgressBar, batch)
213211
}()
214212

215213
msg := "kprobe"
@@ -257,69 +255,32 @@ func main() {
257255
}
258256
}
259257

258+
pwruKprobes := make([]pwru.Kprobe, 0, len(funcs))
260259
funcsByPos := pwru.GetFuncsByPos(funcs)
261260
for pos, fns := range funcsByPos {
262261
fn, ok := coll.Programs[fmt.Sprintf("kprobe_skb_%d", pos)]
263-
if !ok {
264-
ignored += len(fns)
265-
bar.Add(len(fns))
266-
continue
267-
}
268-
269-
if !useKprobeMulti {
270-
for _, name := range fns {
271-
select {
272-
case <-ctx.Done():
273-
bar.Finish()
274-
return
275-
default:
276-
}
277-
278-
kp, err := link.Kprobe(name, fn, nil)
279-
bar.Increment()
280-
if err != nil {
281-
if !errors.Is(err, os.ErrNotExist) && !errors.Is(err, syscall.EADDRNOTAVAIL) {
282-
log.Fatalf("Opening kprobe %s: %s\n", name, err)
283-
} else {
284-
ignored += 1
285-
}
286-
} else {
287-
kprobes = append(kprobes, kp)
288-
}
289-
}
262+
if ok {
263+
pwruKprobes = append(pwruKprobes, pwru.Kprobe{HookFuncs: fns, Prog: fn})
290264
} else {
291-
select {
292-
case <-ctx.Done():
293-
bar.Finish()
294-
return
295-
default:
296-
}
297-
298-
addrs := make([]uintptr, 0, len(fns))
299-
for _, fn := range fns {
300-
if addr, ok := addr2name.Name2AddrMap[fn]; ok {
301-
addrs = append(addrs, addr...)
302-
} else {
303-
ignored += 1
304-
bar.Increment()
305-
continue
306-
}
307-
}
308-
309-
if len(addrs) == 0 {
310-
continue
311-
}
312-
313-
opts := link.KprobeMultiOptions{Addresses: addrs}
314-
kp, err := link.KprobeMulti(fn, opts)
265+
ignored += len(fns)
315266
bar.Add(len(fns))
316-
if err != nil {
317-
log.Fatalf("Opening kprobe-multi for pos %d: %s\n", pos, err)
318-
}
319-
kprobes = append(kprobes, kp)
320267
}
321268
}
269+
if !useKprobeMulti {
270+
l, i := pwru.AttachKprobes(ctx, bar, pwruKprobes, flags.FilterKprobeBatch)
271+
kprobes = append(kprobes, l...)
272+
ignored += i
273+
} else {
274+
l, i := pwru.AttachKprobeMulti(ctx, bar, pwruKprobes, addr2name)
275+
kprobes = append(kprobes, l...)
276+
ignored += i
277+
}
322278
bar.Finish()
279+
select {
280+
case <-ctx.Done():
281+
return
282+
default:
283+
}
323284
log.Printf("Attached (ignored %d)\n", ignored)
324285

325286
log.Println("Listening for events..")

0 commit comments

Comments
 (0)