Skip to content

Commit 2b618c4

Browse files
travisdownstravisdowns
committed
kgo-verifier: update franz-go to always retry EOF
Tests are flaking flake when kgo-repeater dies with a non-retriable error right after connection. This happens when franz-go gets EOF right after connecting but before receiving any response, as a heuristic it assumes this is a SASL misconfiguration as that is the broker behavior in that case. However, this can also occur because Redpanda is stopped/killed after the connection is made but before the initial requests can be responded to. This means the producer will fail if a producer dies/is killed/stops during a critical window between the connection and receiving the first response. This is reasonably likely in stress/chaos tests where producers are being started and stopped all the time. This is a relatively recent change (~6 months ago) in franz-go, which was brought in a few months ago by a franz-go upgrade. To fix this, we make use a proposed new option to franz-go, from this PR: twmb/franz-go#1198 This is not merged, so we pull in the SHA from this PR directly. When a franz-go release is made with this change, we can update to that. Details in CORE-14849. Fixes CORE-14898.
1 parent dd731a7 commit 2b618c4

File tree

4 files changed

+18
-17
lines changed

4 files changed

+18
-17
lines changed

‎tests/go/kgo-verifier/go.mod‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,11 @@ require (
99
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
1010
github.com/sirupsen/logrus v1.9.3
1111
github.com/stretchr/testify v1.11.1
12-
github.com/twmb/franz-go v1.20.2
12+
github.com/twmb/franz-go v1.20.6-0.20251204171952-b7b6b8e44d30
1313
github.com/twmb/franz-go/pkg/kadm v1.17.1
1414
github.com/twmb/franz-go/pkg/kmsg v1.12.0
1515
github.com/vectorizedio/redpanda/src/go/rpk v0.0.0-20211217123319-86af7226d9f0
16-
golang.org/x/sync v0.17.0
16+
golang.org/x/sync v0.18.0
1717
golang.org/x/time v0.9.0
1818
)
1919

@@ -37,9 +37,9 @@ require (
3737
github.com/spf13/viper v1.7.0 // indirect
3838
github.com/subosito/gotenv v1.2.0 // indirect
3939
github.com/twmb/tlscfg v1.2.0 // indirect
40-
golang.org/x/crypto v0.43.0 // indirect
41-
golang.org/x/sys v0.37.0 // indirect
42-
golang.org/x/text v0.30.0 // indirect
40+
golang.org/x/crypto v0.45.0 // indirect
41+
golang.org/x/sys v0.38.0 // indirect
42+
golang.org/x/text v0.31.0 // indirect
4343
gopkg.in/ini.v1 v1.51.0 // indirect
4444
gopkg.in/yaml.v2 v2.4.0 // indirect
4545
gopkg.in/yaml.v3 v3.0.1 // indirect

‎tests/go/kgo-verifier/go.sum‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -290,10 +290,8 @@ github.com/tklauser/numcpus v0.1.0/go.mod h1:i3up9VjARpkV00NkBbexuDd0RAVQz4rV5Gl
290290
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
291291
github.com/twmb/franz-go v1.2.3-0.20211104052441-7952375c09c0/go.mod h1:e5ZOdNswX/wv+jebWNX49yc9U7zgR18Xovj9ckk6mx8=
292292
github.com/twmb/franz-go v1.2.4/go.mod h1:e5ZOdNswX/wv+jebWNX49yc9U7zgR18Xovj9ckk6mx8=
293-
github.com/twmb/franz-go v1.20.1 h1:ql6+OXi0DPJPSEeOY2zApQu+IssoRLTazl+u2cy5xAo=
294-
github.com/twmb/franz-go v1.20.1/go.mod h1:YCnepDd4gl6vdzG03I5Wa57RnCTIC6DVEyMpDX/J8UA=
295-
github.com/twmb/franz-go v1.20.2 h1:CiwhyKZHW6vqSHJkh+RTxFAJkio0jBjM/JQhx/HZ72A=
296-
github.com/twmb/franz-go v1.20.2/go.mod h1:YCnepDd4gl6vdzG03I5Wa57RnCTIC6DVEyMpDX/J8UA=
293+
github.com/twmb/franz-go v1.20.6-0.20251204171952-b7b6b8e44d30 h1:YB/zq6808q6XU8FE0kmyUOF1rwDK0Txd6gImVAZa5Fo=
294+
github.com/twmb/franz-go v1.20.6-0.20251204171952-b7b6b8e44d30/go.mod h1:gZmp2nTNfKuiKKND8qAsv28VdMlr/Gf4BIcsj99Bmtk=
297295
github.com/twmb/franz-go/pkg/kadm v0.0.0-20211116225244-e97ad6b8ef3e/go.mod h1:fuA2THFeFx/ms1w1R432uxWJqq7o3Q+olvJR4NFpWcM=
298296
github.com/twmb/franz-go/pkg/kadm v1.17.1 h1:Bt02Y/RLgnFO2NP2HVP1kd2TFtGRiJZx+fSArjZDtpw=
299297
github.com/twmb/franz-go/pkg/kadm v1.17.1/go.mod h1:s4duQmrDbloVW9QTMXhs6mViTepze7JLG43xwPcAeTg=
@@ -326,8 +324,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
326324
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
327325
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
328326
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
329-
golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
330-
golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
327+
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
328+
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
331329
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
332330
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
333331
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -378,8 +376,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
378376
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
379377
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
380378
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
381-
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
382-
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
379+
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
380+
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
383381
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
384382
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
385383
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -409,8 +407,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
409407
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
410408
golang.org/x/sys v0.0.0-20211101204403-39c9dd37992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
411409
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
412-
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
413-
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
410+
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
411+
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
414412
golang.org/x/term v0.0.0-20191110171634-ad39bd3f0407/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
415413
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
416414
golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
@@ -421,8 +419,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
421419
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
422420
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
423421
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
424-
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
425-
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
422+
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
423+
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
426424
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
427425
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
428426
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=

‎tests/go/kgo-verifier/pkg/worker/repeater/repeater_worker.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,7 @@ func (v *Worker) Init() {
274274
kgo.ProducerBatchMaxBytes(1024 * 1024),
275275
kgo.RecordPartitioner(kgo.StickyKeyPartitioner(nil)),
276276
kgo.ConsumerGroup(v.config.Group),
277+
kgo.AlwaysRetryEOF(), // workaround for CORE-14849
277278
}...)
278279

279280
client, err := kgo.NewClient(opts...)

‎tests/go/kgo-verifier/pkg/worker/worker.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,8 @@ func (wc *WorkerConfig) MakeKgoOpts() []kgo.Opt {
244244
kgo.ProducerBatchMaxBytes(int32(wc.BatchMaxbytes)),
245245
kgo.MaxBufferedRecords(int(wc.MaxBufferedRecords)),
246246
kgo.RequiredAcks(kgo.AllISRAcks()),
247+
248+
kgo.AlwaysRetryEOF(), // workaround for CORE-14849
247249
}
248250

249251
if wc.CompressionType != "" {

0 commit comments

Comments
 (0)