Skip to content

Commit 371f8ee

Browse files
qyryqqyryq
authored
Add more tests for partition metrics (#31206)
1 parent 09f1cae commit 371f8ee

File tree

8 files changed

+319
-21
lines changed

8 files changed

+319
-21
lines changed

‎ydb/core/persqueue/ut/counters_ut.cpp‎

Lines changed: 46 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -129,17 +129,35 @@ Y_UNIT_TEST(Partition) {
129129
}
130130
}
131131

132+
struct TPartitionLevelMetricsTestParameters {
133+
bool EnableMetricsLevel;
134+
bool FirstClassCitizen;
135+
std::optional<TString> MonitoringProjectId;
136+
};
137+
132138

133-
void PartitionLevelCounters(bool featureFlagEnabled, bool firstClassCitizen, TString referenceDir) {
139+
void PartitionLevelMetrics(TPartitionLevelMetricsTestParameters p) {
140+
Cerr << (TStringBuilder() << "Run PartitionLevelMetrics(EnableMetricsLevel=" << p.EnableMetricsLevel << ", "
141+
"FirstClassCitizen=" << p.FirstClassCitizen << ", "
142+
"MonitoringProjectId=" << p.MonitoringProjectId << ")\n");
143+
TString referenceDir = p.FirstClassCitizen ? "first_class_citizen" : "federation";
144+
if (p.MonitoringProjectId.has_value() && !p.MonitoringProjectId->empty()) {
145+
referenceDir += "_with_monitoring_project_id";
146+
}
134147
TTestContext tc;
135148
TFinalizer finalizer(tc);
136149
bool activeZone{false};
137-
tc.Prepare("", [](TTestActorRuntime&) {}, activeZone, firstClassCitizen, true);
150+
tc.Prepare("", [](TTestActorRuntime&) {}, activeZone, p.FirstClassCitizen, true);
138151
tc.Runtime->SetScheduledLimit(100);
139152

140-
tc.Runtime->GetAppData(0).FeatureFlags.SetEnableMetricsLevel(featureFlagEnabled);
153+
tc.Runtime->GetAppData(0).FeatureFlags.SetEnableMetricsLevel(p.EnableMetricsLevel);
141154

142-
PQTabletPrepare({ .metricsLevel = METRICS_LEVEL_OBJECT }, {}, tc);
155+
TTabletPreparationParameters parameters{
156+
.metricsLevel = METRICS_LEVEL_OBJECT,
157+
.monitoringProjectId = p.MonitoringProjectId,
158+
};
159+
160+
PQTabletPrepare(parameters, {}, tc);
143161
CmdWrite(0, "sourceid0", TestData(), tc, false, {}, true);
144162
CmdWrite(0, "sourceid1", TestData(), tc, false);
145163
CmdWrite(0, "sourceid2", TestData(), tc, false);
@@ -179,7 +197,8 @@ void PartitionLevelCounters(bool featureFlagEnabled, bool firstClassCitizen, TSt
179197
{
180198
// Turn on per partition counters, check counters.
181199

182-
PQTabletPrepare({ .metricsLevel = METRICS_LEVEL_DETAILED }, {}, tc);
200+
parameters.metricsLevel = METRICS_LEVEL_DETAILED;
201+
PQTabletPrepare(parameters, {}, tc);
183202

184203
// partition, sourceId, data, text
185204
CmdWrite({ .Partition = 0, .SourceId = "sourceid3", .Data = TestData(), .TestContext = tc, .Error = false });
@@ -194,8 +213,9 @@ void PartitionLevelCounters(bool featureFlagEnabled, bool firstClassCitizen, TSt
194213
Cerr << "after write: " << counters << "\n";
195214
Cerr << "after write zeroed: " << zeroUnreliableValues(counters) << "\n";
196215
TString referenceCounters = NResource::Find(TStringBuilder() << referenceDir << "_after_write.html");
197-
counters = zeroUnreliableValues(counters) + (featureFlagEnabled ? "\n" : "");
198-
UNIT_ASSERT_VALUES_EQUAL(counters, featureFlagEnabled ? referenceCounters : EMPTY_COUNTERS);
216+
Cerr << "referenceCounters: " << referenceCounters << "\n";
217+
counters = zeroUnreliableValues(counters) + (p.EnableMetricsLevel ? "\n" : "");
218+
UNIT_ASSERT_VALUES_EQUAL(counters, p.EnableMetricsLevel ? referenceCounters : EMPTY_COUNTERS);
199219
}
200220

201221
{
@@ -240,33 +260,38 @@ void PartitionLevelCounters(bool featureFlagEnabled, bool firstClassCitizen, TSt
240260
UNIT_ASSERT_C(result->Record.GetPartitionResponse().HasCmdReadResult(), result->Record.GetPartitionResponse().DebugString());
241261
}
242262

243-
TString counters = getCountersHtml("topics_per_partition");
263+
TString counters = getCountersHtml();
244264
TString referenceCounters = NResource::Find(TStringBuilder() << referenceDir << "_after_read.html");
245-
counters = zeroUnreliableValues(counters) + (featureFlagEnabled ? "\n" : "");
265+
counters = zeroUnreliableValues(counters) + (p.EnableMetricsLevel ? "\n" : "");
246266
Cerr << "XXXXX after read: " << counters << "\n";
247-
UNIT_ASSERT_VALUES_EQUAL(counters, featureFlagEnabled ? referenceCounters : EMPTY_COUNTERS);
267+
UNIT_ASSERT_VALUES_EQUAL(counters, p.EnableMetricsLevel ? referenceCounters : EMPTY_COUNTERS);
248268
}
249269

250270
{
251271
// Disable per partition counters, the counters should be empty.
252272

253-
PQTabletPrepare({ .metricsLevel = METRICS_LEVEL_OBJECT }, {}, tc);
273+
parameters.metricsLevel = METRICS_LEVEL_OBJECT;
274+
PQTabletPrepare(parameters, {}, tc);
254275
TString counters = getCountersHtml();
255276
TString referenceCounters = NResource::Find(TStringBuilder() << referenceDir << "_turned_off.html");
256-
counters = zeroUnreliableValues(counters) + (featureFlagEnabled ? "\n" : "");
277+
counters = zeroUnreliableValues(counters) + (p.EnableMetricsLevel ? "\n" : "");
257278
Cerr << "XXXXX after read counters disabled: " << counters << "\n";
258-
UNIT_ASSERT_VALUES_EQUAL(counters, featureFlagEnabled ? referenceCounters : EMPTY_COUNTERS);
279+
UNIT_ASSERT_VALUES_EQUAL(counters, p.EnableMetricsLevel ? referenceCounters : EMPTY_COUNTERS);
259280
}
260281
}
261282

262-
Y_UNIT_TEST(PartitionLevelCounters_Federation) {
263-
PartitionLevelCounters(false, false, "federation");
264-
PartitionLevelCounters(true, false, "federation");
265-
}
266-
267-
Y_UNIT_TEST(PartitionLevelCounters_FirstClassCitizen) {
268-
PartitionLevelCounters(false, true, "first_class_citizen");
269-
PartitionLevelCounters(true, true, "first_class_citizen");
283+
Y_UNIT_TEST(PartitionLevelMetrics) {
284+
for (bool enableMetricsLevel : {false, true}) {
285+
for (bool firstClassCitizen : {false, true}) {
286+
for (std::optional<TString> monitoringProjectId : TVector<std::optional<TString>>{std::nullopt, "", "first-monitoring-project-id"}) {
287+
PartitionLevelMetrics({
288+
.EnableMetricsLevel = enableMetricsLevel,
289+
.FirstClassCitizen = firstClassCitizen,
290+
.MonitoringProjectId = monitoringProjectId,
291+
});
292+
}
293+
}
294+
}
270295
}
271296

272297
Y_UNIT_TEST(PartitionWriteQuota) {

‎ydb/core/persqueue/ut/resources/partition_counters/federation_with_monitoring_project_id/after_read.html‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
<pre>
2+
monitoring_project_id=first-monitoring-project-id:
3+
4+
Account=asdfgs:
5+
6+
TopicPath=asdfgs/topic:
7+
8+
OriginDC=Dc1:
9+
10+
Partition=0:
11+
sensor=BytesWrittenPerPartition: 540
12+
sensor=MessagesWrittenPerPartition: 30
13+
sensor=SourceIdCount: 3
14+
sensor=TimeSinceLastWriteMs: 0
15+
sensor=WriteTimeLagMsByLastWrite: 0
16+
17+
ConsumerPath=shared/user:
18+
sensor=BytesReadPerPartition: 0
19+
sensor=MessageLagByCommittedPerPartition: 30
20+
sensor=MessageLagByLastReadPerPartition: 29
21+
sensor=MessagesReadPerPartition: 0
22+
sensor=ReadTimeLagMsPerPartition: 0
23+
sensor=TimeSinceLastReadMsPerPartition: 0
24+
sensor=WriteTimeLagMsByCommittedPerPartition: 0
25+
sensor=WriteTimeLagMsByLastReadPerPartition: 0
26+
27+
ConsumerPath=shared/user1:
28+
sensor=BytesReadPerPartition: 106
29+
sensor=MessageLagByCommittedPerPartition: 0
30+
sensor=MessageLagByLastReadPerPartition: 0
31+
sensor=MessagesReadPerPartition: 2
32+
sensor=ReadTimeLagMsPerPartition: 0
33+
sensor=TimeSinceLastReadMsPerPartition: 0
34+
sensor=WriteTimeLagMsByCommittedPerPartition: 0
35+
sensor=WriteTimeLagMsByLastReadPerPartition: 0
36+
37+
Partition=1:
38+
sensor=BytesWrittenPerPartition: 360
39+
sensor=MessagesWrittenPerPartition: 20
40+
sensor=SourceIdCount: 2
41+
sensor=TimeSinceLastWriteMs: 0
42+
sensor=WriteTimeLagMsByLastWrite: 0
43+
44+
ConsumerPath=shared/user:
45+
sensor=BytesReadPerPartition: 0
46+
sensor=MessageLagByCommittedPerPartition: 20
47+
sensor=MessageLagByLastReadPerPartition: 19
48+
sensor=MessagesReadPerPartition: 0
49+
sensor=ReadTimeLagMsPerPartition: 0
50+
sensor=TimeSinceLastReadMsPerPartition: 0
51+
sensor=WriteTimeLagMsByCommittedPerPartition: 0
52+
sensor=WriteTimeLagMsByLastReadPerPartition: 0
53+
54+
ConsumerPath=shared/user1:
55+
sensor=BytesReadPerPartition: 693
56+
sensor=MessageLagByCommittedPerPartition: 0
57+
sensor=MessageLagByLastReadPerPartition: 0
58+
sensor=MessagesReadPerPartition: 17
59+
sensor=ReadTimeLagMsPerPartition: 0
60+
sensor=TimeSinceLastReadMsPerPartition: 0
61+
sensor=WriteTimeLagMsByCommittedPerPartition: 0
62+
sensor=WriteTimeLagMsByLastReadPerPartition: 0
63+
</pre>

‎ydb/core/persqueue/ut/resources/partition_counters/federation_with_monitoring_project_id/after_write.html‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<pre>
2+
monitoring_project_id=first-monitoring-project-id:
3+
4+
Account=asdfgs:
5+
6+
TopicPath=asdfgs/topic:
7+
8+
OriginDC=Dc1:
9+
10+
Partition=0:
11+
sensor=BytesWrittenPerPartition: 540
12+
sensor=MessagesWrittenPerPartition: 30
13+
sensor=SourceIdCount: 3
14+
sensor=TimeSinceLastWriteMs: 0
15+
sensor=WriteTimeLagMsByLastWrite: 0
16+
17+
ConsumerPath=shared/user:
18+
sensor=BytesReadPerPartition: 0
19+
sensor=MessageLagByCommittedPerPartition: 30
20+
sensor=MessageLagByLastReadPerPartition: 29
21+
sensor=MessagesReadPerPartition: 0
22+
sensor=ReadTimeLagMsPerPartition: 0
23+
sensor=TimeSinceLastReadMsPerPartition: 0
24+
sensor=WriteTimeLagMsByCommittedPerPartition: 0
25+
sensor=WriteTimeLagMsByLastReadPerPartition: 0
26+
27+
Partition=1:
28+
sensor=BytesWrittenPerPartition: 360
29+
sensor=MessagesWrittenPerPartition: 20
30+
sensor=SourceIdCount: 2
31+
sensor=TimeSinceLastWriteMs: 0
32+
sensor=WriteTimeLagMsByLastWrite: 0
33+
34+
ConsumerPath=shared/user:
35+
sensor=BytesReadPerPartition: 0
36+
sensor=MessageLagByCommittedPerPartition: 20
37+
sensor=MessageLagByLastReadPerPartition: 19
38+
sensor=MessagesReadPerPartition: 0
39+
sensor=ReadTimeLagMsPerPartition: 0
40+
sensor=TimeSinceLastReadMsPerPartition: 0
41+
sensor=WriteTimeLagMsByCommittedPerPartition: 0
42+
sensor=WriteTimeLagMsByLastReadPerPartition: 0
43+
</pre>

‎ydb/core/persqueue/ut/resources/partition_counters/federation_with_monitoring_project_id/turned_off.html‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
<pre>
2+
monitoring_project_id=first-monitoring-project-id:
3+
4+
Account=asdfgs:
5+
6+
TopicPath=asdfgs/topic:
7+
8+
OriginDC=Dc1:
9+
10+
Partition=0:
11+
12+
ConsumerPath=shared/user:
13+
14+
ConsumerPath=shared/user1:
15+
16+
Partition=1:
17+
18+
ConsumerPath=shared/user:
19+
20+
ConsumerPath=shared/user1:
21+
</pre>

‎ydb/core/persqueue/ut/resources/partition_counters/first_class_citizen_with_monitoring_project_id/after_read.html‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
<pre>
2+
monitoring_project_id=first-monitoring-project-id:
3+
4+
database=/Root/PQ:
5+
6+
cloud_id=somecloud:
7+
8+
folder_id=somefolder:
9+
10+
database_id=PQ:
11+
12+
topic=topic:
13+
14+
partition_id=0:
15+
name=topic.partition.producers_count: 3
16+
name=topic.partition.write.bytes: 540
17+
name=topic.partition.write.idle_milliseconds: 0
18+
name=topic.partition.write.lag_milliseconds: 0
19+
name=topic.partition.write.messages: 30
20+
21+
consumer=user:
22+
name=topic.partition.committed_lag_messages: 30
23+
name=topic.partition.committed_read_lag_milliseconds: 0
24+
name=topic.partition.read.bytes: 0
25+
name=topic.partition.read.idle_milliseconds: 0
26+
name=topic.partition.read.lag_messages: 29
27+
name=topic.partition.read.lag_milliseconds: 0
28+
name=topic.partition.read.messages: 0
29+
name=topic.partition.write.lag_milliseconds: 0
30+
31+
consumer=user1:
32+
name=topic.partition.committed_lag_messages: 0
33+
name=topic.partition.committed_read_lag_milliseconds: 0
34+
name=topic.partition.read.bytes: 106
35+
name=topic.partition.read.idle_milliseconds: 0
36+
name=topic.partition.read.lag_messages: 0
37+
name=topic.partition.read.lag_milliseconds: 0
38+
name=topic.partition.read.messages: 2
39+
name=topic.partition.write.lag_milliseconds: 0
40+
41+
partition_id=1:
42+
name=topic.partition.producers_count: 2
43+
name=topic.partition.write.bytes: 360
44+
name=topic.partition.write.idle_milliseconds: 0
45+
name=topic.partition.write.lag_milliseconds: 0
46+
name=topic.partition.write.messages: 20
47+
48+
consumer=user:
49+
name=topic.partition.committed_lag_messages: 20
50+
name=topic.partition.committed_read_lag_milliseconds: 0
51+
name=topic.partition.read.bytes: 0
52+
name=topic.partition.read.idle_milliseconds: 0
53+
name=topic.partition.read.lag_messages: 19
54+
name=topic.partition.read.lag_milliseconds: 0
55+
name=topic.partition.read.messages: 0
56+
name=topic.partition.write.lag_milliseconds: 0
57+
58+
consumer=user1:
59+
name=topic.partition.committed_lag_messages: 0
60+
name=topic.partition.committed_read_lag_milliseconds: 0
61+
name=topic.partition.read.bytes: 693
62+
name=topic.partition.read.idle_milliseconds: 0
63+
name=topic.partition.read.lag_messages: 0
64+
name=topic.partition.read.lag_milliseconds: 0
65+
name=topic.partition.read.messages: 17
66+
name=topic.partition.write.lag_milliseconds: 0
67+
</pre>

‎ydb/core/persqueue/ut/resources/partition_counters/first_class_citizen_with_monitoring_project_id/after_write.html‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
<pre>
2+
monitoring_project_id=first-monitoring-project-id:
3+
4+
database=/Root/PQ:
5+
6+
cloud_id=somecloud:
7+
8+
folder_id=somefolder:
9+
10+
database_id=PQ:
11+
12+
topic=topic:
13+
14+
partition_id=0:
15+
name=topic.partition.producers_count: 3
16+
name=topic.partition.write.bytes: 540
17+
name=topic.partition.write.idle_milliseconds: 0
18+
name=topic.partition.write.lag_milliseconds: 0
19+
name=topic.partition.write.messages: 30
20+
21+
consumer=user:
22+
name=topic.partition.committed_lag_messages: 30
23+
name=topic.partition.committed_read_lag_milliseconds: 0
24+
name=topic.partition.read.bytes: 0
25+
name=topic.partition.read.idle_milliseconds: 0
26+
name=topic.partition.read.lag_messages: 29
27+
name=topic.partition.read.lag_milliseconds: 0
28+
name=topic.partition.read.messages: 0
29+
name=topic.partition.write.lag_milliseconds: 0
30+
31+
partition_id=1:
32+
name=topic.partition.producers_count: 2
33+
name=topic.partition.write.bytes: 360
34+
name=topic.partition.write.idle_milliseconds: 0
35+
name=topic.partition.write.lag_milliseconds: 0
36+
name=topic.partition.write.messages: 20
37+
38+
consumer=user:
39+
name=topic.partition.committed_lag_messages: 20
40+
name=topic.partition.committed_read_lag_milliseconds: 0
41+
name=topic.partition.read.bytes: 0
42+
name=topic.partition.read.idle_milliseconds: 0
43+
name=topic.partition.read.lag_messages: 19
44+
name=topic.partition.read.lag_milliseconds: 0
45+
name=topic.partition.read.messages: 0
46+
name=topic.partition.write.lag_milliseconds: 0
47+
</pre>

‎ydb/core/persqueue/ut/resources/partition_counters/first_class_citizen_with_monitoring_project_id/turned_off.html‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<pre>
2+
monitoring_project_id=first-monitoring-project-id:
3+
4+
database=/Root/PQ:
5+
6+
cloud_id=somecloud:
7+
8+
folder_id=somefolder:
9+
10+
database_id=PQ:
11+
12+
topic=topic:
13+
14+
partition_id=0:
15+
16+
consumer=user:
17+
18+
consumer=user1:
19+
20+
partition_id=1:
21+
22+
consumer=user:
23+
24+
consumer=user1:
25+
</pre>

0 commit comments

Comments
 (0)