Skip to content

Handle InferPDiskSlotCount settings in nodewarden #31207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rosik wants to merge 2 commits into
base: main
Choose a base branch
from
+325 −190
Open

Handle InferPDiskSlotCount settings in nodewarden #31207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
39aec02
Handle InferPDiskSlotCount settings in nodewarden
rosik Dec 15, 2025
5db8379
Fix tests
rosik Dec 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions ydb/apps/dstool/lib/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,8 +290,6 @@ def get_vslot_extended_id(vslot):
def get_pdisk_inferred_settings(pdisk):
if (pdisk.PDiskMetrics.HasField('SlotCount')):
return pdisk.PDiskMetrics.SlotCount, pdisk.PDiskMetrics.SlotSizeInUnits
elif (pdisk.InferPDiskSlotCountFromUnitSize != 0):
return 0, 0
else:
return pdisk.ExpectedSlotCount, pdisk.PDiskConfig.SlotSizeInUnits

Expand Down
63 changes: 53 additions & 10 deletions ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include <ydb/core/blobstorage/vdisk/common/vdisk_events.h>
#include <ydb/core/mind/bscontroller/bsc.h>
#include <ydb/core/util/actorsys_test/testactorsys.h>
#include <ydb/core/cms/console/console.h>

#include <ydb/library/pdisk_io/sector_map.h>
#include <ydb/core/testlib/actors/block_events.h>
Expand Down Expand Up @@ -268,8 +269,7 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) {
}

ui32 CreatePDisk(TTestActorRuntime &runtime, ui32 nodeIdx, TString path, ui64 guid, ui32 pdiskId, ui64 pDiskCategory,
const NKikimrBlobStorage::TPDiskConfig* pdiskConfig = nullptr, ui64 inferPDiskSlotCountFromUnitSize = 0,
ui32 inferPDiskSlotCountMax = 0, TActorId nodeWarden = {}) {
const NKikimrBlobStorage::TPDiskConfig* pdiskConfig = nullptr, TActorId nodeWarden = {}) {
VERBOSE_COUT(" Creating pdisk");

ui32 nodeId = runtime.GetNodeId(nodeIdx);
Expand All @@ -285,10 +285,6 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) {
if (pdiskConfig) {
pdisk->MutablePDiskConfig()->CopyFrom(*pdiskConfig);
}
if (inferPDiskSlotCountFromUnitSize != 0) {
pdisk->SetInferPDiskSlotCountFromUnitSize(inferPDiskSlotCountFromUnitSize);
pdisk->SetInferPDiskSlotCountMax(inferPDiskSlotCountMax);
}

if (!nodeWarden) {
nodeWarden = MakeBlobStorageNodeWardenID(nodeId);
Expand Down Expand Up @@ -1091,47 +1087,94 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) {
return realNodeWarden;
}

void UpdateInferPDiskSlotCountSettings(TTestBasicRuntime& runtime, TActorId realNodeWarden,
ui64 unitSize, ui32 maxSlots, bool preferInferredSettings) {
auto request = std::make_unique<NConsole::TEvConsole::TEvConfigNotificationRequest>();
auto& record = request->Record;
auto* blobStorageConfig = record.MutableConfig()->MutableBlobStorageConfig();
auto* inferSettings = blobStorageConfig->MutableInferPDiskSlotCountSettings();

inferSettings->MutableRot()->SetUnitSize(unitSize);
inferSettings->MutableRot()->SetMaxSlots(maxSlots);
inferSettings->SetPreferInferredSettingsOverExplicit(preferInferredSettings);

TActorId sender = runtime.AllocateEdgeActor();
runtime.Send(new IEventHandle(realNodeWarden, sender, request.release()));

auto response = runtime.GrabEdgeEventRethrow<NConsole::TEvConsole::TEvConfigNotificationResponse>(sender);
Y_UNUSED(response);
}

CUSTOM_UNIT_TEST(TestInferPDiskSlotCountExplicitConfig) {
TTestBasicRuntime runtime(1, false);
TActorId realNodeWarden = SetupNodeWardenOnly(runtime);
UpdateInferPDiskSlotCountSettings(runtime, realNodeWarden,
100_GB, 16, false);

const ui32 nodeId = runtime.GetNodeId(0);
const ui32 pdiskId = 1001;
const TString pdiskPath = "SectorMap:TestInferPDiskSlotCountExplicitConfig:2400";
const ui64 inferPDiskSlotCountFromUnitSize = 100_GB; // 12 * 2u slots

TActorId fakeNodeWarden = runtime.AllocateEdgeActor();
runtime.RegisterService(MakeBlobStorageNodeWardenID(nodeId), fakeNodeWarden);
TActorId fakeWhiteboard = runtime.AllocateEdgeActor();
runtime.RegisterService(NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId), fakeWhiteboard);

VERBOSE_COUT("- Test case 1 - create PDisk");
NKikimrBlobStorage::TPDiskConfig pdiskConfig;
pdiskConfig.SetExpectedSlotCount(13);
CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0,
&pdiskConfig, inferPDiskSlotCountFromUnitSize, 16, realNodeWarden);
&pdiskConfig, realNodeWarden);
CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden,
pdiskId, 13, 0u);

VERBOSE_COUT("- Test case 2 - enable PreferInferredSettingsOverExplicit");
UpdateInferPDiskSlotCountSettings(runtime, realNodeWarden,
100_GB, 16, true);
CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden,
pdiskId, 12, 2u);

VERBOSE_COUT("- Test case 3 - update InferPDiskSlotCountSettings");
UpdateInferPDiskSlotCountSettings(runtime, realNodeWarden,
50_GB, 9, true);
CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden,
pdiskId, 6, 8u);
}

CUSTOM_UNIT_TEST(TestInferPDiskSlotCountWithRealNodeWarden) {
TTestBasicRuntime runtime(1, false);
TActorId realNodeWarden = SetupNodeWardenOnly(runtime);
UpdateInferPDiskSlotCountSettings(runtime, realNodeWarden,
100_GB, 16, false);

const ui32 nodeId = runtime.GetNodeId(0);
const ui32 pdiskId = 1002;
const TString pdiskPath = "SectorMap:TestInferPDiskSlotCount:2400";
const ui64 inferPDiskSlotCountFromUnitSize = 100_GB; // 12 * 2u slots

TActorId fakeNodeWarden = runtime.AllocateEdgeActor();
runtime.RegisterService(MakeBlobStorageNodeWardenID(nodeId), fakeNodeWarden);
TActorId fakeWhiteboard = runtime.AllocateEdgeActor();
runtime.RegisterService(NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId), fakeWhiteboard);

VERBOSE_COUT("- Test case 1 - create PDisk");
NKikimrBlobStorage::TPDiskConfig pdiskConfig;
CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0,
&pdiskConfig, inferPDiskSlotCountFromUnitSize, 16, realNodeWarden);
&pdiskConfig, realNodeWarden);
CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden,
pdiskId, 12, 2u);

VERBOSE_COUT("- Test case 2 - update InferPDiskSlotCountSettings");
UpdateInferPDiskSlotCountSettings(runtime, realNodeWarden,
100_GB, 24, false);
CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden,
pdiskId, 24, 1u);

VERBOSE_COUT("- Test case 3 - set ExpectedSlotCount explicitly");
pdiskConfig.SetExpectedSlotCount(17);
CreatePDisk(runtime, 0, pdiskPath, 0, pdiskId, 0,
&pdiskConfig, realNodeWarden);
CheckInferredPDiskSettings(runtime, fakeWhiteboard, fakeNodeWarden,
pdiskId, 17, 0u);
}

void ChangeGroupSizeInUnits(TTestBasicRuntime& runtime, TString poolName, ui32 groupId, ui32 groupSizeInUnits) {
Expand Down
54 changes: 54 additions & 0 deletions ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,10 @@ STATEFN(TNodeWarden::StateOnline) {

hFunc(TEvNodeWardenNotifySyncerFinished, Handle);

hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, Handle);
hFunc(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse, Handle);
hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, Handle);

default:
EnqueuePendingMessage(ev);
break;
Expand Down Expand Up @@ -338,6 +342,10 @@ void TNodeWarden::StartRequestReportingThrottler() {

void TNodeWarden::PassAway() {
STLOG(PRI_DEBUG, BS_NODE, NW25, "PassAway");

Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
new NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest(SelfId()));

NTabletPipe::CloseClient(SelfId(), PipeClientId);
StopInvalidGroupProxy();
TActivationContext::Send(new IEventHandle(TEvents::TSystem::Poison, 0, DsProxyNodeMonActor, {}, nullptr, 0));
Expand Down Expand Up @@ -504,6 +512,11 @@ void TNodeWarden::Bootstrap() {

YamlConfig = std::move(Cfg->YamlConfig);

InferPDiskSlotCountSettings.CopyFrom(Cfg->BlobStorageConfig.GetInferPDiskSlotCountSettings());
ui32 blobStorageConfigItem = NKikimrConsole::TConfigItem::BlobStorageConfigItem;
Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()),
new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(blobStorageConfigItem));

// Start a statically configured set
if (Cfg->BlobStorageConfig.HasServiceSet()) {
const auto& serviceSet = Cfg->BlobStorageConfig.GetServiceSet();
Expand Down Expand Up @@ -1261,6 +1274,47 @@ void TNodeWarden::Handle(TEvStatusUpdate::TPtr ev) {
}
}

void TNodeWarden::Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr /*ev*/)
{}

void TNodeWarden::Handle(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse::TPtr /*ev*/)
{}

void TNodeWarden::Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr ev) {
auto& record = ev->Get()->Record;
if (record.HasConfig() &&
record.GetConfig().HasBlobStorageConfig() &&
record.GetConfig().GetBlobStorageConfig().HasInferPDiskSlotCountSettings()) {
auto inferSettings = record.GetConfig().GetBlobStorageConfig().GetInferPDiskSlotCountSettings();
InferPDiskSlotCountSettings.CopyFrom(inferSettings);

for (auto& [key, localPDisk] : LocalPDisks) {
const TServiceSetPDisk& serviceSetPDisk = localPDisk.Record;
ui64 oldExpectedSlotCount = serviceSetPDisk.GetPDiskConfig().GetExpectedSlotCount();
ui32 oldSlotSizeInUnits = serviceSetPDisk.GetPDiskConfig().GetSlotSizeInUnits();
TIntrusivePtr<TPDiskConfig> newPDiskConfig = CreatePDiskConfig(serviceSetPDisk);
ui64 newExpectedSlotCount = newPDiskConfig->ExpectedSlotCount;
ui32 newSlotSizeInUnits = newPDiskConfig->SlotSizeInUnits;

if (newExpectedSlotCount != oldExpectedSlotCount ||
newSlotSizeInUnits != oldSlotSizeInUnits) {
STLOG(PRI_DEBUG, BS_NODE, NW112, "SendChangeExpectedSlotCount from config notification",
(PDiskId, key.PDiskId),
(ExpectedSlotCount, newExpectedSlotCount),
(SlotSizeInUnits, newSlotSizeInUnits));

const TActorId pdiskActorId = MakeBlobStoragePDiskID(LocalNodeId, key.PDiskId);
Send(pdiskActorId, new NPDisk::TEvChangeExpectedSlotCount(newExpectedSlotCount, newSlotSizeInUnits));

NKikimrBlobStorage::TPDiskConfig* pdiskConfig = localPDisk.Record.MutablePDiskConfig();
pdiskConfig->SetExpectedSlotCount(newExpectedSlotCount);
pdiskConfig->SetSlotSizeInUnits(newSlotSizeInUnits);
}
}
}
Send(ev->Sender, new NConsole::TEvConsole::TEvConfigNotificationResponse(record), 0, ev->Cookie);
}

void TNodeWarden::FillInVDiskStatus(google::protobuf::RepeatedPtrField<NKikimrBlobStorage::TVDiskStatus> *pb, bool initial) {
for (auto& [vslotId, vdisk] : LocalVDisks) {
const NKikimrBlobStorage::EVDiskStatus status = vdisk.RuntimeData
Expand Down
7 changes: 7 additions & 0 deletions ydb/core/blobstorage/nodewarden/node_warden_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#include <ydb/core/blobstorage/dsproxy/group_sessions.h>
#include <ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h>
#include <ydb/core/blobstorage/incrhuge/incrhuge.h>
#include <ydb/core/cms/console/configs_dispatcher.h>
#include <ydb/core/cms/console/console.h>
#include <ydb/core/node_whiteboard/node_whiteboard.h>
#include <ydb/core/protos/blobstorage_distributed_config.pb.h>
#include <ydb/core/util/backoff.h>
Expand Down Expand Up @@ -163,6 +165,7 @@ namespace NKikimr::NStorage {

bool EnableProxyMock = false;
NKikimrBlobStorage::TMockDevicesConfig MockDevicesConfig;
NKikimrBlobStorage::TInferPDiskSlotCountSettings InferPDiskSlotCountSettings;

struct TEvPrivate {
enum EEv {
Expand Down Expand Up @@ -655,6 +658,10 @@ namespace NKikimr::NStorage {

void Handle(NNodeWhiteboard::TEvWhiteboard::TEvBSGroupStateUpdate::TPtr ev);

void Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr ev);
void Handle(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse::TPtr ev);
void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr ev);

////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Bridge syncer operation

Expand Down
23 changes: 18 additions & 5 deletions ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,17 @@ namespace NKikimr::NStorage {
pdiskConfig->EnableSectorEncryption = !pdiskConfig->SectorMap;
}

if (pdisk.GetPDiskConfig().GetExpectedSlotCount() != 0) {
// Skip inferring PDisk SlotCount
} else if (ui64 unitSizeInBytes = pdisk.GetInferPDiskSlotCountFromUnitSize()) {
auto inferSettings = TInferPDiskSlotCountSettingsForDriveType(InferPDiskSlotCountSettings, deviceType);
bool preferInferredSettings = InferPDiskSlotCountSettings.GetPreferInferredSettingsOverExplicit();
if (!inferSettings) {
STLOG(PRI_DEBUG, BS_NODE, NW102, "Inferring PDisk slot count not configured", (Path, path),
(SlotCount, pdiskConfig->ExpectedSlotCount),
(SlotSizeInUnits, pdiskConfig->SlotSizeInUnits));
} else if (pdiskConfig->ExpectedSlotCount != 0 && !preferInferredSettings) {
STLOG(PRI_DEBUG, BS_NODE, NW102, "Skipped inferring PDisk slot count, using explicit settings", (Path, path),
(SlotCount, pdiskConfig->ExpectedSlotCount),
(SlotSizeInUnits, pdiskConfig->SlotSizeInUnits));
} else {
ui64 driveSize = 0;
TStringStream outDetails;
if (pdiskConfig->SectorMap) {
Expand All @@ -155,8 +163,13 @@ namespace NKikimr::NStorage {
STLOG(PRI_ERROR, BS_NODE, NW96, "Unable to determine drive size for inferring PDisk slot count",
(Path, path), (Details, outDetails.Str()));
} else {
ui32 maxSlots = pdisk.GetInferPDiskSlotCountMax();
InferPDiskSlotCount(pdiskConfig, driveSize, unitSizeInBytes, maxSlots);
InferPDiskSlotCount(pdiskConfig, driveSize, inferSettings.UnitSize, inferSettings.MaxSlots);
STLOG(PRI_DEBUG, BS_NODE, NW102, "Inferred PDisk slot count", (Path, path),
(SlotCount, pdiskConfig->ExpectedSlotCount),
(SlotSizeInUnits, pdiskConfig->SlotSizeInUnits),
(FromDriveSize, driveSize),
(FromUnitSize, inferSettings.UnitSize),
(FromMaxSlots, inferSettings.MaxSlots));
}
}

Expand Down
25 changes: 25 additions & 0 deletions ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -452,4 +452,29 @@ struct TPDiskConfig : public TThrRefBase {
}
};

struct TInferPDiskSlotCountSettingsForDriveType {
ui64 UnitSize = 0;
ui32 MaxSlots = 0;

TInferPDiskSlotCountSettingsForDriveType(const NKikimrBlobStorage::TInferPDiskSlotCountSettings& settings, NPDisk::EDeviceType type) {
switch (type) {
case NPDisk::DEVICE_TYPE_ROT:
UnitSize = settings.GetRot().GetUnitSize();
MaxSlots = settings.GetRot().GetMaxSlots();
break;
case NPDisk::DEVICE_TYPE_SSD:
case NPDisk::DEVICE_TYPE_NVME:
UnitSize = settings.GetSsd().GetUnitSize();
MaxSlots = settings.GetSsd().GetMaxSlots();
break;
case NPDisk::DEVICE_TYPE_UNKNOWN:
break;
}
}

explicit operator bool() const {
return UnitSize && MaxSlots;
}
};

} // NKikimr
12 changes: 8 additions & 4 deletions ydb/core/mind/bscontroller/cmds_host_config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ namespace NKikimr::NBsController {
driveInfo.SharedWithOs = drive.GetSharedWithOs();
driveInfo.ReadCentric = drive.GetReadCentric();
driveInfo.Kind = drive.GetKind();
driveInfo.InferPDiskSlotCountFromUnitSize = drive.GetInferPDiskSlotCountFromUnitSize();
driveInfo.InferPDiskSlotCountMax = drive.GetInferPDiskSlotCountMax();
// driveInfo.InferPDiskSlotCountFromUnitSize = drive.GetInferPDiskSlotCountFromUnitSize();
// driveInfo.InferPDiskSlotCountMax = drive.GetInferPDiskSlotCountMax();

if (drive.HasPDiskConfig()) {
TString config;
Expand All @@ -48,8 +48,8 @@ namespace NKikimr::NBsController {
driveInfo.SharedWithOs = false;
driveInfo.ReadCentric = false;
driveInfo.Kind = 0;
driveInfo.InferPDiskSlotCountFromUnitSize = 0;
driveInfo.InferPDiskSlotCountMax = 0;
// driveInfo.InferPDiskSlotCountFromUnitSize = 0;
// driveInfo.InferPDiskSlotCountMax = 0;
driveInfo.PDiskConfig = defaultPDiskConfig;

for (const auto& path : field) {
Expand Down Expand Up @@ -121,4 +121,8 @@ namespace NKikimr::NBsController {
hostConfigs.erase(it);
}

// void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TInferPDiskSlotCountSettings& cmd, TStatus& /*status*/) {
// Y_UNUSED(cmd);
// }

} // NKikimr::NBsController
4 changes: 2 additions & 2 deletions ydb/core/mind/bscontroller/cmds_storage_pool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -616,8 +616,8 @@ namespace NKikimr::NBsController {
x->SetDriveStatus(NKikimrBlobStorage::EDriveStatus::ACTIVE);
x->SetExpectedSlotCount(pdisk.ExpectedSlotCount);
x->SetDecommitStatus(NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE);
x->SetInferPDiskSlotCountFromUnitSize(pdisk.InferPDiskSlotCountFromUnitSize);
x->SetInferPDiskSlotCountMax(pdisk.InferPDiskSlotCountMax);
// x->SetInferPDiskSlotCountFromUnitSize(pdisk.InferPDiskSlotCountFromUnitSize);
// x->SetInferPDiskSlotCountMax(pdisk.InferPDiskSlotCountMax);
if (pdisk.PDiskMetrics) {
x->MutablePDiskMetrics()->CopyFrom(*pdisk.PDiskMetrics);
x->MutablePDiskMetrics()->ClearPDiskId();
Expand Down
Loading
Loading