Rename materialized index options

ydb/apps/ydb/CHANGELOG.md

@@ -1,3 +1,5 @@
+* Added the `--include-index-data` option to the `ydb export s3` command, enabling index data export.
+* Added the `--index-population-mode` option to the `ydb import s3` command, allowing selection of the index population mode (e.g. build or import).
 
 ## 2.28.0 ##
 
@@ -12,8 +14,6 @@
 * The `ydb workload vector` now supports the `import files` subcommand to populate the table from CSV or parquet files.
 * The `ydb workload vector` now supports the `import generate` subcommand to populate the table with random data.
 * Named expression-containing view restoration and restoration of views that access secondary indexes have been fixed.
-* Added the `--materialize-indexes` option to the `ydb export s3` command, enabling index materialization during export.
-* Added the `--index-filling-mode` option to the `ydb import s3` command, allowing selection of the index filling mode (e.g. build or import a materialized index).
 * Added new options for shared consumers only to the `ydb topic consumer add` command to configure DLQ and max attempts policy
 
 ## 2.27.0 ##

ydb/core/tx/schemeshard/schemeshard__init.cpp

@@ -4458,7 +4458,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
                     exportInfo->EndTime = TInstant::Seconds(rowset.GetValueOrDefault<Schema::Exports::EndTime>());
                     exportInfo->EnableChecksums = rowset.GetValueOrDefault<Schema::Exports::EnableChecksums>(false);
                     exportInfo->EnablePermissions = rowset.GetValueOrDefault<Schema::Exports::EnablePermissions>(false);
-                    exportInfo->MaterializeIndexes = rowset.GetValueOrDefault<Schema::Exports::MaterializeIndexes>(false);
+                    exportInfo->IncludeIndexData = rowset.GetValueOrDefault<Schema::Exports::IncludeIndexData>(false);
 
                     if (rowset.HaveValue<Schema::Exports::ExportMetadata>()) {
                         exportInfo->ExportMetadata = rowset.GetValue<Schema::Exports::ExportMetadata>();

ydb/core/tx/schemeshard/schemeshard_export.cpp

@@ -157,7 +157,7 @@ void TSchemeShard::PersistCreateExport(NIceDb::TNiceDb& db, const TExportInfo& e
         NIceDb::TUpdate<Schema::Exports::Items>(exportInfo.Items.size()),
         NIceDb::TUpdate<Schema::Exports::EnableChecksums>(exportInfo.EnableChecksums),
         NIceDb::TUpdate<Schema::Exports::EnablePermissions>(exportInfo.EnablePermissions),
-        NIceDb::TUpdate<Schema::Exports::MaterializeIndexes>(exportInfo.MaterializeIndexes),
+        NIceDb::TUpdate<Schema::Exports::IncludeIndexData>(exportInfo.IncludeIndexData),
         NIceDb::TUpdate<Schema::Exports::PeerName>(exportInfo.PeerName),
         NIceDb::TUpdate<Schema::Exports::SanitizedToken>(exportInfo.SanitizedToken)
     );

ydb/core/tx/schemeshard/schemeshard_export__create.cpp

@@ -37,8 +37,8 @@ bool IsPathTypeTransferrable(const NKikimr::NSchemeShard::TExportInfo::TItem& it
 }
 
 template <typename T>
-concept HasMaterializeIndexes = requires(const T& t) {
-    { t.materialize_indexes() } -> std::same_as<bool>;
+concept HasIncludeIndexData = requires(const T& t) {
+    { t.include_index_data() } -> std::same_as<bool>;
 };
 
 }
@@ -123,8 +123,8 @@ struct TSchemeShard::TExport::TTxCreate: public TSchemeShard::TXxport::TTxBase {
 
         auto processExportSettings = [&]<typename TSettings>(const TSettings& settings, TExportInfo::EKind kind, bool enableFeatureFlags) -> bool {
             exportInfo = new TExportInfo(id, uid, kind, settings, domainPath.Base()->PathId, request.GetPeerName());
-            if constexpr (HasMaterializeIndexes<TSettings>) {
-                exportInfo->MaterializeIndexes = settings.materialize_indexes();
+            if constexpr (HasIncludeIndexData<TSettings>) {
+                exportInfo->IncludeIndexData = settings.include_index_data();
             }
             if (enableFeatureFlags) {
                 exportInfo->EnableChecksums = AppData()->FeatureFlags.GetEnableChecksumsExport();
@@ -154,7 +154,7 @@ struct TSchemeShard::TExport::TTxCreate: public TSchemeShard::TXxport::TTxBase {
                 if (!settings.scheme()) {
                     settings.set_scheme(Ydb::Export::ExportToS3Settings::HTTPS);
                 }
-                if (settings.materialize_indexes() && !AppData()->FeatureFlags.GetEnableIndexMaterialization()) {
+                if (settings.include_index_data() && !AppData()->FeatureFlags.GetEnableIndexMaterialization()) {
                     return Reply(
                         std::move(response),
                         Ydb::StatusIds::PRECONDITION_FAILED,
@@ -271,7 +271,7 @@ struct TSchemeShard::TExport::TTxCreate: public TSchemeShard::TXxport::TTxBase {
             exportInfo.Items.emplace_back(item.source_path(), path.Base()->PathId, path->PathType);
             exportInfo.PendingItems.push_back(exportInfo.Items.size() - 1);
 
-            if (exportInfo.MaterializeIndexes && path.Base()->IsTable()) {
+            if (exportInfo.IncludeIndexData && path.Base()->IsTable()) {
                 for (const auto& [childName, childPathId] : path.Base()->GetChildren()) {
                     TVector<TString> childParts;
                     childParts.push_back(childName);

ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp

@@ -61,7 +61,7 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CopyTablesPropose(
         auto& desc = *copyTables.Add();
         desc.SetSrcPath(item.SourcePathName);
         desc.SetDstPath(ExportItemPathName(ss, exportInfo, itemIdx));
-        desc.SetOmitIndexes(!exportInfo.MaterializeIndexes);
+        desc.SetOmitIndexes(!exportInfo.IncludeIndexData);
         desc.SetOmitFollowers(true);
         desc.SetIsBackup(true);
     }

ydb/core/tx/schemeshard/schemeshard_import.cpp

@@ -454,12 +454,12 @@ bool NeedToBuildIndexes(const TImportInfo& importInfo, ui32 itemIdx) {
     Y_ABORT_UNLESS(itemIdx < importInfo.Items.size());
     auto& item = importInfo.Items.at(itemIdx);
 
-    switch (importInfo.GetS3Settings().index_filling_mode()) {
-        case Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_BUILD:
+    switch (importInfo.GetS3Settings().index_population_mode()) {
+        case Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_BUILD:
             return true;
-        case Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_AUTO:
+        case Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_AUTO:
             return item.ChildItems.empty();
-        case Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_IMPORT:
+        case Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_IMPORT:
             return false;
         default:
             return true;

ydb/core/tx/schemeshard/schemeshard_import__create.cpp

@@ -193,9 +193,9 @@ struct TSchemeShard::TImport::TTxCreate: public TSchemeShard::TXxport::TTxBase {
                 }
 
                 if (!AppData()->FeatureFlags.GetEnableIndexMaterialization()) {
-                    switch (settings.index_filling_mode()) {
-                    case Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_IMPORT:
-                    case Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_AUTO:
+                    switch (settings.index_population_mode()) {
+                    case Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_IMPORT:
+                    case Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_AUTO:
                         return Reply(
                             std::move(response),
                             Ydb::StatusIds::PRECONDITION_FAILED,

ydb/core/tx/schemeshard/schemeshard_import_getters.cpp

@@ -409,7 +409,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
             << ": self# " << SelfId()
             << ", result# " << result);
 
-        const bool canSkip = IndexFillingMode != Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_IMPORT;
+        const bool canSkip = IndexPopulationMode != Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_IMPORT;
         if (canSkip && NoObjectFound(result.GetError().GetErrorType())) {
             Y_ABORT_UNLESS(ItemIdx < ImportInfo->Items.size());
             auto& item = ImportInfo->Items.at(ItemIdx);
@@ -798,10 +798,10 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
         Download(PermissionsKey);
     }
 
-    static bool NeedToCheckMaterializedIndexes(Ydb::Import::ImportFromS3Settings::IndexFillingMode mode) {
+    static bool NeedToCheckMaterializedIndexes(Ydb::Import::ImportFromS3Settings::IndexPopulationMode mode) {
         switch (mode) {
-        case Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_IMPORT:
-        case Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_AUTO:
+        case Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_IMPORT:
+        case Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_AUTO:
             return true;
         default:
             return false;
@@ -915,7 +915,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
 
     void StartCheckingMaterializedIndexes() {
         ResetRetries();
-        if (NeedToCheckMaterializedIndexes(IndexFillingMode)) {
+        if (NeedToCheckMaterializedIndexes(IndexPopulationMode)) {
             CheckMaterializedIndexes();
         } else {
             StartDownloadingChangefeeds();
@@ -936,7 +936,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
         , MetadataKey(MetadataKeyFromSettings(*ImportInfo, itemIdx))
         , SchemeKey(SchemeKeyFromSettings(*ImportInfo, itemIdx, "scheme.pb"))
         , PermissionsKey(PermissionsKeyFromSettings(*ImportInfo, itemIdx))
-        , IndexFillingMode(ImportInfo->GetS3Settings().index_filling_mode())
+        , IndexPopulationMode(ImportInfo->GetS3Settings().index_population_mode())
         , NeedDownloadPermissions(!ImportInfo->GetNoAcl())
         , NeedValidateChecksums(!ImportInfo->GetSkipChecksumValidation())
     {
@@ -1023,7 +1023,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
 
     TVector<NBackup::TIndexMetadata> IndexImplTablePrefixes;
     ui64 IndexCheckedMaterializedIndexImplTable = 0;
-    Ydb::Import::ImportFromS3Settings::IndexFillingMode IndexFillingMode;
+    Ydb::Import::ImportFromS3Settings::IndexPopulationMode IndexPopulationMode;
 
     bool NeedDownloadPermissions = true;
     bool NeedValidateChecksums = true;

ydb/core/tx/schemeshard/schemeshard_info_types.h

@@ -3029,7 +3029,7 @@ struct TExportInfo: public TSimpleRefCount<TExportInfo> {
 
     bool EnableChecksums = false;
     bool EnablePermissions = false;
-    bool MaterializeIndexes = false;
+    bool IncludeIndexData = false;
 
     NKikimrSchemeOp::TExportMetadata ExportMetadata;
     TActorId ExportMetadataUploader;

ydb/core/tx/schemeshard/schemeshard_schema.h

@@ -1247,7 +1247,7 @@ struct Schema : NIceDb::Schema {
 
         struct EnableChecksums : Column<17, NScheme::NTypeIds::Bool> {};
         struct EnablePermissions : Column<18, NScheme::NTypeIds::Bool> {};
-        struct MaterializeIndexes : Column<21, NScheme::NTypeIds::Bool> {};
+        struct IncludeIndexData : Column<21, NScheme::NTypeIds::Bool> {};
 
         struct ExportMetadata : Column<19, NScheme::NTypeIds::String> { using Type = NKikimrSchemeOp::TExportMetadata; };
 
@@ -1273,7 +1273,7 @@ struct Schema : NIceDb::Schema {
             EnablePermissions,
             ExportMetadata,
             SanitizedToken,
-            MaterializeIndexes
+            IncludeIndexData
         >;
     };
 

ydb/core/tx/schemeshard/ut_export/ut_export.cpp

@@ -3114,7 +3114,7 @@ attributes {
             ExportToS3Settings {
               endpoint: "localhost:%d"
               scheme: HTTP
-              materialize_indexes: true
+              include_index_data: true
               items {
                 source_path: "/MyRoot/Table"
                 destination_prefix: ""

ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp

@@ -757,7 +757,7 @@ Y_UNIT_TEST_SUITE(TExportToS3WithRebootsTests) {
             ExportToS3Settings {
               endpoint: "localhost:%d"
               scheme: HTTP
-              materialize_indexes: true
+              include_index_data: true
               items {
                 source_path: "/MyRoot/Table"
                 destination_prefix: ""

ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp

@@ -6315,7 +6315,7 @@ Y_UNIT_TEST_SUITE(TImportTests) {
         UNIT_ASSERT_EQUAL(issues.begin()->message(), "Unsupported scheme object type");
     }
 
-    void MaterializedIndex(Ydb::Import::ImportFromS3Settings::IndexFillingMode mode) {
+    void MaterializedIndex(Ydb::Import::ImportFromS3Settings::IndexPopulationMode mode) {
         TTestBasicRuntime runtime;
         TTestEnv env(runtime, TTestEnvOptions().EnableIndexMaterialization(true));
 
@@ -6353,13 +6353,13 @@ Y_UNIT_TEST_SUITE(TImportTests) {
             ImportFromS3Settings {
               endpoint: "localhost:%%d"
               scheme: HTTP
-              index_filling_mode: %s
+              index_population_mode: %s
               items {
                 source_prefix: "a"
                 destination_path: "/MyRoot/Table"
               }
             }
-        )", Ydb::Import::ImportFromS3Settings::IndexFillingMode_Name(mode).c_str()));
+        )", Ydb::Import::ImportFromS3Settings::IndexPopulationMode_Name(mode).c_str()));
 
         TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {
             NLs::PathExist,
@@ -6368,18 +6368,18 @@ Y_UNIT_TEST_SUITE(TImportTests) {
     }
 
     Y_UNIT_TEST(MaterializedIndexBuild) {
-        MaterializedIndex(Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_BUILD);
+        MaterializedIndex(Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_BUILD);
     }
 
     Y_UNIT_TEST(MaterializedIndexImport) {
-        MaterializedIndex(Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_IMPORT);
+        MaterializedIndex(Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_IMPORT);
     }
 
     Y_UNIT_TEST(MaterializedIndexAuto) {
-        MaterializedIndex(Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_AUTO);
+        MaterializedIndex(Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_AUTO);
     }
 
-    void MaterializedIndexAbsent(Ydb::Import::ImportFromS3Settings::IndexFillingMode mode, bool shouldFail) {
+    void MaterializedIndexAbsent(Ydb::Import::ImportFromS3Settings::IndexPopulationMode mode, bool shouldFail) {
         TTestBasicRuntime runtime;
         TTestEnv env(runtime, TTestEnvOptions().EnableIndexMaterialization(true));
 
@@ -6405,13 +6405,13 @@ Y_UNIT_TEST_SUITE(TImportTests) {
             ImportFromS3Settings {
               endpoint: "localhost:%%d"
               scheme: HTTP
-              index_filling_mode: %s
+              index_population_mode: %s
               items {
                 source_prefix: "a"
                 destination_path: "/MyRoot/Table"
               }
             }
-        )", Ydb::Import::ImportFromS3Settings::IndexFillingMode_Name(mode).c_str()), expectedStatus);
+        )", Ydb::Import::ImportFromS3Settings::IndexPopulationMode_Name(mode).c_str()), expectedStatus);
 
         if (shouldFail) {
             return;
@@ -6424,15 +6424,15 @@ Y_UNIT_TEST_SUITE(TImportTests) {
     }
 
     Y_UNIT_TEST(MaterializedIndexAbsentBuild) {
-        MaterializedIndexAbsent(Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_BUILD, false);
+        MaterializedIndexAbsent(Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_BUILD, false);
     }
 
     Y_UNIT_TEST(MaterializedIndexAbsentImport) {
-        MaterializedIndexAbsent(Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_IMPORT, true);
+        MaterializedIndexAbsent(Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_IMPORT, true);
     }
 
     Y_UNIT_TEST(MaterializedIndexAbsentAuto) {
-        MaterializedIndexAbsent(Ydb::Import::ImportFromS3Settings::INDEX_FILLING_MODE_AUTO, false);
+        MaterializedIndexAbsent(Ydb::Import::ImportFromS3Settings::INDEX_POPULATION_MODE_AUTO, false);
     }
 }
 
@@ -7089,7 +7089,7 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
                 ImportFromS3Settings {
                   endpoint: "localhost:%d"
                   scheme: HTTP
-                  index_filling_mode: INDEX_FILLING_MODE_IMPORT
+                  index_population_mode: INDEX_POPULATION_MODE_IMPORT
                   items {
                     source_prefix: "a"
                     destination_path: "/MyRoot/Table"

ydb/public/api/protos/ydb_export.proto

@@ -145,11 +145,11 @@ message ExportToS3Settings {
     // the resulting data will not be encrypted.
     EncryptionSettings encryption_settings = 15;
 
-    // Materialization of index table data.
+    // Include index data or not.
     // By default, only index metadata is uploaded and indexes are built during import — it saves space
     // and reduces export time, but it can potentially increase the import time.
-    // Indexes can be materialized, then their data will be uploaded during export and downloaded during import.
-    bool materialize_indexes = 16;
+    // Index data can be uploaded and downloaded back during import.
+    bool include_index_data = 16;
 
     // Patterns (PCRE) for paths excluded from export operation.
     // - Patterns are matched against the object path relative to the export's source_path.

ydb/public/api/protos/ydb_import.proto

@@ -62,15 +62,15 @@ message ImportFromS3Settings {
         string destination_path = 2;
     }
 
-    enum IndexFillingMode {
+    enum IndexPopulationMode {
         // If unspecified, use default - Build
-        INDEX_FILLING_MODE_UNSPECIFIED = 0;
+        INDEX_POPULATION_MODE_UNSPECIFIED = 0;
         // Build index
-        INDEX_FILLING_MODE_BUILD = 1;
+        INDEX_POPULATION_MODE_BUILD = 1;
         // Import materialized index
-        INDEX_FILLING_MODE_IMPORT = 2;
+        INDEX_POPULATION_MODE_IMPORT = 2;
         // Try to import materialized index, build otherwise
-        INDEX_FILLING_MODE_AUTO = 3;
+        INDEX_POPULATION_MODE_AUTO = 3;
     }
 
     string endpoint = 1 [(required) = true];
@@ -112,9 +112,9 @@ message ImportFromS3Settings {
     // the resulting data is considered not encrypted.
     Ydb.Export.EncryptionSettings encryption_settings = 15;
 
-    // Index filling mode.
+    // Index population mode.
     // If not specified, indexes will be built.
-    IndexFillingMode index_filling_mode = 16;
+    IndexPopulationMode index_population_mode = 16;
 
     // Patterns (PCRE) for paths excluded from import operation.
     // - Patterns are matched against the database object names stored in the backup listing.

ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp

@@ -318,14 +318,14 @@ void TCommandExportToS3::Config(TConfig& config) {
 
     {
         TStringBuilder help;
-        help << "Materialize index table data or not";
+        help << "Include index data or not";
         if (config.HelpCommandVerbosiltyLevel >= 2) {
             help << Endl << "    By default, only index metadata is uploaded and indexes are built during import — it"
                  << Endl << "    saves space and reduces export time, but it can potentially increase the import time."
-                 << Endl << "    Indexes can be materialized, then their data will be uploaded during export and downloaded during import.";
+                 << Endl << "    Index data can be uploaded and downloaded back during import.";
         }
-        config.Opts->AddLongOption("materialize-indexes", help)
-            .RequiredArgument("BOOL").StoreResult<bool>(&MaterializeIndexes).DefaultValue("false");
+        config.Opts->AddLongOption("include-index-data", help)
+            .RequiredArgument("BOOL").StoreResult<bool>(&IncludeIndexData).DefaultValue("false");
     }
 
     {
@@ -421,7 +421,7 @@ int TCommandExportToS3::Run(TConfig& config) {
     settings.AccessKey(AwsAccessKey);
     settings.SecretKey(AwsSecretKey);
     settings.UseVirtualAddressing(UseVirtualAddressing);
-    settings.MaterializeIndexes(MaterializeIndexes);
+    settings.IncludeIndexData(IncludeIndexData);
 
     for (const auto& item : Items) {
         settings.AppendItem({item.Source, item.Destination});

ydb/public/lib/ydb_cli/commands/ydb_service_export.h

@@ -74,7 +74,7 @@ class TCommandExportToS3 : public TYdbOperationCommand,
     ui32 NumberOfRetries = 10;
     TString Compression;
     bool UseVirtualAddressing = true;
-    bool MaterializeIndexes = false;
+    bool IncludeIndexData = false;
     TString CommonSourcePath;
     TString CommonDestinationPrefix;