From a903c03af780ba93a914e25dfb3ec73f2c00ea47 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 6 May 2026 13:54:11 -0300 Subject: [PATCH 1/4] implement export partition all --- src/Common/ErrorCodes.cpp | 4 +- src/Core/Settings.cpp | 17 ++++ src/Core/Settings.h | 1 + src/Core/SettingsChangesHistory.cpp | 1 + src/Core/SettingsEnums.cpp | 2 + src/Core/SettingsEnums.h | 8 ++ src/Storages/MergeTree/MergeTreeData.cpp | 7 +- src/Storages/StorageReplicatedMergeTree.cpp | 83 +++++++++++++++- .../test.py | 26 +++++ .../test.py | 97 +++++++++++++++++++ ..._export_merge_tree_partition_all.reference | 17 ++++ .../03609_export_merge_tree_partition_all.sh | 59 +++++++++++ 12 files changed, 317 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/03609_export_merge_tree_partition_all.reference create mode 100755 tests/queries/0_stateless/03609_export_merge_tree_partition_all.sh diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 423ec04acb49..93381aaf1a02 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -664,6 +664,8 @@ M(1003, SSH_EXCEPTION) \ M(1004, STARTUP_SCRIPTS_ERROR) \ M(1005, PENDING_MUTATIONS_NOT_ALLOWED) \ + M(1006, EXPORT_PARTITION_ALREADY_EXPORTED) \ + M(1007, PARTITION_EXPORT_FAILED) \ /* See END */ #ifdef APPLY_FOR_EXTERNAL_ERROR_CODES @@ -680,7 +682,7 @@ namespace ErrorCodes APPLY_FOR_ERROR_CODES(M) #undef M - constexpr ErrorCode END = 1005; + constexpr ErrorCode END = 1007; ErrorPairHolder values[END + 1]{}; struct ErrorCodesNames diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index cc4328b8aaa9..76d978d619b9 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7550,6 +7550,23 @@ On the other hand, there is a chance once the task executes that part has alread DECLARE(Bool, export_merge_tree_partition_system_table_prefer_remote_information, false, R"( Controls whether the system.replicated_partition_exports will prefer to query ZooKeeper to get the most up to date information or use the local information. Querying ZooKeeper is expensive, and only available if the ZooKeeper feature flag MULTI_READ is enabled. +)", 0) \ + DECLARE(ExportPartitionAllOnError, export_merge_tree_partition_all_on_error, ExportPartitionAllOnError::throw_first, R"( +Failure handling for `ALTER TABLE ... EXPORT PARTITION ALL ...`. +Possible values: +- `throw_first` (default) - stop at the first failed partition; partitions already scheduled remain scheduled. +- `collect` - try every partition and throw a single aggregated exception at the end if any failed; partitions that succeeded remain scheduled. +- `skip_conflicts` - silently skip partitions that are already exported / being exported (errors with code EXPORT_PARTITION_ALREADY_EXPORTED); fail-fast on every other error. +Has no effect on `EXPORT PARTITION ` (single-partition export). +)", 0) \ + DECLARE(Timezone, iceberg_partition_timezone, "", R"( +Time zone by which partitioning of Iceberg tables was performed. +Possible values: + +- Any valid timezone, e.g. `Europe/Berlin`, `UTC` or `Zulu` +- `` (empty value) - use server or session timezone + +Default value is empty. )", 0) \ DECLARE(String, export_merge_tree_part_filename_pattern, "{part_name}_{checksum}", R"( Pattern for the filename of the exported merge tree part. The `part_name` and `checksum` are calculated and replaced on the fly. Additional macros are supported. diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 7b9f64f33e67..312d7a1f706f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -118,6 +118,7 @@ class WriteBuffer; M(CLASS_NAME, JoinOrderAlgorithm) \ M(CLASS_NAME, DeduplicateInsertSelectMode) \ M(CLASS_NAME, DeduplicateInsertMode) \ + M(CLASS_NAME, ExportPartitionAllOnError) \ COMMON_SETTINGS_SUPPORTED_TYPES(Settings, DECLARE_SETTING_TRAIT) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 245677b8a86d..3626589aef9e 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -331,6 +331,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"export_merge_tree_part_max_rows_per_file", 0, 0, "New setting."}, {"export_merge_tree_partition_lock_inside_the_task", false, false, "New setting."}, {"export_merge_tree_partition_system_table_prefer_remote_information", true, true, "New setting."}, + {"export_merge_tree_partition_all_on_error", "throw_first", "throw_first", "New setting."}, {"export_merge_tree_part_throw_on_pending_mutations", true, true, "New setting."}, {"export_merge_tree_part_throw_on_pending_patch_parts", true, true, "New setting."}, // {"object_storage_cluster", "", "", "Antalya: New setting"}, diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index c2b1a37f5c5a..bae824f4b32e 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -482,4 +482,6 @@ IMPLEMENT_SETTING_ENUM(JemallocProfileFormat, ErrorCodes::BAD_ARGUMENTS, IMPLEMENT_SETTING_AUTO_ENUM(MergeTreePartExportFileAlreadyExistsPolicy, ErrorCodes::BAD_ARGUMENTS); +IMPLEMENT_SETTING_AUTO_ENUM(ExportPartitionAllOnError, ErrorCodes::BAD_ARGUMENTS); + } diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 240d294188f9..9585540354a4 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -573,5 +573,13 @@ enum class MergeTreePartExportFileAlreadyExistsPolicy : uint8_t DECLARE_SETTING_ENUM(MergeTreePartExportFileAlreadyExistsPolicy) +enum class ExportPartitionAllOnError : uint8_t +{ + throw_first, + collect, + skip_conflicts, +}; + +DECLARE_SETTING_ENUM(ExportPartitionAllOnError) } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 3e1e5e87d62e..79fd78b06d70 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6361,8 +6361,11 @@ void MergeTreeData::checkAlterPartitionIsPossible( const auto * partition_ast = command.partition->as(); if (partition_ast && partition_ast->all) { - if (command.type != PartitionCommand::DROP_PARTITION && command.type != PartitionCommand::ATTACH_PARTITION && !(command.type == PartitionCommand::REPLACE_PARTITION && !command.replace)) - throw DB::Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support DROP/DETACH/ATTACH PARTITION ALL currently"); + if (command.type != PartitionCommand::DROP_PARTITION + && command.type != PartitionCommand::ATTACH_PARTITION + && command.type != PartitionCommand::EXPORT_PARTITION + && !(command.type == PartitionCommand::REPLACE_PARTITION && !command.replace)) + throw DB::Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support DROP/DETACH/ATTACH/EXPORT PARTITION ALL currently"); } else { diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ee4e7811378a..ec0269e73e39 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -224,6 +224,7 @@ namespace Setting extern const SettingsBool export_merge_tree_part_throw_on_pending_mutations; extern const SettingsBool export_merge_tree_part_throw_on_pending_patch_parts; extern const SettingsBool export_merge_tree_partition_lock_inside_the_task; + extern const SettingsExportPartitionAllOnError export_merge_tree_partition_all_on_error; extern const SettingsString export_merge_tree_part_filename_pattern; extern const SettingsBool write_full_path_in_iceberg_metadata; extern const SettingsBool allow_insert_into_iceberg; @@ -340,6 +341,8 @@ namespace ErrorCodes extern const int TIMEOUT_EXCEEDED; extern const int INVALID_SETTING_VALUE; extern const int PENDING_MUTATIONS_NOT_ALLOWED; + extern const int EXPORT_PARTITION_ALREADY_EXPORTED; + extern const int PARTITION_EXPORT_FAILED; } namespace ServerSetting @@ -8295,6 +8298,82 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & "If you are exporting to an Apache Iceberg table, you also need to enable the setting `allow_experimental_insert_into_iceberg` on all replicas. The same goes for `allow_experimental_export_merge_tree_part`"); } + /// EXPORT PARTITION ALL: expand into one sub-call per active partition id. + /// Failure handling is controlled by `export_merge_tree_partition_all_on_error`. + if (const auto * partition_ast = command.partition->as(); partition_ast && partition_ast->all) + { + auto partition_id_set = getAllPartitionIds(); + if (partition_id_set.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Table {} has no active partitions to export", + getStorageID().getNameForLogs()); + + /// Sort for deterministic ordering (so failure messages and tests are stable). + std::vector partition_ids(partition_id_set.begin(), partition_id_set.end()); + std::sort(partition_ids.begin(), partition_ids.end()); + + const auto & on_error_setting = query_context->getSettingsRef()[Setting::export_merge_tree_partition_all_on_error]; + const ExportPartitionAllOnError on_error = on_error_setting.value; + + LOG_INFO(log, "EXPORT PARTITION ALL: scheduling export for {} partitions, on_error={}", + partition_ids.size(), on_error_setting.toString()); + + std::vector> failures; /// (partition_id, message) + size_t skipped_conflicts = 0; + + for (const auto & partition_id : partition_ids) + { + PartitionCommand sub = command; + auto synthetic = make_intrusive(); + synthetic->setPartitionID(make_intrusive(partition_id)); + sub.partition = synthetic; + + try + { + exportPartitionToTable(sub, query_context); + } + catch (const Exception & e) + { + switch (on_error) + { + case ExportPartitionAllOnError::throw_first: + throw; + case ExportPartitionAllOnError::skip_conflicts: + if (e.code() == ErrorCodes::EXPORT_PARTITION_ALREADY_EXPORTED) + { + ++skipped_conflicts; + LOG_INFO(log, + "EXPORT PARTITION ALL: skipping partition {} (already exported / concurrent): {}", + partition_id, e.message()); + break; + } + throw; + case ExportPartitionAllOnError::collect: + LOG_WARNING(log, "EXPORT PARTITION ALL: partition {} failed: {}", + partition_id, e.message()); + failures.emplace_back(partition_id, e.message()); + break; + } + } + } + + if (!failures.empty()) + { + String aggregated = fmt::format( + "EXPORT PARTITION ALL: {}/{} partitions failed to schedule. Per-partition errors:", + failures.size(), partition_ids.size()); + for (const auto & [pid, msg] : failures) + aggregated += fmt::format("\n {}: {}", pid, msg); + throw Exception(ErrorCodes::PARTITION_EXPORT_FAILED, "{}", aggregated); + } + + if (skipped_conflicts > 0) + LOG_INFO(log, "EXPORT PARTITION ALL: skipped {} partitions due to existing exports", + skipped_conflicts); + + return; + } + const auto dest_database = query_context->resolveDatabase(command.to_database); const auto dest_table = command.to_table; const auto dest_storage_id = StorageID(dest_database, dest_table); @@ -8374,7 +8453,7 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & if (!has_expired && !query_context->getSettingsRef()[Setting::export_merge_tree_partition_force_export]) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Export with key {} already exported or it is being exported, and it has not expired. Set `export_merge_tree_partition_force_export` to overwrite it.", export_key); + throw Exception(ErrorCodes::EXPORT_PARTITION_ALREADY_EXPORTED, "Export with key {} already exported or it is being exported, and it has not expired. Set `export_merge_tree_partition_force_export` to overwrite it.", export_key); } LOG_INFO(log, "Overwriting export with key {}", export_key); @@ -8567,7 +8646,7 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & { /// Lost the race on the root export node. Current code already /// validated (exists / expired / force) — so this is *always* a race. - throw Exception(ErrorCodes::BAD_ARGUMENTS, + throw Exception(ErrorCodes::EXPORT_PARTITION_ALREADY_EXPORTED, "Export with key {} was created concurrently by another replica. Retry if needed", export_key); } diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index af6ca75bafd7..574296f90c49 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -196,6 +196,32 @@ def test_export_two_partitions_to_iceberg(cluster): assert count_2021 == 1, f"Expected 1 row for year=2021, got {count_2021}" +def test_export_partition_all_to_iceberg(cluster): + """ + `ALTER TABLE ... EXPORT PARTITION ALL TO TABLE ...` schedules every active partition + in one statement and exercises the Iceberg-specific destination compatibility checks + (which are repeated per sub-call inside the loop). + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}") + + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + wait_for_export_status(node, mt_table, iceberg_table, "2021", "COMPLETED") + + count_2020 = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + count_2021 = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2021").strip()) + + assert count_2020 == 3, f"Expected 3 rows for year=2020, got {count_2020}" + assert count_2021 == 1, f"Expected 1 row for year=2021, got {count_2021}" + + def test_failure_is_logged_in_system_table(cluster): """ When S3 is unreachable the export must be marked FAILED in diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index a5e60f81194e..852e8cf78c72 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1506,3 +1506,100 @@ def test_export_partition_resumes_after_stop_moves_during_export(cluster): row_count = int(node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip()) assert row_count == 3, f"Expected 3 rows in S3 after export completed, got {row_count}" + + +def test_export_partition_all(cluster): + """Happy path for `ALTER TABLE ... EXPORT PARTITION ALL TO TABLE ...`. + + Schedules one export task per active partition in a single ALTER, then + verifies every partition lands in the destination S3 table. + """ + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"export_all_mt_{uid}" + s3_table = f"export_all_s3_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, year UInt16)" + f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" + f" PARTITION BY year ORDER BY tuple()" + ) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2021), (3, 2022)") + create_s3_table(node, s3_table) + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + + for partition_id in ("2020", "2021", "2022"): + wait_for_export_status(node, mt_table, s3_table, partition_id, "COMPLETED", timeout=60) + + row_count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert row_count == 3, f"Expected 3 rows in S3 after EXPORT PARTITION ALL, got {row_count}" + + +def test_export_partition_all_failure_modes(cluster): + """Cover the three values of `export_merge_tree_partition_all_on_error`. + + Set up an already-fully-exported source table, then re-run EXPORT PARTITION ALL + with each failure mode and assert the documented behavior. + """ + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"export_all_modes_mt_{uid}" + s3_table = f"export_all_modes_s3_{uid}" + empty_mt = f"export_all_empty_mt_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, year UInt16)" + f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" + f" PARTITION BY year ORDER BY tuple()" + ) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2021), (3, 2022)") + create_s3_table(node, s3_table) + + # First run: schedule + wait for all partitions to complete. + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + for partition_id in ("2020", "2021", "2022"): + wait_for_export_status(node, mt_table, s3_table, partition_id, "COMPLETED", timeout=60) + + # Empty table: throws BAD_ARGUMENTS (no active partitions). + node.query( + f"CREATE TABLE {empty_mt} (id UInt64, year UInt16)" + f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{empty_mt}', 'replica1')" + f" PARTITION BY year ORDER BY tuple()" + ) + error = node.query_and_get_error( + f"ALTER TABLE {empty_mt} EXPORT PARTITION ALL TO TABLE {s3_table}" + ) + assert "no active partitions to export" in error, ( + f"Expected 'no active partitions' error, got: {error}" + ) + + # throw_first (default): re-run aborts on the first conflicting partition. + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}" + f" SETTINGS export_merge_tree_partition_all_on_error = 'throw_first'" + ) + assert "EXPORT_PARTITION_ALREADY_EXPORTED" in error, ( + f"Expected EXPORT_PARTITION_ALREADY_EXPORTED in error, got: {error}" + ) + + # collect: aggregated PARTITION_EXPORT_FAILED message lists every conflicting partition. + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}" + f" SETTINGS export_merge_tree_partition_all_on_error = 'collect'" + ) + assert "PARTITION_EXPORT_FAILED" in error, ( + f"Expected PARTITION_EXPORT_FAILED in error, got: {error}" + ) + for partition_id in ("2020", "2021", "2022"): + assert partition_id in error, ( + f"Expected aggregated error to mention partition {partition_id}, got: {error}" + ) + + # skip_conflicts: succeeds silently because every partition conflicts and is skipped. + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}" + f" SETTINGS export_merge_tree_partition_all_on_error = 'skip_conflicts'" + ) diff --git a/tests/queries/0_stateless/03609_export_merge_tree_partition_all.reference b/tests/queries/0_stateless/03609_export_merge_tree_partition_all.reference new file mode 100644 index 000000000000..b1d058c87a64 --- /dev/null +++ b/tests/queries/0_stateless/03609_export_merge_tree_partition_all.reference @@ -0,0 +1,17 @@ +---- Happy path: EXPORT PARTITION ALL schedules every active partition (default throw_first) +---- Source +1 2020 +2 2021 +3 2022 +---- Destination +1 2020 +2 2021 +3 2022 +---- Empty table: throws BAD_ARGUMENTS +OK +---- throw_first conflict: re-running aborts on first already-exported partition +OK +---- collect conflict: aggregated PARTITION_EXPORT_FAILED at the end +OK +---- skip_conflicts: re-run succeeds silently +OK diff --git a/tests/queries/0_stateless/03609_export_merge_tree_partition_all.sh b/tests/queries/0_stateless/03609_export_merge_tree_partition_all.sh new file mode 100755 index 000000000000..1f3d58cf1fcd --- /dev/null +++ b/tests/queries/0_stateless/03609_export_merge_tree_partition_all.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, replica, no-parallel, no-replicated-database + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +rmt_table="rmt_all_${RANDOM}" +s3_table="s3_all_${RANDOM}" +empty_rmt="empty_rmt_${RANDOM}" + +query() { + $CLICKHOUSE_CLIENT --query "$1" +} + +expect_error() { + local code_name="$1" + local sql="$2" + $CLICKHOUSE_CLIENT --query "$sql" 2>&1 | tr '\n' ' ' | grep -oF "$code_name" > /dev/null \ + && echo "OK" || echo "FAIL" +} + +query "DROP TABLE IF EXISTS $rmt_table, $s3_table, $empty_rmt" + +query "CREATE TABLE $rmt_table (id UInt64, year UInt16) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/$rmt_table', 'replica1') PARTITION BY year ORDER BY tuple()" +query "CREATE TABLE $s3_table (id UInt64, year UInt16) ENGINE = S3(s3_conn, filename='$s3_table', format=Parquet, partition_strategy='hive') PARTITION BY year" + +query "INSERT INTO $rmt_table VALUES (1, 2020), (2, 2021), (3, 2022)" +query "SYSTEM SYNC REPLICA $rmt_table" + +echo "---- Happy path: EXPORT PARTITION ALL schedules every active partition (default throw_first)" +query "ALTER TABLE $rmt_table EXPORT PARTITION ALL TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" + +# Wait for the async per-partition exports to complete. +sleep 15 + +echo "---- Source" +query "SELECT * FROM $rmt_table ORDER BY id" +echo "---- Destination" +query "SELECT * FROM $s3_table ORDER BY id" + +echo "---- Empty table: throws BAD_ARGUMENTS" +query "CREATE TABLE $empty_rmt (id UInt64, year UInt16) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/$empty_rmt', 'replica1') PARTITION BY year ORDER BY tuple()" +expect_error "BAD_ARGUMENTS" \ + "ALTER TABLE $empty_rmt EXPORT PARTITION ALL TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" + +echo "---- throw_first conflict: re-running aborts on first already-exported partition" +expect_error "EXPORT_PARTITION_ALREADY_EXPORTED" \ + "ALTER TABLE $rmt_table EXPORT PARTITION ALL TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_partition_all_on_error = 'throw_first'" + +echo "---- collect conflict: aggregated PARTITION_EXPORT_FAILED at the end" +expect_error "PARTITION_EXPORT_FAILED" \ + "ALTER TABLE $rmt_table EXPORT PARTITION ALL TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_partition_all_on_error = 'collect'" + +echo "---- skip_conflicts: re-run succeeds silently" +query "ALTER TABLE $rmt_table EXPORT PARTITION ALL TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_partition_all_on_error = 'skip_conflicts'" +echo "OK" + +query "DROP TABLE IF EXISTS $rmt_table, $s3_table, $empty_rmt" From 9683b88aa2e2367e6536d64cc47df250a188a9a2 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 7 May 2026 10:57:48 -0300 Subject: [PATCH 2/4] Delete tests/queries/0_stateless/03609_export_merge_tree_partition_all.sh --- .../03609_export_merge_tree_partition_all.sh | 59 ------------------- 1 file changed, 59 deletions(-) delete mode 100755 tests/queries/0_stateless/03609_export_merge_tree_partition_all.sh diff --git a/tests/queries/0_stateless/03609_export_merge_tree_partition_all.sh b/tests/queries/0_stateless/03609_export_merge_tree_partition_all.sh deleted file mode 100755 index 1f3d58cf1fcd..000000000000 --- a/tests/queries/0_stateless/03609_export_merge_tree_partition_all.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest, replica, no-parallel, no-replicated-database - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -rmt_table="rmt_all_${RANDOM}" -s3_table="s3_all_${RANDOM}" -empty_rmt="empty_rmt_${RANDOM}" - -query() { - $CLICKHOUSE_CLIENT --query "$1" -} - -expect_error() { - local code_name="$1" - local sql="$2" - $CLICKHOUSE_CLIENT --query "$sql" 2>&1 | tr '\n' ' ' | grep -oF "$code_name" > /dev/null \ - && echo "OK" || echo "FAIL" -} - -query "DROP TABLE IF EXISTS $rmt_table, $s3_table, $empty_rmt" - -query "CREATE TABLE $rmt_table (id UInt64, year UInt16) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/$rmt_table', 'replica1') PARTITION BY year ORDER BY tuple()" -query "CREATE TABLE $s3_table (id UInt64, year UInt16) ENGINE = S3(s3_conn, filename='$s3_table', format=Parquet, partition_strategy='hive') PARTITION BY year" - -query "INSERT INTO $rmt_table VALUES (1, 2020), (2, 2021), (3, 2022)" -query "SYSTEM SYNC REPLICA $rmt_table" - -echo "---- Happy path: EXPORT PARTITION ALL schedules every active partition (default throw_first)" -query "ALTER TABLE $rmt_table EXPORT PARTITION ALL TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" - -# Wait for the async per-partition exports to complete. -sleep 15 - -echo "---- Source" -query "SELECT * FROM $rmt_table ORDER BY id" -echo "---- Destination" -query "SELECT * FROM $s3_table ORDER BY id" - -echo "---- Empty table: throws BAD_ARGUMENTS" -query "CREATE TABLE $empty_rmt (id UInt64, year UInt16) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/$empty_rmt', 'replica1') PARTITION BY year ORDER BY tuple()" -expect_error "BAD_ARGUMENTS" \ - "ALTER TABLE $empty_rmt EXPORT PARTITION ALL TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" - -echo "---- throw_first conflict: re-running aborts on first already-exported partition" -expect_error "EXPORT_PARTITION_ALREADY_EXPORTED" \ - "ALTER TABLE $rmt_table EXPORT PARTITION ALL TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_partition_all_on_error = 'throw_first'" - -echo "---- collect conflict: aggregated PARTITION_EXPORT_FAILED at the end" -expect_error "PARTITION_EXPORT_FAILED" \ - "ALTER TABLE $rmt_table EXPORT PARTITION ALL TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_partition_all_on_error = 'collect'" - -echo "---- skip_conflicts: re-run succeeds silently" -query "ALTER TABLE $rmt_table EXPORT PARTITION ALL TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_partition_all_on_error = 'skip_conflicts'" -echo "OK" - -query "DROP TABLE IF EXISTS $rmt_table, $s3_table, $empty_rmt" From 8803c6bd239cc047ac38b72f3f1d77ca2a9a4f65 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 7 May 2026 10:58:08 -0300 Subject: [PATCH 3/4] Delete tests/queries/0_stateless/03609_export_merge_tree_partition_all.reference --- ...09_export_merge_tree_partition_all.reference | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 tests/queries/0_stateless/03609_export_merge_tree_partition_all.reference diff --git a/tests/queries/0_stateless/03609_export_merge_tree_partition_all.reference b/tests/queries/0_stateless/03609_export_merge_tree_partition_all.reference deleted file mode 100644 index b1d058c87a64..000000000000 --- a/tests/queries/0_stateless/03609_export_merge_tree_partition_all.reference +++ /dev/null @@ -1,17 +0,0 @@ ----- Happy path: EXPORT PARTITION ALL schedules every active partition (default throw_first) ----- Source -1 2020 -2 2021 -3 2022 ----- Destination -1 2020 -2 2021 -3 2022 ----- Empty table: throws BAD_ARGUMENTS -OK ----- throw_first conflict: re-running aborts on first already-exported partition -OK ----- collect conflict: aggregated PARTITION_EXPORT_FAILED at the end -OK ----- skip_conflicts: re-run succeeds silently -OK From 5c3b95b3ed5aa22af2bac26f9f65b5c21bb05fcd Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 7 May 2026 11:00:24 -0300 Subject: [PATCH 4/4] Update Settings.cpp --- src/Core/Settings.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 76d978d619b9..8f2b857f42c7 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7558,15 +7558,6 @@ Possible values: - `collect` - try every partition and throw a single aggregated exception at the end if any failed; partitions that succeeded remain scheduled. - `skip_conflicts` - silently skip partitions that are already exported / being exported (errors with code EXPORT_PARTITION_ALREADY_EXPORTED); fail-fast on every other error. Has no effect on `EXPORT PARTITION ` (single-partition export). -)", 0) \ - DECLARE(Timezone, iceberg_partition_timezone, "", R"( -Time zone by which partitioning of Iceberg tables was performed. -Possible values: - -- Any valid timezone, e.g. `Europe/Berlin`, `UTC` or `Zulu` -- `` (empty value) - use server or session timezone - -Default value is empty. )", 0) \ DECLARE(String, export_merge_tree_part_filename_pattern, "{part_name}_{checksum}", R"( Pattern for the filename of the exported merge tree part. The `part_name` and `checksum` are calculated and replaced on the fly. Additional macros are supported.