diff --git a/.gitmodules b/.gitmodules index 1c7304defc3c..835ad0b2e922 100644 --- a/.gitmodules +++ b/.gitmodules @@ -154,7 +154,7 @@ url = https://github.com/ClickHouse/rocksdb [submodule "contrib/xz"] path = contrib/xz - url = https://github.com/xz-mirror/xz + url = https://github.com/tukaani-project/xz [submodule "contrib/abseil-cpp"] path = contrib/abseil-cpp url = https://github.com/ClickHouse/abseil-cpp.git diff --git a/ci/jobs/scripts/docker_server/config.sh b/ci/jobs/scripts/docker_server/config.sh new file mode 100644 index 000000000000..84882b1745df --- /dev/null +++ b/ci/jobs/scripts/docker_server/config.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Get current file directory +currentDir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" + +# iterate over all directories in current path +clickhouseTests=$( find "$currentDir"/tests/ -maxdepth 1 -name 'clickhouse-*' -not -name 'clickhouse-distroless-*' -type d -exec basename {} \; ) +clickhouseDistrolessTests=$( find "$currentDir"/tests/ -maxdepth 1 -name 'clickhouse-distroless-*' -type d -exec basename {} \; ) +keeperTests=$( find "$currentDir"/tests/ -maxdepth 1 -name 'keeper-*' -type d -exec basename {} \; ) + +imageTests+=( + ['clickhouse/clickhouse-server']="${clickhouseTests}" + ['clickhouse/clickhouse-server:distroless']="${clickhouseTests} ${clickhouseDistrolessTests}" + ['clickhouse/clickhouse-keeper']="${keeperTests}" + ['clickhouse/clickhouse-keeper:distroless']="${keeperTests}" +) diff --git a/ci/jobs/scripts/docker_server/tests/clickhouse-distroless-initdb/initdb.sql b/ci/jobs/scripts/docker_server/tests/clickhouse-distroless-initdb/initdb.sql new file mode 100644 index 000000000000..16304daf2aa6 --- /dev/null +++ b/ci/jobs/scripts/docker_server/tests/clickhouse-distroless-initdb/initdb.sql @@ -0,0 +1,3 @@ +CREATE DATABASE IF NOT EXISTS test_db; +CREATE TABLE IF NOT EXISTS test_db.test_table (id UInt32, value UInt32) ENGINE = MergeTree ORDER BY id; +INSERT INTO test_db.test_table VALUES (1, 100), (2, 200); diff --git a/ci/jobs/scripts/docker_server/tests/clickhouse-distroless-initdb/run.sh b/ci/jobs/scripts/docker_server/tests/clickhouse-distroless-initdb/run.sh new file mode 100755 index 000000000000..597818de6c21 --- /dev/null +++ b/ci/jobs/scripts/docker_server/tests/clickhouse-distroless-initdb/run.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Verify that clickhouse docker-init executes SQL initdb scripts correctly. +# The distroless image has no shell so initdb scripts must be handled +# by the compiled docker-init entrypoint, not by entrypoint.sh. +set -eo pipefail + +dir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +source "$dir/../lib.sh" + +image="$1" + +export CLICKHOUSE_USER='init_test_user' +export CLICKHOUSE_PASSWORD='init_test_password' + +cid="$( + docker run -d \ + -e CLICKHOUSE_USER \ + -e CLICKHOUSE_PASSWORD \ + -v "$dir/initdb.sql":/docker-entrypoint-initdb.d/initdb.sql:ro \ + --name "$(cname)" \ + "$image" +)" +trap 'docker rm -vf $cid > /dev/null' EXIT + +chCli() { + docker run --rm -i \ + --link "$cid":clickhouse \ + -e CLICKHOUSE_USER \ + -e CLICKHOUSE_PASSWORD \ + "$image" \ + clickhouse-client \ + --host clickhouse \ + --user "$CLICKHOUSE_USER" \ + --password "$CLICKHOUSE_PASSWORD" \ + --query "$*" +} + +# shellcheck source=../../../../../tmp/docker-library/official-images/test/retry.sh +. "$TESTS_LIB_DIR/retry.sh" \ + --tries "$CLICKHOUSE_TEST_TRIES" \ + --sleep "$CLICKHOUSE_TEST_SLEEP" \ + chCli SELECT 1 + +# Verify the initdb script ran and created the table with the expected data +chCli SHOW TABLES IN test_db | grep '^test_table$' >/dev/null +[ "$(chCli 'SELECT SUM(value) FROM test_db.test_table')" = 300 ] diff --git a/ci/jobs/scripts/docker_server/tests/clickhouse-distroless-no-shell/run.sh b/ci/jobs/scripts/docker_server/tests/clickhouse-distroless-no-shell/run.sh new file mode 100755 index 000000000000..c5319f120446 --- /dev/null +++ b/ci/jobs/scripts/docker_server/tests/clickhouse-distroless-no-shell/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Verify the distroless production image contains no shell. +# This is the key property of a distroless image: /bin/sh, /bin/bash, +# and other shells must be absent to reduce the attack surface. +set -eo pipefail + +image="$1" + +if docker run --rm --entrypoint /bin/sh "$image" -c "echo bad" 2>/dev/null; then + echo "FAIL: /bin/sh should not exist in the distroless image" >&2 + exit 1 +fi + +if docker run --rm --entrypoint /bin/bash "$image" -c "echo bad" 2>/dev/null; then + echo "FAIL: /bin/bash should not exist in the distroless image" >&2 + exit 1 +fi diff --git a/ci/jobs/scripts/workflow_hooks/filter_job.py b/ci/jobs/scripts/workflow_hooks/filter_job.py index e8921e2b378c..97e6ea6799a8 100644 --- a/ci/jobs/scripts/workflow_hooks/filter_job.py +++ b/ci/jobs/scripts/workflow_hooks/filter_job.py @@ -54,6 +54,15 @@ def should_skip_job(job_name): if _info_cache is None: _info_cache = Info() + # There is no way to prevent GitHub Actions from running the PR workflow on + # release branches, so we skip all jobs here. The ReleaseCI workflow is used + # for testing on release branches instead. + if ( + Labels.RELEASE in _info_cache.pr_labels + or Labels.RELEASE_LTS in _info_cache.pr_labels + ): + return True, "Skipped for release PR" + changed_files = _info_cache.get_kv_data("changed_files") if not changed_files: print("WARNING: no changed files found for PR - do not filter jobs") diff --git a/ci/workflows/backport_branches.py b/ci/workflows/backport_branches.py index 86172c171eb4..e476f4b4c15e 100644 --- a/ci/workflows/backport_branches.py +++ b/ci/workflows/backport_branches.py @@ -49,7 +49,7 @@ enable_job_filtering_by_changes=True, enable_cache=True, enable_report=True, - enable_automerge=True, + enable_automerge=False, enable_cidb=True, enable_commit_status_on_failure=True, pre_hooks=[ diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 44a97b4ee9f3..4ea045d55b28 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,15 +2,15 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54516) +SET(VERSION_REVISION 54523) SET(VERSION_MAJOR 25) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 16) -SET(VERSION_GITHASH 7a0b36cf8934881236312e9fea094baaf5c709a4) -SET(VERSION_DESCRIBE v25.8.16.10002.altinitytest) -SET(VERSION_STRING 25.8.16.10002.altinitytest) +SET(VERSION_PATCH 23) +SET(VERSION_GITHASH 6d29497525664acca46a1d1cd0d5787e9ad0857d) +SET(VERSION_DESCRIBE v25.8.23.10001.altinitytest) +SET(VERSION_STRING 25.8.23.10001.altinitytest) # end of autochange -SET(VERSION_TWEAK 10002) +SET(VERSION_TWEAK 10001) SET(VERSION_FLAVOUR altinitytest) diff --git a/contrib/aws b/contrib/aws index a86b913abc27..22f694afbdc7 160000 --- a/contrib/aws +++ b/contrib/aws @@ -1 +1 @@ -Subproject commit a86b913abc2795ee23941b24dd51e862214ec6b0 +Subproject commit 22f694afbdc7e9766894998c3745e23f004f8b86 diff --git a/contrib/aws-c-auth b/contrib/aws-c-auth index baeffa791d9d..fc4b87655e5c 160000 --- a/contrib/aws-c-auth +++ b/contrib/aws-c-auth @@ -1 +1 @@ -Subproject commit baeffa791d9d1cf61460662a6d9ac2186aaf05df +Subproject commit fc4b87655e5cd3921f18d1859193c74af4102071 diff --git a/contrib/aws-c-cal b/contrib/aws-c-cal index 1586846816e6..1cb941215889 160000 --- a/contrib/aws-c-cal +++ b/contrib/aws-c-cal @@ -1 +1 @@ -Subproject commit 1586846816e6d7d5ff744a2db943107a3a74a082 +Subproject commit 1cb9412158890201a6ffceed779f90fe1f48180c diff --git a/contrib/aws-c-common b/contrib/aws-c-common index 80f21b3cac5a..95515a8b1ff4 160000 --- a/contrib/aws-c-common +++ b/contrib/aws-c-common @@ -1 +1 @@ -Subproject commit 80f21b3cac5ac51c6b8a62c7d2a5ef58a75195ee +Subproject commit 95515a8b1ff40d5bb14f965ca4cbbe99ad1843df diff --git a/contrib/aws-c-compression b/contrib/aws-c-compression index 99ec79ee2970..d8264e64f698 160000 --- a/contrib/aws-c-compression +++ b/contrib/aws-c-compression @@ -1 +1 @@ -Subproject commit 99ec79ee2970f1a045d4ced1501b97ee521f2f85 +Subproject commit d8264e64f698341eb03039b96b4f44702a9b3f83 diff --git a/contrib/aws-c-event-stream b/contrib/aws-c-event-stream index 08f24e384e5b..f43a3d24a7c1 160000 --- a/contrib/aws-c-event-stream +++ b/contrib/aws-c-event-stream @@ -1 +1 @@ -Subproject commit 08f24e384e5be20bcffa42b49213d24dad7881ae +Subproject commit f43a3d24a7c1f8b50f709ccb4fdf4c7fd2827fff diff --git a/contrib/aws-c-http b/contrib/aws-c-http index a082f8a2067e..a9745ea9998f 160000 --- a/contrib/aws-c-http +++ b/contrib/aws-c-http @@ -1 +1 @@ -Subproject commit a082f8a2067e4a31db73f1d4ffd702a8dc0f7089 +Subproject commit a9745ea9998f679cd7456e7d23cc8820e38c97d4 diff --git a/contrib/aws-c-io b/contrib/aws-c-io index 11ce3c750a1d..89a18aea93e7 160000 --- a/contrib/aws-c-io +++ b/contrib/aws-c-io @@ -1 +1 @@ -Subproject commit 11ce3c750a1dac7b04069fc5bff89e97e91bad4d +Subproject commit 89a18aea93e7b13cd3bfeef46cd0398937013be7 diff --git a/contrib/aws-c-mqtt b/contrib/aws-c-mqtt index 6d36cd372623..1d512d92709f 160000 --- a/contrib/aws-c-mqtt +++ b/contrib/aws-c-mqtt @@ -1 +1 @@ -Subproject commit 6d36cd3726233cb757468d0ea26f6cd8dad151ec +Subproject commit 1d512d92709f60b74e2cafa018e69a2e647f28e9 diff --git a/contrib/aws-c-s3 b/contrib/aws-c-s3 index de36fee8fe7a..e9d1bde139f8 160000 --- a/contrib/aws-c-s3 +++ b/contrib/aws-c-s3 @@ -1 +1 @@ -Subproject commit de36fee8fe7ab02f10987877ae94a805bf440c1f +Subproject commit e9d1bde139f88b08aaa3bf0507f443f31ccede93 diff --git a/contrib/aws-c-sdkutils b/contrib/aws-c-sdkutils index fd8c0ba2e233..f678bda9e21f 160000 --- a/contrib/aws-c-sdkutils +++ b/contrib/aws-c-sdkutils @@ -1 +1 @@ -Subproject commit fd8c0ba2e233997eaaefe82fb818b8b444b956d3 +Subproject commit f678bda9e21f7217e4bbf35e0d1ea59540687933 diff --git a/contrib/aws-checksums b/contrib/aws-checksums index 321b805559c8..1d5f2f1f3e5d 160000 --- a/contrib/aws-checksums +++ b/contrib/aws-checksums @@ -1 +1 @@ -Subproject commit 321b805559c8e911be5bddba13fcbd222a3e2d3a +Subproject commit 1d5f2f1f3e5d013aae8810878ceb5b3f6f258c4e diff --git a/contrib/aws-cmake/AwsGetVersion.cmake b/contrib/aws-cmake/AwsGetVersion.cmake new file mode 100644 index 000000000000..8930f25b2e38 --- /dev/null +++ b/contrib/aws-cmake/AwsGetVersion.cmake @@ -0,0 +1,24 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +function(aws_get_version var_version_major var_version_minor var_version_patch var_version_full var_git_hash) + # Simple version is "MAJOR.MINOR.PATCH" from VERSION file + file(READ "${AWS_CRT_DIR}/VERSION" version_simple) + string(STRIP ${version_simple} version_simple) + set(${var_version_simple} ${version_simple} PARENT_SCOPE) + + string(REPLACE "." ";" VERSION_LIST ${version_simple}) + list(GET VERSION_LIST 0 version_major) + list(GET VERSION_LIST 1 version_minor) + list(GET VERSION_LIST 2 version_patch) + set(${var_version_major} ${version_major} PARENT_SCOPE) + set(${var_version_minor} ${version_minor} PARENT_SCOPE) + set(${var_version_patch} ${version_patch} PARENT_SCOPE) + + # By default, full version is same as simple version. + # But we'll make it more specific later, if we determine that we're not at an exact tagged commit. + set(${var_version_full} ${version_simple} PARENT_SCOPE) + + # Don't include the hash of HEAD in a config file. It's just terrible for build caching and useless + set(var_git_hash "" PARENT_SCOPE) +endfunction() diff --git a/contrib/aws-cmake/CMakeLists.txt b/contrib/aws-cmake/CMakeLists.txt index cc9932a89b13..c36c574215cc 100644 --- a/contrib/aws-cmake/CMakeLists.txt +++ b/contrib/aws-cmake/CMakeLists.txt @@ -25,7 +25,7 @@ include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsFeatureTests.cmake") include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsThreadAffinity.cmake") include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsThreadName.cmake") include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsSIMD.cmake") -include("${ClickHouse_SOURCE_DIR}/contrib/aws-crt-cpp/cmake/AwsGetVersion.cmake") +include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsGetVersion.cmake") set (AWS_STUBS "${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/aws_stubs.cpp") @@ -44,6 +44,12 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DDEBUG_BUILD") endif() +if (OS_LINUX) + list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DAWS_ENABLE_EPOLL") +elseif (OS_DARWIN) + list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DAWS_ENABLE_KQUEUE") +endif() + set(ENABLE_OPENSSL_ENCRYPTION ON) if (ENABLE_OPENSSL_ENCRYPTION) list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DENABLE_OPENSSL_ENCRYPTION") @@ -76,8 +82,8 @@ file(GLOB AWS_SDK_CORE_SRC "${AWS_SDK_CORE_DIR}/source/*.cpp" "${AWS_SDK_CORE_DIR}/source/auth/*.cpp" "${AWS_SDK_CORE_DIR}/source/auth/bearer-token-provider/*.cpp" - "${AWS_SDK_CORE_DIR}/source/auth/signer/*.cpp" "${AWS_SDK_CORE_DIR}/source/auth/signer-provider/*.cpp" + "${AWS_SDK_CORE_DIR}/source/auth/signer/*.cpp" "${AWS_SDK_CORE_DIR}/source/client/*.cpp" "${AWS_SDK_CORE_DIR}/source/config/*.cpp" "${AWS_SDK_CORE_DIR}/source/config/defaults/*.cpp" @@ -96,8 +102,11 @@ file(GLOB AWS_SDK_CORE_SRC "${AWS_SDK_CORE_DIR}/source/smithy/tracing/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/base64/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/cbor/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/checksum/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/component-registry/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/crypto/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/crypto/crt/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/crypto/factory/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/crypto/openssl/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/event/*.cpp" @@ -123,8 +132,6 @@ configure_file("${AWS_SDK_CORE_DIR}/include/aws/core/SDKConfig.h.in" "${CMAKE_CURRENT_BINARY_DIR}/include/aws/core/SDKConfig.h" @ONLY) aws_get_version(AWS_CRT_CPP_VERSION_MAJOR AWS_CRT_CPP_VERSION_MINOR AWS_CRT_CPP_VERSION_PATCH FULL_VERSION GIT_HASH) -# Don't include the hash of HEAD in a config file. It's just terrible for build caching and useless -set(GIT_HASH "") set(FULL_VERSION "${AWS_CRT_CPP_VERSION_MAJOR}.${AWS_CRT_CPP_VERSION_MINOR}.${AWS_CRT_CPP_VERSION_PATCH}-clickhouse") configure_file("${AWS_CRT_DIR}/include/aws/crt/Config.h.in" "${CMAKE_CURRENT_BINARY_DIR}/include/aws/crt/Config.h" @ONLY) @@ -168,12 +175,19 @@ list(APPEND AWS_PUBLIC_INCLUDES "${AWS_AUTH_DIR}/include/") # aws-c-cal file(GLOB AWS_CAL_SRC "${AWS_CAL_DIR}/source/*.c" + "${AWS_CAL_DIR}/source/shared/*.c" ) if (ENABLE_OPENSSL_ENCRYPTION) - file(GLOB AWS_CAL_OS_SRC - "${AWS_CAL_DIR}/source/unix/*.c" - ) + if (OS_LINUX) + file(GLOB AWS_CAL_OS_SRC + "${AWS_CAL_DIR}/source/unix/*.c" + ) + else (OS_DARWIN) + file(GLOB AWS_CAL_OS_SRC + "${AWS_CAL_DIR}/source/darwin/*.c" + ) + endif() list(APPEND AWS_PRIVATE_LIBS OpenSSL::Crypto) endif() @@ -196,6 +210,9 @@ file(GLOB AWS_COMMON_SRC "${AWS_COMMON_DIR}/source/external/*.c" "${AWS_COMMON_DIR}/source/posix/*.c" "${AWS_COMMON_DIR}/source/linux/*.c" + "${AWS_COMMON_DIR}/source/external/libcbor/*.c" + "${AWS_COMMON_DIR}/source/external/libcbor/cbor/*.c" + "${AWS_COMMON_DIR}/source/external/libcbor/cbor/internal/*.c" ) file(GLOB AWS_COMMON_ARCH_SRC @@ -208,9 +225,13 @@ if (AWS_ARCH_INTEL) "${AWS_COMMON_DIR}/source/arch/intel/asm/*.c" ) elseif (AWS_ARCH_ARM64 OR AWS_ARCH_ARM32) - if (AWS_HAVE_AUXV) + if (OS_LINUX) + file(GLOB AWS_COMMON_ARCH_SRC + "${AWS_COMMON_DIR}/source/arch/arm/auxv/cpuid.c" + ) + elseif(OS_DARWIN) file(GLOB AWS_COMMON_ARCH_SRC - "${AWS_COMMON_DIR}/source/arch/arm/asm/*.c" + "${AWS_COMMON_DIR}/source/arch/arm/darwin/cpuid.c" ) endif() endif() @@ -232,17 +253,20 @@ list(APPEND AWS_PUBLIC_INCLUDES "${CMAKE_CURRENT_BINARY_DIR}/include" ) +list(APPEND AWS_PRIVATE_INCLUDES + "${AWS_COMMON_DIR}/source/external/libcbor/cbor/" + "${AWS_COMMON_DIR}/source/external/libcbor/" +) + # aws-checksums file(GLOB AWS_CHECKSUMS_SRC "${AWS_CHECKSUMS_DIR}/source/*.c" - "${AWS_CHECKSUMS_DIR}/source/intel/*.c" - "${AWS_CHECKSUMS_DIR}/source/intel/asm/*.c" - "${AWS_CHECKSUMS_DIR}/source/arm/*.c" ) if(AWS_ARCH_INTEL AND AWS_HAVE_GCC_INLINE_ASM) file(GLOB AWS_CHECKSUMS_ARCH_SRC + "${AWS_CHECKSUMS_DIR}/source/intel/*.c" "${AWS_CHECKSUMS_DIR}/source/intel/asm/*.c" ) endif() @@ -296,6 +320,8 @@ file(GLOB AWS_CRT_SRC "${AWS_CRT_DIR}/source/external/*.cpp" "${AWS_CRT_DIR}/source/http/*.cpp" "${AWS_CRT_DIR}/source/io/*.cpp" + "${AWS_CRT_DIR}/source/cbor/*.cpp" + "${AWS_CRT_DIR}/source/checksum/*.cpp" ) list(APPEND AWS_SOURCES ${AWS_CRT_SRC}) @@ -359,6 +385,12 @@ target_include_directories(_aws SYSTEM BEFORE PUBLIC ${AWS_PUBLIC_INCLUDES}) target_include_directories(_aws SYSTEM BEFORE PRIVATE ${AWS_PRIVATE_INCLUDES}) target_compile_definitions(_aws PUBLIC ${AWS_PUBLIC_COMPILE_DEFS}) target_compile_definitions(_aws PRIVATE ${AWS_PRIVATE_COMPILE_DEFS}) + +if (OS_DARWIN) + target_link_libraries(_aws PRIVATE "-framework CoreFoundation") + target_link_libraries(_aws PRIVATE "-framework Security") +endif() + target_link_libraries(_aws PRIVATE ${AWS_PRIVATE_LIBS}) aws_set_thread_affinity_method(_aws) @@ -366,7 +398,7 @@ aws_set_thread_name_method(_aws) # The library is large - avoid bloat. if (OMIT_HEAVY_DEBUG_SYMBOLS) - target_compile_options (_aws PRIVATE -g0) + target_compile_options (_aws PRIVATE -g1) endif() add_library(ch_contrib::aws_s3 ALIAS _aws) diff --git a/contrib/aws-crt-cpp b/contrib/aws-crt-cpp index e5aa45cacfdc..8776fd0dba27 160000 --- a/contrib/aws-crt-cpp +++ b/contrib/aws-crt-cpp @@ -1 +1 @@ -Subproject commit e5aa45cacfdcda7719ead38760e7c61076f5745f +Subproject commit 8776fd0dba27695736939f47d71b3e8ecf69a06d diff --git a/contrib/curl b/contrib/curl index 2eebc58c4b8d..8c908d2d0a6d 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit 2eebc58c4b8d68c98c8344381a9f6df4cca838fd +Subproject commit 8c908d2d0a6d32abdedda2c52e90bd56ec76c24d diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index 77dd32ea2a6c..8b74a53a51e0 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -118,7 +118,6 @@ set (SRCS "${LIBRARY_DIR}/lib/socks_sspi.c" "${LIBRARY_DIR}/lib/splay.c" "${LIBRARY_DIR}/lib/strcase.c" - "${LIBRARY_DIR}/lib/strdup.c" "${LIBRARY_DIR}/lib/strequal.c" "${LIBRARY_DIR}/lib/strerror.c" "${LIBRARY_DIR}/lib/system_win32.c" @@ -166,6 +165,7 @@ set (SRCS "${LIBRARY_DIR}/lib/vtls/wolfssl.c" "${LIBRARY_DIR}/lib/vtls/x509asn1.c" "${LIBRARY_DIR}/lib/curlx/base64.c" + "${LIBRARY_DIR}/lib/curlx/basename.c" "${LIBRARY_DIR}/lib/curlx/dynbuf.c" "${LIBRARY_DIR}/lib/curlx/fopen.c" "${LIBRARY_DIR}/lib/curlx/inet_ntop.c" @@ -173,6 +173,7 @@ set (SRCS "${LIBRARY_DIR}/lib/curlx/multibyte.c" "${LIBRARY_DIR}/lib/curlx/nonblock.c" "${LIBRARY_DIR}/lib/curlx/strcopy.c" + "${LIBRARY_DIR}/lib/curlx/strdup.c" "${LIBRARY_DIR}/lib/curlx/strerr.c" "${LIBRARY_DIR}/lib/curlx/strparse.c" "${LIBRARY_DIR}/lib/curlx/timediff.c" diff --git a/contrib/libarchive b/contrib/libarchive index 7f53fce04e4e..3a9249b4eeb2 160000 --- a/contrib/libarchive +++ b/contrib/libarchive @@ -1 +1 @@ -Subproject commit 7f53fce04e4e672230f4eb80b219af17975e4f83 +Subproject commit 3a9249b4eeb2a101ca4e0d2b12e4007642bac126 diff --git a/contrib/libxml2 b/contrib/libxml2 index 74f3154320df..b7fa62cbe8ef 160000 --- a/contrib/libxml2 +++ b/contrib/libxml2 @@ -1 +1 @@ -Subproject commit 74f3154320df8950eceae4951975cc9dfc3a254d +Subproject commit b7fa62cbe8ef0df5869e000d5b690bdedd07f33e diff --git a/contrib/libxml2-cmake/CMakeLists.txt b/contrib/libxml2-cmake/CMakeLists.txt index 482793ee0c35..dfb35933e96c 100644 --- a/contrib/libxml2-cmake/CMakeLists.txt +++ b/contrib/libxml2-cmake/CMakeLists.txt @@ -30,7 +30,6 @@ set(SRCS "${LIBXML2_SOURCE_DIR}/xmlschemas.c" "${LIBXML2_SOURCE_DIR}/xmlschemastypes.c" "${LIBXML2_SOURCE_DIR}/xmlregexp.c" - "${LIBXML2_SOURCE_DIR}/xmlunicode.c" "${LIBXML2_SOURCE_DIR}/relaxng.c" "${LIBXML2_SOURCE_DIR}/catalog.c" "${LIBXML2_SOURCE_DIR}/HTMLparser.c" diff --git a/contrib/libxml2-cmake/README.MD b/contrib/libxml2-cmake/README.MD index 10782eccfc53..439da6b9d6ba 100644 --- a/contrib/libxml2-cmake/README.MD +++ b/contrib/libxml2-cmake/README.MD @@ -1,2 +1,2 @@ -./configure CPPFLAGS="-DHAVE_GETENTROPY=0" \ No newline at end of file +./configure CPPFLAGS="-DHAVE_GETENTROPY=0" diff --git a/contrib/libxml2-cmake/linux_x86_64/include/config.h b/contrib/libxml2-cmake/linux_x86_64/include/config.h index c1fbd48d765b..600e7570e990 100644 --- a/contrib/libxml2-cmake/linux_x86_64/include/config.h +++ b/contrib/libxml2-cmake/linux_x86_64/include/config.h @@ -1,6 +1,10 @@ /* config.h. Generated from config.h.in by configure. */ /* config.h.in. Generated from configure.ac by autoheader. */ +/* Define to 1 if you have the declaration of `getentropy', and to 0 if you + don't. */ +#define HAVE_DECL_GETENTROPY 0 + /* Define to 1 if you have the declaration of `glob', and to 0 if you don't. */ #define HAVE_DECL_GLOB 1 @@ -27,12 +31,6 @@ /* Define if readline library is available */ /* #undef HAVE_LIBREADLINE */ -/* Define to 1 if you have the header file. */ -/* #undef HAVE_LZMA_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_POLL_H */ - /* Define to 1 if you have the header file. */ #define HAVE_PTHREAD_H 1 @@ -79,7 +77,7 @@ #define PACKAGE_NAME "libxml2" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "libxml2 2.14.5" +#define PACKAGE_STRING "libxml2 2.15.1" /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "libxml2" @@ -88,7 +86,7 @@ #define PACKAGE_URL "" /* Define to the version of this package. */ -#define PACKAGE_VERSION "2.14.5" +#define PACKAGE_VERSION "2.15.1" /* Define to 1 if all of the C90 standard headers exist (not just the ones required in a freestanding environment). This macro is provided for @@ -96,7 +94,7 @@ #define STDC_HEADERS 1 /* Version number of package */ -#define VERSION "2.14.5" +#define VERSION "2.15.1" /* System configuration directory (/etc) */ #define XML_SYSCONFDIR "/usr/local/etc" diff --git a/contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h b/contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h index c8303df8f838..ff8b0ed8011b 100644 --- a/contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h +++ b/contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h @@ -1,330 +1,253 @@ -/* - * Summary: compile-time version information - * Description: compile-time version information for the XML library +/** + * @file + * + * @brief compile-time version information + * + * compile-time version information for the XML library * - * Copy: See Copyright for the status of this software. + * @copyright See Copyright for the status of this software. * - * Author: Daniel Veillard + * @author Daniel Veillard */ #ifndef __XML_VERSION_H__ #define __XML_VERSION_H__ /** - * LIBXML_DOTTED_VERSION: - * * the version string like "1.2.3" */ -#define LIBXML_DOTTED_VERSION "2.14.5" +#define LIBXML_DOTTED_VERSION "2.15.1" /** - * LIBXML_VERSION: - * * the version number: 1.2.3 value is 10203 */ -#define LIBXML_VERSION 21403 +#define LIBXML_VERSION 21501 /** - * LIBXML_VERSION_STRING: - * * the version number string, 1.2.3 value is "10203" */ -#define LIBXML_VERSION_STRING "21403" +#define LIBXML_VERSION_STRING "21501" /** - * LIBXML_VERSION_EXTRA: - * * extra version information, used to show a git commit description */ #define LIBXML_VERSION_EXTRA "" /** - * LIBXML_TEST_VERSION: - * * Macro to check that the libxml version in use is compatible with * the version the software has been compiled against */ -#define LIBXML_TEST_VERSION xmlCheckVersion(21403); +#define LIBXML_TEST_VERSION xmlCheckVersion(21501); +#if 1 /** - * LIBXML_THREAD_ENABLED: - * * Whether the thread support is configured in */ -#if 1 #define LIBXML_THREAD_ENABLED #endif +#if 0 /** - * LIBXML_THREAD_ALLOC_ENABLED: - * * Whether the allocation hooks are per-thread */ -#if 0 #define LIBXML_THREAD_ALLOC_ENABLED #endif /** - * LIBXML_TREE_ENABLED: - * * Always enabled since 2.14.0 */ #define LIBXML_TREE_ENABLED +#if 1 /** - * LIBXML_OUTPUT_ENABLED: - * * Whether the serialization/saving support is configured in */ -#if 1 #define LIBXML_OUTPUT_ENABLED #endif +#if 1 /** - * LIBXML_PUSH_ENABLED: - * * Whether the push parsing interfaces are configured in */ -#if 1 #define LIBXML_PUSH_ENABLED #endif +#if 1 /** - * LIBXML_READER_ENABLED: - * * Whether the xmlReader parsing interface is configured in */ -#if 1 #define LIBXML_READER_ENABLED #endif +#if 1 /** - * LIBXML_PATTERN_ENABLED: - * * Whether the xmlPattern node selection interface is configured in */ -#if 1 #define LIBXML_PATTERN_ENABLED #endif +#if 1 /** - * LIBXML_WRITER_ENABLED: - * * Whether the xmlWriter saving interface is configured in */ -#if 1 #define LIBXML_WRITER_ENABLED #endif +#if 1 /** - * LIBXML_SAX1_ENABLED: - * * Whether the older SAX1 interface is configured in */ -#if 1 #define LIBXML_SAX1_ENABLED #endif +#if 0 /** - * LIBXML_HTTP_ENABLED: - * - * Whether the HTTP support is configured in + * HTTP support was removed in 2.15 */ -#if 0 -#define LIBXML_HTTP_ENABLED +#define LIBXML_HTTP_STUBS_ENABLED #endif +#if 1 /** - * LIBXML_VALID_ENABLED: - * * Whether the DTD validation support is configured in */ -#if 1 #define LIBXML_VALID_ENABLED #endif +#if 1 /** - * LIBXML_HTML_ENABLED: - * * Whether the HTML support is configured in */ -#if 1 #define LIBXML_HTML_ENABLED #endif -/** - * LIBXML_LEGACY_ENABLED: - * +/* * Removed in 2.14 */ #undef LIBXML_LEGACY_ENABLED +#if 1 /** - * LIBXML_C14N_ENABLED: - * * Whether the Canonicalization support is configured in */ -#if 1 #define LIBXML_C14N_ENABLED #endif +#if 1 /** - * LIBXML_CATALOG_ENABLED: - * * Whether the Catalog support is configured in */ -#if 1 #define LIBXML_CATALOG_ENABLED +#define LIBXML_SGML_CATALOG_ENABLED #endif +#if 1 /** - * LIBXML_XPATH_ENABLED: - * * Whether XPath is configured in */ -#if 1 #define LIBXML_XPATH_ENABLED #endif +#if 1 /** - * LIBXML_XPTR_ENABLED: - * * Whether XPointer is configured in */ -#if 1 #define LIBXML_XPTR_ENABLED #endif +#if 1 /** - * LIBXML_XINCLUDE_ENABLED: - * * Whether XInclude is configured in */ -#if 1 #define LIBXML_XINCLUDE_ENABLED #endif +#if 1 /** - * LIBXML_ICONV_ENABLED: - * * Whether iconv support is available */ -#if 1 #define LIBXML_ICONV_ENABLED #endif +#if 0 /** - * LIBXML_ICU_ENABLED: - * * Whether icu support is available */ -#if 0 #define LIBXML_ICU_ENABLED #endif +#if 1 /** - * LIBXML_ISO8859X_ENABLED: - * * Whether ISO-8859-* support is made available in case iconv is not */ -#if 1 #define LIBXML_ISO8859X_ENABLED #endif +#if 1 /** - * LIBXML_DEBUG_ENABLED: - * * Whether Debugging module is configured in */ -#if 1 #define LIBXML_DEBUG_ENABLED #endif -/** - * LIBXML_UNICODE_ENABLED: - * +/* * Removed in 2.14 */ #undef LIBXML_UNICODE_ENABLED +#if 1 /** - * LIBXML_REGEXP_ENABLED: - * * Whether the regular expressions interfaces are compiled in */ -#if 1 #define LIBXML_REGEXP_ENABLED #endif +#if 1 /** - * LIBXML_AUTOMATA_ENABLED: - * * Whether the automata interfaces are compiled in */ -#if 1 #define LIBXML_AUTOMATA_ENABLED #endif +#if 1 /** - * LIBXML_RELAXNG_ENABLED: - * * Whether the RelaxNG validation interfaces are compiled in */ -#if 1 #define LIBXML_RELAXNG_ENABLED #endif +#if 1 /** - * LIBXML_SCHEMAS_ENABLED: - * * Whether the Schemas validation interfaces are compiled in */ -#if 1 #define LIBXML_SCHEMAS_ENABLED #endif +#if 0 /** - * LIBXML_SCHEMATRON_ENABLED: - * * Whether the Schematron validation interfaces are compiled in */ -#if 1 #define LIBXML_SCHEMATRON_ENABLED #endif +#if 1 /** - * LIBXML_MODULES_ENABLED: - * * Whether the module interfaces are compiled in */ -#if 1 #define LIBXML_MODULES_ENABLED /** - * LIBXML_MODULE_EXTENSION: - * * the string suffix used by dynamic modules (usually shared libraries) */ -#define LIBXML_MODULE_EXTENSION ".so" +#define LIBXML_MODULE_EXTENSION ".so" #endif +#if 0 /** - * LIBXML_ZLIB_ENABLED: - * * Whether the Zlib support is compiled in */ -#if 0 #define LIBXML_ZLIB_ENABLED #endif -/** - * LIBXML_LZMA_ENABLED: - * - * Whether the Lzma support is compiled in - */ -#if 0 -#define LIBXML_LZMA_ENABLED -#endif - #include #endif diff --git a/contrib/mongo-c-driver b/contrib/mongo-c-driver index 4ee76b070b26..529d7f0af3b8 160000 --- a/contrib/mongo-c-driver +++ b/contrib/mongo-c-driver @@ -1 +1 @@ -Subproject commit 4ee76b070b260de5da1e8c8144c028dfc37efbaf +Subproject commit 529d7f0af3b86d7da6ce90562cf9bcd10e8a3a0a diff --git a/contrib/mongo-c-driver-cmake/CMakeLists.txt b/contrib/mongo-c-driver-cmake/CMakeLists.txt index 0139a052aef0..0b608517607c 100644 --- a/contrib/mongo-c-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-c-driver-cmake/CMakeLists.txt @@ -4,14 +4,14 @@ if(NOT USE_MONGODB) return() endif() -set(libbson_VERSION_MAJOR 1) -set(libbson_VERSION_MINOR 27) -set(libbson_VERSION_PATCH 0) -set(libbson_VERSION 1.27.0) -set(libmongoc_VERSION_MAJOR 1) -set(libmongoc_VERSION_MINOR 27) -set(libmongoc_VERSION_PATCH 0) -set(libmongoc_VERSION 1.27.0) +set(libbson_VERSION_MAJOR 2) +set(libbson_VERSION_MINOR 2) +set(libbson_VERSION_PATCH 2) +set(libbson_VERSION 2.2.2) +set(libmongoc_VERSION_MAJOR 2) +set(libmongoc_VERSION_MINOR 2) +set(libmongoc_VERSION_PATCH 2) +set(libmongoc_VERSION 2.2.2) set(LIBBSON_SOURCES_ROOT "${ClickHouse_SOURCE_DIR}/contrib/mongo-c-driver/src") set(LIBBSON_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libbson/src") @@ -102,12 +102,12 @@ set(MONGOC_HAVE_SCHED_GETCPU 0) set(MONGOC_HAVE_SS_FAMILY 0) configure_file( - ${LIBBSON_SOURCE_DIR}/bson/bson-config.h.in - ${LIBBSON_BINARY_DIR}/bson/bson-config.h + ${LIBBSON_SOURCE_DIR}/bson/config.h.in + ${LIBBSON_BINARY_DIR}/bson/config.h ) configure_file( - ${LIBBSON_SOURCE_DIR}/bson/bson-version.h.in - ${LIBBSON_BINARY_DIR}/bson/bson-version.h + ${LIBBSON_SOURCE_DIR}/bson/version.h.in + ${LIBBSON_BINARY_DIR}/bson/version.h ) set(COMMON_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/common/src") @@ -136,6 +136,8 @@ set(UTF8PROC_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/utf8proc-2.8.0") set(UTF8PROC_SOURCES "${UTF8PROC_SOURCE_DIR}/utf8proc.c") set(UTHASH_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/uthash") +set(MONGOC_CXX_COMPILER_ID "${CMAKE_CXX_COMPILER_ID}") +set(MONGOC_CXX_COMPILER_VERSION "${CMAKE_CXX_COMPILER_VERSION}") configure_file( ${LIBMONGOC_SOURCE_DIR}/mongoc/mongoc-config.h.in ${LIBMONGOC_BINARY_DIR}/mongoc/mongoc-config.h @@ -144,6 +146,10 @@ configure_file( ${LIBMONGOC_SOURCE_DIR}/mongoc/mongoc-version.h.in ${LIBMONGOC_BINARY_DIR}/mongoc/mongoc-version.h ) +configure_file( + ${LIBMONGOC_SOURCE_DIR}/mongoc/mongoc-config-private.h.in + ${LIBMONGOC_BINARY_DIR}/mongoc/mongoc-config-private.h +) add_library(_libmongoc ${LIBMONGOC_SOURCES} ${COMMON_SOURCES} ${UTF8PROC_SOURCES}) add_library(ch_contrib::libmongoc ALIAS _libmongoc) target_include_directories(_libmongoc SYSTEM PUBLIC ${LIBMONGOC_SOURCE_DIR} ${LIBMONGOC_BINARY_DIR} ${LIBMONGOC_SOURCE_DIR}/mongoc ${LIBMONGOC_BINARY_DIR}/mongoc ${COMMON_SOURCE_DIR} ${UTF8PROC_SOURCE_DIR} ${UTHASH_SOURCE_DIR} ) diff --git a/contrib/mongo-cxx-driver b/contrib/mongo-cxx-driver index 3166bdb49b71..4f5273939b5c 160000 --- a/contrib/mongo-cxx-driver +++ b/contrib/mongo-cxx-driver @@ -1 +1 @@ -Subproject commit 3166bdb49b717ce1bc30f46cc2b274ab1de7005b +Subproject commit 4f5273939b5cde587b34719f7c26364e502c00f4 diff --git a/contrib/mongo-cxx-driver-cmake/CMakeLists.txt b/contrib/mongo-cxx-driver-cmake/CMakeLists.txt index 212e099d378c..8c750e1082a4 100644 --- a/contrib/mongo-cxx-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-cxx-driver-cmake/CMakeLists.txt @@ -8,66 +8,115 @@ endif() set(BSONCXX_SOURCES_DIR "${ClickHouse_SOURCE_DIR}/contrib/mongo-cxx-driver/src/bsoncxx") set(BSONCXX_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/mongo-cxx-driver/src/bsoncxx") +include(GenerateExportHeader) + set(BSONCXX_SOURCES + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/private/itoa.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/private/version.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/config.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/export.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/version.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/detail/postlude.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/detail/prelude.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/array/element.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/array/value.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/array/view.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/builder/core.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/config.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/export.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/version.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/decimal128.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/document/element.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/document/value.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/document/view.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/exception/error_code.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/exception/exception.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/json.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/oid.cpp - ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/private/itoa.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/string/view_or_value.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/types.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/types/bson_value/value.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/types/bson_value/view.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/validate.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/vector.cpp ) + set(BSONCXX_POLY_USE_IMPLS ON) configure_file( - ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/config.hpp.in - ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/config.hpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/config.hpp.in + ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v1/config/config.hpp ) configure_file( - ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/version.hpp.in - ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/version.hpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/version.hpp.in + ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v1/config/version.hpp ) configure_file( - ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/private/config.hh.in - ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/private/config.hh + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/private/config/config.hh.in + ${BSONCXX_BINARY_DIR}/lib/bsoncxx/private/config/config.hh ) add_library(_bsoncxx ${BSONCXX_SOURCES}) add_library(ch_contrib::bsoncxx ALIAS _bsoncxx) -target_include_directories(_bsoncxx SYSTEM PUBLIC "${BSONCXX_SOURCES_DIR}/include/bsoncxx/v_noabi" "${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi" "${BSONCXX_BINARY_DIR}/lib/bsoncxx/v_noabi") -target_compile_definitions(_bsoncxx PUBLIC BSONCXX_STATIC) +target_include_directories(_bsoncxx SYSTEM PUBLIC + "${BSONCXX_SOURCES_DIR}/include" + "${BSONCXX_SOURCES_DIR}/include/bsoncxx/v_noabi" + "${BSONCXX_SOURCES_DIR}/lib" + "${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi" + + "${BSONCXX_BINARY_DIR}/lib" + "${BSONCXX_BINARY_DIR}/lib/bsoncxx/v_noabi" +) target_link_libraries(_bsoncxx ch_contrib::libbson) -include(GenerateExportHeader) +# Taken from mongo-cxx-driver/src/bsoncxx/CMakeLists.txt +set(bsoncxx_export_header_custom_content "") +string(APPEND bsoncxx_export_header_custom_content [[ + +#undef BSONCXX_DEPRECATED_EXPORT +#undef BSONCXX_DEPRECATED_NO_EXPORT + +#if defined(_MSC_VER) +#define BSONCXX_ABI_CDECL __cdecl +#else +#define BSONCXX_ABI_CDECL +#endif + +#define BSONCXX_ABI_EXPORT_CDECL(...) BSONCXX_ABI_EXPORT __VA_ARGS__ BSONCXX_ABI_CDECL + +]] +) generate_export_header(_bsoncxx - BASE_NAME BSONCXX - EXPORT_MACRO_NAME BSONCXX_API - NO_EXPORT_MACRO_NAME BSONCXX_PRIVATE - EXPORT_FILE_NAME ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/export.hpp + BASE_NAME BSONCXX_ABI + EXPORT_MACRO_NAME BSONCXX_ABI_EXPORT + DEPRECATED_MACRO_NAME BSONCXX_DEPRECATED + EXPORT_FILE_NAME ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v_noabi/bsoncxx/v1/config/export.hpp STATIC_DEFINE BSONCXX_STATIC + CUSTOM_CONTENT_FROM_VARIABLE bsoncxx_export_header_custom_content ) - set(MONGOCXX_SOURCES_DIR "${ClickHouse_SOURCE_DIR}/contrib/mongo-cxx-driver/src/mongocxx") set(MONGOCXX_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/mongo-cxx-driver/src/mongocxx") set(MONGOCXX_SOURCES + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/private/bson.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/private/conversions.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/private/mongoc.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/private/numeric_casting.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/config.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/export.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/version.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/detail/postlude.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/detail/prelude.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/bulk_write.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/change_stream.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/client.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/client_encryption.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/client_session.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/collection.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/config.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/export.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/version.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/cursor.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/database.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/events/command_failed_event.cpp @@ -84,9 +133,16 @@ set(MONGOCXX_SOURCES ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/events/topology_closed_event.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/events/topology_description.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/events/topology_opening_event.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/exception/authentication_exception.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/exception/bulk_write_exception.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/exception/error_code.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/exception/exception.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/exception/gridfs_exception.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/exception/logic_error.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/exception/operation_exception.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/exception/query_exception.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/exception/server_error_code.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/exception/write_exception.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/gridfs/bucket.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/gridfs/downloader.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/gridfs/uploader.cpp @@ -111,7 +167,6 @@ set(MONGOCXX_SOURCES ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/options/client_encryption.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/options/client_session.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/options/count.cpp - ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/options/create_collection.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/options/data_key.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/options/delete.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/options/distinct.cpp @@ -136,10 +191,6 @@ set(MONGOCXX_SOURCES ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/options/update.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/pipeline.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/pool.cpp - ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/private/conversions.cpp - ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/private/libbson.cpp - ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/private/libmongoc.cpp - ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/private/numeric_casting.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/read_concern.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/read_preference.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/result/bulk_write.cpp @@ -156,37 +207,65 @@ set(MONGOCXX_SOURCES ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/validation_criteria.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/write_concern.cpp ) + set(MONGOCXX_COMPILER_VERSION "${CMAKE_CXX_COMPILER_VERSION}") set(MONGOCXX_COMPILER_ID "${CMAKE_CXX_COMPILER_ID}") -set(MONGOCXX_LINK_WITH_STATIC_MONGOC 1) -set(MONGOCXX_BUILD_STATIC 1) + if(ENABLE_SSL) set(MONGOCXX_ENABLE_SSL 1) endif() +set(BSONCXX_STATIC 1) +set(MONGOCXX_STATIC 1) + configure_file( - ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/config.hpp.in - ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v_noabi/mongocxx/config/config.hpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/config.hpp.in + ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v1/config/config.hpp ) configure_file( - ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/version.hpp.in - ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v_noabi/mongocxx/config/version.hpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/version.hpp.in + ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v1/config/version.hpp ) configure_file( - ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/private/config.hh.in - ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v_noabi/mongocxx/config/private/config.hh + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/private/config/config.hh.in + ${MONGOCXX_BINARY_DIR}/lib/mongocxx/private/config/config.hh ) add_library(_mongocxx ${MONGOCXX_SOURCES}) add_library(ch_contrib::mongocxx ALIAS _mongocxx) -target_include_directories(_mongocxx SYSTEM PUBLIC "${MONGOCXX_SOURCES_DIR}/include/mongocxx/v_noabi" "${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi" "${MONGOCXX_BINARY_DIR}/lib/mongocxx/v_noabi") -target_compile_definitions(_mongocxx PUBLIC MONGOCXX_STATIC) +target_include_directories(_mongocxx SYSTEM PUBLIC + "${MONGOCXX_SOURCES_DIR}/include" + "${MONGOCXX_SOURCES_DIR}/include/mongocxx/v_noabi" + "${MONGOCXX_SOURCES_DIR}/lib" + "${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi" + + "${MONGOCXX_BINARY_DIR}/lib" + "${MONGOCXX_BINARY_DIR}/lib/mongocxx/v_noabi" +) target_link_libraries(_mongocxx ch_contrib::bsoncxx ch_contrib::libmongoc) +# Taken from mongo-cxx-driver/src/mongocxx/CMakeLists.txt +set(mongocxx_export_header_custom_content "") +string(APPEND mongocxx_export_header_custom_content [[ + +#undef MONGOCXX_DEPRECATED_EXPORT +#undef MONGOCXX_DEPRECATED_NO_EXPORT + +#if defined(_MSC_VER) +#define MONGOCXX_ABI_CDECL __cdecl +#else +#define MONGOCXX_ABI_CDECL +#endif + +#define MONGOCXX_ABI_EXPORT_CDECL(...) MONGOCXX_ABI_EXPORT __VA_ARGS__ MONGOCXX_ABI_CDECL + +]] +) generate_export_header(_mongocxx - BASE_NAME MONGOCXX - EXPORT_MACRO_NAME MONGOCXX_API - NO_EXPORT_MACRO_NAME MONGOCXX_PRIVATE - EXPORT_FILE_NAME ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v_noabi/mongocxx/config/export.hpp + BASE_NAME MONGOCXX_ABI + EXPORT_MACRO_NAME MONGOCXX_ABI_EXPORT + DEPRECATED_MACRO_NAME MONGOCXX_DEPRECATED + EXPORT_FILE_NAME ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v_noabi/mongocxx/v1/config/export.hpp STATIC_DEFINE MONGOCXX_STATIC + CUSTOM_CONTENT_FROM_VARIABLE mongocxx_export_header_custom_content ) diff --git a/contrib/postgres b/contrib/postgres index 5ad0c31d0c3a..c37596dd61c5 160000 --- a/contrib/postgres +++ b/contrib/postgres @@ -1 +1 @@ -Subproject commit 5ad0c31d0c3a76ed64655f4d397934b5ecc9696f +Subproject commit c37596dd61c5f2b8b7521fdbcdabc651bd9412c4 diff --git a/contrib/postgres-cmake/pg_config.h b/contrib/postgres-cmake/pg_config.h index 169b0af039ea..12767588b94d 100644 --- a/contrib/postgres-cmake/pg_config.h +++ b/contrib/postgres-cmake/pg_config.h @@ -593,7 +593,7 @@ #define PACKAGE_NAME "PostgreSQL" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "PostgreSQL 18.0" +#define PACKAGE_STRING "PostgreSQL 18.3" /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "postgresql" @@ -602,7 +602,7 @@ #define PACKAGE_URL "https://www.postgresql.org/" /* Define to the version of this package. */ -#define PACKAGE_VERSION "18.0" +#define PACKAGE_VERSION "18.3" /* Define to the name of a signed 128-bit integer type. */ #define PG_INT128_TYPE __int128 @@ -618,19 +618,19 @@ #define PG_MAJORVERSION_NUM 18 /* PostgreSQL minor version number */ -#define PG_MINORVERSION_NUM 0 +#define PG_MINORVERSION_NUM 3 /* Define to best printf format archetype, usually gnu_printf if available. */ #define PG_PRINTF_ATTRIBUTE gnu_printf /* PostgreSQL version as a string */ -#define PG_VERSION "18.0" +#define PG_VERSION "18.3" /* PostgreSQL version as a number */ -#define PG_VERSION_NUM 180000 +#define PG_VERSION_NUM 180003 /* A string containing the version number, platform, and C compiler */ -#define PG_VERSION_STR "PostgreSQL 18.0 on x86_64-pc-linux-gnu, compiled by gcc (GCC) 15.2.1 20250813, 64-bit" +#define PG_VERSION_STR "PostgreSQL 18.3 on x86_64-pc-linux-gnu, compiled by gcc (GCC) 15.2.1 20250813, 64-bit" /* Define to 1 to allow profiling output to be saved separately for each process. */ diff --git a/contrib/xz b/contrib/xz index 869b9d1b4edd..4b73f2ec19a9 160000 --- a/contrib/xz +++ b/contrib/xz @@ -1 +1 @@ -Subproject commit 869b9d1b4edd6df07f819d360d306251f8147353 +Subproject commit 4b73f2ec19a99ef465282fbce633e8deb33691b3 diff --git a/contrib/xz-cmake/CMakeLists.txt b/contrib/xz-cmake/CMakeLists.txt index c73433d9863c..0848fc753a10 100644 --- a/contrib/xz-cmake/CMakeLists.txt +++ b/contrib/xz-cmake/CMakeLists.txt @@ -30,22 +30,26 @@ add_compile_definitions( HAVE_CHECK_SHA256 HAVE_DECODERS HAVE_DECODER_ARM + HAVE_DECODER_ARM64 HAVE_DECODER_ARMTHUMB HAVE_DECODER_DELTA HAVE_DECODER_IA64 HAVE_DECODER_LZMA1 HAVE_DECODER_LZMA2 HAVE_DECODER_POWERPC + HAVE_DECODER_RISCV HAVE_DECODER_SPARC HAVE_DECODER_X86 HAVE_ENCODERS HAVE_ENCODER_ARM + HAVE_ENCODER_ARM64 HAVE_ENCODER_ARMTHUMB HAVE_ENCODER_DELTA HAVE_ENCODER_IA64 HAVE_ENCODER_LZMA1 HAVE_ENCODER_LZMA2 HAVE_ENCODER_POWERPC + HAVE_ENCODER_RISCV HAVE_ENCODER_SPARC HAVE_ENCODER_X86 HAVE_MF_BT2 @@ -126,15 +130,16 @@ add_library(_liblzma ${SRC_DIR}/src/liblzma/api/lzma/vli.h ${SRC_DIR}/src/liblzma/check/check.c ${SRC_DIR}/src/liblzma/check/check.h + ${SRC_DIR}/src/liblzma/check/crc32_arm64.h ${SRC_DIR}/src/liblzma/check/crc32_fast.c - ${SRC_DIR}/src/liblzma/check/crc32_table.c + ${SRC_DIR}/src/liblzma/check/crc32_loongarch.h ${SRC_DIR}/src/liblzma/check/crc32_table_be.h ${SRC_DIR}/src/liblzma/check/crc32_table_le.h ${SRC_DIR}/src/liblzma/check/crc64_fast.c - ${SRC_DIR}/src/liblzma/check/crc64_table.c ${SRC_DIR}/src/liblzma/check/crc64_table_be.h ${SRC_DIR}/src/liblzma/check/crc64_table_le.h - ${SRC_DIR}/src/liblzma/check/crc_macros.h + ${SRC_DIR}/src/liblzma/check/crc_common.h + ${SRC_DIR}/src/liblzma/check/crc_x86_clmul.h ${SRC_DIR}/src/liblzma/check/sha256.c ${SRC_DIR}/src/liblzma/common/alone_decoder.c ${SRC_DIR}/src/liblzma/common/alone_decoder.h @@ -178,19 +183,25 @@ add_library(_liblzma ${SRC_DIR}/src/liblzma/common/index_encoder.c ${SRC_DIR}/src/liblzma/common/index_encoder.h ${SRC_DIR}/src/liblzma/common/index_hash.c + ${SRC_DIR}/src/liblzma/common/lzip_decoder.c + ${SRC_DIR}/src/liblzma/common/lzip_decoder.h ${SRC_DIR}/src/liblzma/common/memcmplen.h + ${SRC_DIR}/src/liblzma/common/microlzma_decoder.c + ${SRC_DIR}/src/liblzma/common/microlzma_encoder.c ${SRC_DIR}/src/liblzma/common/outqueue.c ${SRC_DIR}/src/liblzma/common/outqueue.h ${SRC_DIR}/src/liblzma/common/stream_buffer_decoder.c ${SRC_DIR}/src/liblzma/common/stream_buffer_encoder.c ${SRC_DIR}/src/liblzma/common/stream_decoder.c ${SRC_DIR}/src/liblzma/common/stream_decoder.h + ${SRC_DIR}/src/liblzma/common/stream_decoder_mt.c ${SRC_DIR}/src/liblzma/common/stream_encoder.c ${SRC_DIR}/src/liblzma/common/stream_encoder_mt.c ${SRC_DIR}/src/liblzma/common/stream_flags_common.c ${SRC_DIR}/src/liblzma/common/stream_flags_common.h ${SRC_DIR}/src/liblzma/common/stream_flags_decoder.c ${SRC_DIR}/src/liblzma/common/stream_flags_encoder.c + ${SRC_DIR}/src/liblzma/common/string_conversion.c ${SRC_DIR}/src/liblzma/common/vli_decoder.c ${SRC_DIR}/src/liblzma/common/vli_encoder.c ${SRC_DIR}/src/liblzma/common/vli_size.c @@ -229,9 +240,11 @@ add_library(_liblzma ${SRC_DIR}/src/liblzma/rangecoder/range_decoder.h ${SRC_DIR}/src/liblzma/rangecoder/range_encoder.h ${SRC_DIR}/src/liblzma/simple/arm.c + ${SRC_DIR}/src/liblzma/simple/arm64.c ${SRC_DIR}/src/liblzma/simple/armthumb.c ${SRC_DIR}/src/liblzma/simple/ia64.c ${SRC_DIR}/src/liblzma/simple/powerpc.c + ${SRC_DIR}/src/liblzma/simple/riscv.c ${SRC_DIR}/src/liblzma/simple/simple_coder.c ${SRC_DIR}/src/liblzma/simple/simple_coder.h ${SRC_DIR}/src/liblzma/simple/simple_decoder.c diff --git a/docker/keeper/Dockerfile.distroless b/docker/keeper/Dockerfile.distroless new file mode 100644 index 000000000000..1adc6a362498 --- /dev/null +++ b/docker/keeper/Dockerfile.distroless @@ -0,0 +1,186 @@ +# Distroless ClickHouse Keeper image. +# Contains no shell, no package manager, no coreutils. +# The entrypoint is the compiled `clickhouse docker-init --keeper` subcommand. +# +# Build targets: +# production — gcr.io/distroless/cc-debian13:nonroot (default) +# debug — gcr.io/distroless/cc-debian13:debug-nonroot (includes busybox shell) +# +# Usage: +# docker build -f Dockerfile.distroless --target production -t clickhouse/clickhouse-keeper:distroless . +# docker build -f Dockerfile.distroless --target debug -t clickhouse/clickhouse-keeper:distroless-debug . + +# ────────────────────────────────────────────────────────────────────────────── +# Stage 1: Install ClickHouse and assemble the output directory tree. +# ────────────────────────────────────────────────────────────────────────────── +FROM ubuntu:22.04 AS ch-builder + +ARG DEBIAN_FRONTEND=noninteractive +ARG apt_archive="http://archive.ubuntu.com" + +# user/group precreated explicitly with fixed uid/gid on purpose. +# The same uid/gid (101) is used both for alpine and ubuntu. +RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list \ + && groupadd -r clickhouse --gid=101 \ + && useradd -r -g clickhouse --uid=101 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse \ + && apt-get update \ + && apt-get install --yes --no-install-recommends \ + ca-certificates \ + wget \ + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* + +ARG REPO_CHANNEL="stable" +ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" +ARG VERSION="26.1.2.11" +ARG PACKAGES="clickhouse-common-static clickhouse-keeper" + +ARG deb_location_url="" +ARG DIRECT_DOWNLOAD_URLS="" +ARG single_binary_location_url="" +ARG TARGETARCH + +# Install from direct URLs (deb packages provided by CI). +RUN if [ -n "${DIRECT_DOWNLOAD_URLS}" ]; then \ + rm -rf /tmp/clickhouse_debs \ + && mkdir -p /tmp/clickhouse_debs \ + && for url in $DIRECT_DOWNLOAD_URLS; do \ + wget --progress=bar:force:noscroll "$url" -P /tmp/clickhouse_debs || exit 1 \ + ; done \ + && dpkg -i /tmp/clickhouse_debs/*.deb \ + && rm -rf /tmp/* ; \ + fi + +# Install from a web location with deb packages. +RUN arch="${TARGETARCH:-amd64}" \ + && if [ -n "${deb_location_url}" ]; then \ + rm -rf /tmp/clickhouse_debs \ + && mkdir -p /tmp/clickhouse_debs \ + && for package in ${PACKAGES}; do \ + { wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_${arch}.deb" -P /tmp/clickhouse_debs || \ + wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_all.deb" -P /tmp/clickhouse_debs ; } \ + || exit 1 \ + ; done \ + && dpkg -i /tmp/clickhouse_debs/*.deb \ + && rm -rf /tmp/* ; \ + fi + +# Install from a single binary. +RUN if [ -n "${single_binary_location_url}" ]; then \ + rm -rf /tmp/clickhouse_binary \ + && mkdir -p /tmp/clickhouse_binary \ + && wget --progress=bar:force:noscroll "${single_binary_location_url}" -O /tmp/clickhouse_binary/clickhouse \ + && chmod +x /tmp/clickhouse_binary/clickhouse \ + && /tmp/clickhouse_binary/clickhouse install --user "clickhouse" --group "clickhouse" \ + && rm -rf /tmp/* ; \ + fi + +# Fall back to installation from the ClickHouse repository. +RUN clickhouse local -q 'SELECT 1' >/dev/null 2>&1 && exit 0 || : \ + ; apt-get update \ + && apt-get install --yes --no-install-recommends dirmngr gnupg2 \ + && mkdir -p /etc/apt/sources.list.d \ + && GNUPGHOME=$(mktemp -d) \ + && ( set +e; \ + for KEYSERVER in \ + hkp://keys.openpgp.org:80 \ + hkp://pgp.mit.edu:80 \ + hkp://keyserver.ubuntu.com:80; do \ + GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \ + --keyring /usr/share/keyrings/clickhouse-keyring.gpg \ + --keyserver "$KEYSERVER" \ + --recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 && break; \ + done || exit 1 \ + ) \ + && rm -rf "$GNUPGHOME" \ + && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \ + && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \ + && echo "installing from repository: ${REPOSITORY}" \ + && apt-get update \ + && for package in ${PACKAGES}; do \ + packages="${packages} ${package}=${VERSION}" \ + ; done \ + && apt-get install --yes --no-install-recommends ${packages} \ + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \ + && apt-get autoremove --purge -yq dirmngr gnupg2 + +# Verify the installation. +# Use "clickhouse local" (not "clickhouse-local") because the clickhouse-local symlink +# is provided by clickhouse-server which is not installed here. +RUN clickhouse local -q 'SELECT version()' + +ARG DEFAULT_DATA_DIR="/var/lib/clickhouse" +ARG DEFAULT_LOG_DIR="/var/log/clickhouse-keeper" +ARG DEFAULT_CONFIG_DIR="/etc/clickhouse-keeper" + +# Assemble the output directory tree. +RUN mkdir -p \ + /output/usr/bin \ + /output${DEFAULT_CONFIG_DIR} \ + /output${DEFAULT_DATA_DIR} \ + /output${DEFAULT_LOG_DIR} \ + /output/tmp \ + && cp /usr/bin/clickhouse /output/usr/bin/clickhouse \ + && for link in clickhouse-keeper clickhouse-keeper-client clickhouse-docker-init; do \ + ln -sf /usr/bin/clickhouse "/output/usr/bin/${link}" ; \ + done \ + && if [ -d "${DEFAULT_CONFIG_DIR}" ]; then cp -rp "${DEFAULT_CONFIG_DIR}/." "/output${DEFAULT_CONFIG_DIR}/"; fi \ + && chown -R clickhouse:clickhouse \ + /output${DEFAULT_CONFIG_DIR} \ + /output${DEFAULT_DATA_DIR} \ + /output${DEFAULT_LOG_DIR} \ + && chmod 1777 /output/tmp \ + && { \ + printf 'root:x:0:0:root:/root:/sbin/nologin\n'; \ + printf 'nobody:x:65534:65534:nobody:/nonexistent:/sbin/nologin\n'; \ + printf 'nonroot:x:65532:65532:nonroot:/home/nonroot:/sbin/nologin\n'; \ + printf 'clickhouse:x:101:101:ClickHouse:/var/lib/clickhouse:/sbin/nologin\n'; \ + } > /output/etc/passwd \ + && { \ + printf 'root:x:0:\n'; \ + printf 'nobody:x:65534:\n'; \ + printf 'nonroot:x:65532:\n'; \ + printf 'clickhouse:x:101:\n'; \ + } > /output/etc/group + +# ────────────────────────────────────────────────────────────────────────────── +# Stage 2: Production distroless image. +# ────────────────────────────────────────────────────────────────────────────── +# Pinned 2026-04-26. Refresh: docker pull gcr.io/distroless/cc-debian13:nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/cc-debian13:nonroot +FROM gcr.io/distroless/cc-debian13:nonroot@sha256:8f960b7fc6a5d6e28bb07f982655925d6206678bd9a6cde2ad00ddb5e2077d78 AS production + +COPY --from=ch-builder /output/ / + +ENV TZ=UTC \ + CLICKHOUSE_WATCHDOG_ENABLE=0 + +# 9181: ZooKeeper-compatible client port +# 9234: Raft port (inter-node communication) +EXPOSE 9181 9234 + +VOLUME ["/var/lib/clickhouse", "/var/log/clickhouse-keeper"] + +USER 101:101 +WORKDIR /var/lib/clickhouse + +ENTRYPOINT ["/usr/bin/clickhouse", "docker-init", "--keeper"] + +# ────────────────────────────────────────────────────────────────────────────── +# Stage 3: Debug image — same as production but includes the busybox shell +# at /busybox/sh for interactive troubleshooting. +# ────────────────────────────────────────────────────────────────────────────── +# Pinned 2026-04-26. Refresh: docker pull gcr.io/distroless/cc-debian13:debug-nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/cc-debian13:debug-nonroot +FROM gcr.io/distroless/cc-debian13:debug-nonroot@sha256:55dd32378f7562c890342098a04726f4ef386bb86c87bec3db6ed4eef27d99fb AS debug + +COPY --from=ch-builder /output/ / + +ENV TZ=UTC \ + CLICKHOUSE_WATCHDOG_ENABLE=0 + +EXPOSE 9181 9234 + +VOLUME ["/var/lib/clickhouse", "/var/log/clickhouse-keeper"] + +USER 101:101 +WORKDIR /var/lib/clickhouse + +ENTRYPOINT ["/usr/bin/clickhouse", "docker-init", "--keeper"] diff --git a/docker/server/Dockerfile.distroless b/docker/server/Dockerfile.distroless new file mode 100644 index 000000000000..bfadfe523c36 --- /dev/null +++ b/docker/server/Dockerfile.distroless @@ -0,0 +1,206 @@ +# Distroless ClickHouse server image. +# Contains no shell, no package manager, no coreutils. +# The entrypoint is the compiled `clickhouse docker-init` subcommand. +# +# Build targets: +# production — gcr.io/distroless/cc-debian13:nonroot (default) +# debug — gcr.io/distroless/cc-debian13:debug-nonroot (includes busybox shell) +# +# Usage: +# docker build -f Dockerfile.distroless --target production -t clickhouse/clickhouse-server:distroless . +# docker build -f Dockerfile.distroless --target debug -t clickhouse/clickhouse-server:distroless-debug . + +# ────────────────────────────────────────────────────────────────────────────── +# Stage 1: Install ClickHouse and assemble the output directory tree. +# ────────────────────────────────────────────────────────────────────────────── +FROM ubuntu:22.04 AS ch-builder + +# see https://github.com/moby/moby/issues/4032#issuecomment-192327844 +ARG DEBIAN_FRONTEND=noninteractive + +ARG apt_archive="http://archive.ubuntu.com" + +# user/group precreated explicitly with fixed uid/gid on purpose. +# It is especially important for rootless containers: in that case entrypoint +# can't do chown and owners of mounted volumes should be configured externally. +# We do that in advance at the beginning of Dockerfile before any packages will be +# installed to prevent picking those uid/gid by some unrelated software. +# The same uid/gid (101) is used both for alpine and ubuntu. +RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list \ + && groupadd -r clickhouse --gid=101 \ + && useradd -r -g clickhouse --uid=101 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse \ + && apt-get update \ + && apt-get install --yes --no-install-recommends \ + ca-certificates \ + wget \ + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* + +ARG REPO_CHANNEL="stable" +ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" +ARG VERSION="26.1.2.11" +ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" + +#docker-official-library:off +ARG deb_location_url="" +ARG DIRECT_DOWNLOAD_URLS="" +ARG single_binary_location_url="" +ARG TARGETARCH + +# Install from direct URLs (deb packages provided by CI). +RUN if [ -n "${DIRECT_DOWNLOAD_URLS}" ]; then \ + rm -rf /tmp/clickhouse_debs \ + && mkdir -p /tmp/clickhouse_debs \ + && for url in $DIRECT_DOWNLOAD_URLS; do \ + wget --progress=bar:force:noscroll "$url" -P /tmp/clickhouse_debs || exit 1 \ + ; done \ + && dpkg -i /tmp/clickhouse_debs/*.deb \ + && rm -rf /tmp/* ; \ + fi + +# Install from a web location with deb packages. +RUN arch="${TARGETARCH:-amd64}" \ + && if [ -n "${deb_location_url}" ]; then \ + rm -rf /tmp/clickhouse_debs \ + && mkdir -p /tmp/clickhouse_debs \ + && for package in ${PACKAGES}; do \ + { wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_${arch}.deb" -P /tmp/clickhouse_debs || \ + wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_all.deb" -P /tmp/clickhouse_debs ; } \ + || exit 1 \ + ; done \ + && dpkg -i /tmp/clickhouse_debs/*.deb \ + && rm -rf /tmp/* ; \ + fi + +# Install from a single binary. +RUN if [ -n "${single_binary_location_url}" ]; then \ + rm -rf /tmp/clickhouse_binary \ + && mkdir -p /tmp/clickhouse_binary \ + && wget --progress=bar:force:noscroll "${single_binary_location_url}" -O /tmp/clickhouse_binary/clickhouse \ + && chmod +x /tmp/clickhouse_binary/clickhouse \ + && /tmp/clickhouse_binary/clickhouse install --user "clickhouse" --group "clickhouse" \ + && rm -rf /tmp/* ; \ + fi + +# Fall back to installation from the ClickHouse repository. +RUN clickhouse local -q 'SELECT 1' >/dev/null 2>&1 && exit 0 || : \ + ; apt-get update \ + && apt-get install --yes --no-install-recommends dirmngr gnupg2 \ + && mkdir -p /etc/apt/sources.list.d \ + && GNUPGHOME=$(mktemp -d) \ + && ( set +e; \ + for KEYSERVER in \ + hkp://keys.openpgp.org:80 \ + hkp://pgp.mit.edu:80 \ + hkp://keyserver.ubuntu.com:80; do \ + GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \ + --keyring /usr/share/keyrings/clickhouse-keyring.gpg \ + --keyserver "$KEYSERVER" \ + --recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 && break; \ + done || exit 1 \ + ) \ + && rm -rf "$GNUPGHOME" \ + && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \ + && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \ + && echo "installing from repository: ${REPOSITORY}" \ + && apt-get update \ + && for package in ${PACKAGES}; do \ + packages="${packages} ${package}=${VERSION}" \ + ; done \ + && apt-get install --yes --no-install-recommends ${packages} \ + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \ + && apt-get autoremove --purge -yq dirmngr gnupg2 + +#docker-official-library:on + +# Verify the installation. +RUN clickhouse-local -q 'SELECT * FROM system.build_options' + +# Assemble the output directory tree that will be COPYed into the distroless image. +# Symlinks pointing to /usr/bin/clickhouse are preserved by Docker COPY. +RUN mkdir -p \ + /output/usr/bin \ + /output/etc/clickhouse-server/config.d \ + /output/etc/clickhouse-server/users.d \ + /output/etc/clickhouse-client \ + /output/var/lib/clickhouse \ + /output/var/log/clickhouse-server \ + /output/docker-entrypoint-initdb.d \ + /output/tmp \ + && cp /usr/bin/clickhouse /output/usr/bin/clickhouse \ + && for link in clickhouse-server clickhouse-client clickhouse-local \ + clickhouse-keeper clickhouse-keeper-client \ + clickhouse-docker-init; do \ + ln -sf /usr/bin/clickhouse "/output/usr/bin/${link}" ; \ + done \ + && cp -r /etc/clickhouse-server/. /output/etc/clickhouse-server/ \ + && if [ -d /etc/clickhouse-client ]; then cp -r /etc/clickhouse-client/. /output/etc/clickhouse-client/; fi + +COPY docker_related_config.xml /output/etc/clickhouse-server/config.d/ + +# Set ownership after all config files are in place (including docker_related_config.xml above). +RUN chown -R clickhouse:clickhouse \ + /output/etc/clickhouse-server \ + /output/var/lib/clickhouse \ + /output/var/log/clickhouse-server \ + /output/docker-entrypoint-initdb.d \ + && chmod 1777 /output/tmp + +# Create /etc/passwd and /etc/group that include the clickhouse user (uid/gid 101). +# These entries are merged with the distroless base's existing entries. +RUN { \ + printf 'root:x:0:0:root:/root:/sbin/nologin\n'; \ + printf 'nobody:x:65534:65534:nobody:/nonexistent:/sbin/nologin\n'; \ + printf 'nonroot:x:65532:65532:nonroot:/home/nonroot:/sbin/nologin\n'; \ + printf 'clickhouse:x:101:101:ClickHouse:/var/lib/clickhouse:/sbin/nologin\n'; \ + } > /output/etc/passwd \ + && { \ + printf 'root:x:0:\n'; \ + printf 'nobody:x:65534:\n'; \ + printf 'nonroot:x:65532:\n'; \ + printf 'clickhouse:x:101:\n'; \ + } > /output/etc/group + +# ────────────────────────────────────────────────────────────────────────────── +# Stage 2: Production distroless image. +# ────────────────────────────────────────────────────────────────────────────── +# Pinned 2026-04-26. Refresh: docker pull gcr.io/distroless/cc-debian13:nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/cc-debian13:nonroot +FROM gcr.io/distroless/cc-debian13:nonroot@sha256:8f960b7fc6a5d6e28bb07f982655925d6206678bd9a6cde2ad00ddb5e2077d78 AS production + +COPY --from=ch-builder /output/ / + +ENV CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml \ + LC_ALL=C.UTF-8 \ + TZ=UTC \ + CLICKHOUSE_WATCHDOG_ENABLE=0 + +EXPOSE 9000 8123 9009 + +VOLUME ["/var/lib/clickhouse", "/var/log/clickhouse-server"] + +USER 101:101 +WORKDIR /var/lib/clickhouse + +ENTRYPOINT ["/usr/bin/clickhouse", "docker-init"] + +# ────────────────────────────────────────────────────────────────────────────── +# Stage 3: Debug image — same as production but includes the busybox shell +# at /busybox/sh for interactive troubleshooting. +# ────────────────────────────────────────────────────────────────────────────── +# Pinned 2026-04-26. Refresh: docker pull gcr.io/distroless/cc-debian13:debug-nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/cc-debian13:debug-nonroot +FROM gcr.io/distroless/cc-debian13:debug-nonroot@sha256:55dd32378f7562c890342098a04726f4ef386bb86c87bec3db6ed4eef27d99fb AS debug + +COPY --from=ch-builder /output/ / + +ENV CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml \ + LC_ALL=C.UTF-8 \ + TZ=UTC \ + CLICKHOUSE_WATCHDOG_ENABLE=0 + +EXPOSE 9000 8123 9009 + +VOLUME ["/var/lib/clickhouse", "/var/log/clickhouse-server"] + +USER 101:101 +WORKDIR /var/lib/clickhouse + +ENTRYPOINT ["/usr/bin/clickhouse", "docker-init"] diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 4071df99a354..370933054309 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -99,6 +99,7 @@ add_subdirectory (checksum-for-compressed-block) add_subdirectory (client) add_subdirectory (compressor) add_subdirectory (disks) +add_subdirectory (docker-init) add_subdirectory (extract-from-config) add_subdirectory (format) add_subdirectory (fst-dump-tree) @@ -195,6 +196,7 @@ clickhouse_program_install(clickhouse-checksum-for-compressed-block checksum-for clickhouse_program_install(clickhouse-client client chc) clickhouse_program_install(clickhouse-compressor compressor) clickhouse_program_install(clickhouse-disks disks) +clickhouse_program_install(clickhouse-docker-init docker-init) clickhouse_program_install(clickhouse-extract-from-config extract-from-config) clickhouse_program_install(clickhouse-format format) clickhouse_program_install(clickhouse-fst-dump-tree fst-dump-tree) diff --git a/programs/docker-init/CMakeLists.txt b/programs/docker-init/CMakeLists.txt new file mode 100644 index 000000000000..10ff94881396 --- /dev/null +++ b/programs/docker-init/CMakeLists.txt @@ -0,0 +1,8 @@ +set (CLICKHOUSE_DOCKER_INIT_SOURCES docker-init.cpp) + +set (CLICKHOUSE_DOCKER_INIT_LINK + PRIVATE + clickhouse_common_io +) + +clickhouse_program_add(docker-init) diff --git a/programs/docker-init/docker-init.cpp b/programs/docker-init/docker-init.cpp new file mode 100644 index 000000000000..4f49cd43b2bc --- /dev/null +++ b/programs/docker-init/docker-init.cpp @@ -0,0 +1,1075 @@ +/// clickhouse docker-init — Docker entrypoint for distroless ClickHouse images. +/// Replaces entrypoint.sh in shell-free environments (no bash, no coreutils). +/// +/// Usage: +/// clickhouse docker-init [--keeper] [-- ...] +/// +/// Environment variables (same as entrypoint.sh): +/// CLICKHOUSE_CONFIG, CLICKHOUSE_RUN_AS_ROOT, CLICKHOUSE_DO_NOT_CHOWN, +/// CLICKHOUSE_UID, CLICKHOUSE_GID, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD, +/// CLICKHOUSE_PASSWORD_FILE, CLICKHOUSE_DB, CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT, +/// CLICKHOUSE_SKIP_USER_SETUP, CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS, +/// CLICKHOUSE_INIT_TIMEOUT, CLICKHOUSE_WATCHDOG_ENABLE, KEEPER_CONFIG + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace +{ + +/// Path to the clickhouse multi-tool binary, derived from argv[0]. +/// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +std::string g_clickhouse_binary; + +/// Set by the SIGTERM/SIGINT handler during init to request graceful shutdown. +/// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +volatile sig_atomic_t g_shutdown_requested = 0; + +/// PID of the temporary init server, so the signal handler can forward SIGTERM. +/// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +volatile pid_t g_init_server_pid = 0; + +void shutdownHandler(int sig) +{ + g_shutdown_requested = 1; + + /// Forward the signal to the temporary server if one is running. + pid_t pid = g_init_server_pid; + if (pid > 0) + kill(pid, sig); +} + +/// Get an environment variable value, returning default_value if not set. +std::string getEnv(const char * name, const std::string & default_value = "") +{ + const char * val = std::getenv(name); // NOLINT(concurrency-mt-unsafe) + return val ? std::string(val) : default_value; +} + +/// Build an execvp-compatible argv array from a vector of strings. +std::vector buildArgv(const std::vector & args) +{ + std::vector argv; + argv.reserve(args.size() + 1); + for (const auto & a : args) + argv.push_back(const_cast(a.c_str())); // NOLINT(cppcoreguidelines-pro-type-const-cast) + argv.push_back(nullptr); + return argv; +} + +/// Run a command and wait for it. Returns the exit code (or -1 on error). +int runCommand(const std::vector & args) +{ + pid_t pid = fork(); + if (pid < 0) + return -1; + + if (pid == 0) + { + auto argv = buildArgv(args); + execvp(argv[0], argv.data()); + _exit(127); + } + + int status = 0; + while (waitpid(pid, &status, 0) < 0) + if (errno != EINTR) + return -1; + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; +} + +/// Run a command, capture its stdout, return {exit_code, output_lines}. +std::pair> captureCommand(const std::vector & args) +{ + int pipefd[2]; + if (pipe(pipefd) < 0) + return {-1, {}}; + + pid_t pid = fork(); + if (pid < 0) + { + (void)close(pipefd[0]); + (void)close(pipefd[1]); + return {-1, {}}; + } + + if (pid == 0) + { + (void)close(pipefd[0]); + if (dup2(pipefd[1], STDOUT_FILENO) < 0) + _exit(127); + (void)close(pipefd[1]); + + /// Suppress stderr to avoid noise from --try extractions. + int devnull = open("/dev/null", O_WRONLY); + if (devnull >= 0) + { + (void)dup2(devnull, STDERR_FILENO); + (void)close(devnull); + } + + auto argv = buildArgv(args); + execvp(argv[0], argv.data()); + _exit(127); + } + + (void)close(pipefd[1]); + + std::string output; + char buf[4096]; + ssize_t n; + while ((n = read(pipefd[0], buf, sizeof(buf))) > 0) + output.append(buf, static_cast(n)); + (void)close(pipefd[0]); + + int status = 0; + while (waitpid(pid, &status, 0) < 0) + if (errno != EINTR) + return {-1, {}}; + + /// Split output into non-empty lines. + std::vector lines; + { + size_t pos = 0; + while (pos < output.size()) + { + size_t found = output.find('\n', pos); + if (found == std::string::npos) + found = output.size(); + std::string line = output.substr(pos, found - pos); + if (!line.empty() && line.back() == '\r') + line.pop_back(); + if (!line.empty()) + lines.push_back(std::move(line)); + pos = found + 1; + } + } + + return {WIFEXITED(status) ? WEXITSTATUS(status) : -1, std::move(lines)}; +} + +/// Run two commands connected by a pipe: lhs | rhs. +/// Returns the exit code of the rhs process. +int runPipeline(const std::vector & lhs, const std::vector & rhs) +{ + int pipefd[2]; + if (pipe(pipefd) < 0) + return -1; + + pid_t lhs_pid = fork(); + if (lhs_pid < 0) + { + (void)close(pipefd[0]); + (void)close(pipefd[1]); + return -1; + } + if (lhs_pid == 0) + { + (void)close(pipefd[0]); + (void)dup2(pipefd[1], STDOUT_FILENO); + (void)close(pipefd[1]); + auto argv = buildArgv(lhs); + execvp(argv[0], argv.data()); + _exit(127); + } + + pid_t rhs_pid = fork(); + if (rhs_pid < 0) + { + (void)close(pipefd[0]); + (void)close(pipefd[1]); + kill(lhs_pid, SIGTERM); + while (waitpid(lhs_pid, nullptr, 0) < 0 && errno == EINTR) {} + return -1; + } + if (rhs_pid == 0) + { + (void)close(pipefd[1]); + (void)dup2(pipefd[0], STDIN_FILENO); + (void)close(pipefd[0]); + auto argv = buildArgv(rhs); + execvp(argv[0], argv.data()); + _exit(127); + } + + (void)close(pipefd[0]); + (void)close(pipefd[1]); + + int lhs_status = 0; + int rhs_status = 0; + while (waitpid(lhs_pid, &lhs_status, 0) < 0 && errno == EINTR) {} + while (waitpid(rhs_pid, &rhs_status, 0) < 0 && errno == EINTR) {} + + return WIFEXITED(rhs_status) ? WEXITSTATUS(rhs_status) : -1; +} + +/// Returns true if the string is a safe ClickHouse identifier: +/// alphanumeric + underscore, not starting with a digit. +/// Used to validate CLICKHOUSE_USER and CLICKHOUSE_DB before embedding in SQL/XML. +bool isValidIdentifier(const std::string & s) +{ + if (s.empty()) + return false; + if (std::isdigit(static_cast(s[0]))) + return false; + for (unsigned char c : s) + if (!std::isalnum(c) && c != '_') + return false; + return true; +} + +/// Extract a single value from a ClickHouse config key via `clickhouse extract-from-config`. +/// Returns an empty string if the key is absent (--try flag suppresses errors). +std::string extractConfigValue(const std::string & config_file, const std::string & key, bool use_users = false) +{ + std::vector args = { + g_clickhouse_binary, "extract-from-config", + "--config-file", config_file, + "--key", key, + "--try", + }; + if (use_users) + args.emplace_back("--users"); + + auto [code, lines] = captureCommand(args); + return (code == 0 && !lines.empty()) ? lines[0] : std::string{}; +} + +/// Extract multiple values from a ClickHouse config key (wildcard patterns return multiple lines). +std::vector extractConfigValues(const std::string & config_file, const std::string & key) +{ + auto [code, lines] = captureCommand({ + g_clickhouse_binary, "extract-from-config", + "--config-file", config_file, + "--key", key, + "--try", + }); + return (code == 0) ? std::move(lines) : std::vector{}; +} + +/// Recursively chown a path. Logs warnings but does not abort on failure. +void recursiveChown(const std::string & path_str, uid_t uid, gid_t gid) +{ + if (lchown(path_str.c_str(), uid, gid) < 0) + std::cerr << "docker-init: warning: lchown " << path_str << ": " << strerror(errno) << "\n"; // NOLINT(concurrency-mt-unsafe) + + std::error_code ec; + if (!fs::is_directory(path_str, ec)) + return; + + for (const auto & entry : fs::recursive_directory_iterator(path_str, fs::directory_options::skip_permission_denied, ec)) + { + if (lchown(entry.path().c_str(), uid, gid) < 0) + std::cerr << "docker-init: warning: lchown " << entry.path() << ": " << strerror(errno) << "\n"; // NOLINT(concurrency-mt-unsafe) + } +} + +/// Create a directory (and all parents) and optionally chown it. +/// +/// Three cases: +/// 1. do_chown=true (root, normal mode): create with fs::create_directories, then chown. +/// 2. do_chown=false, running as root (CLICKHOUSE_DO_NOT_CHOWN=1): delegate to +/// `clickhouse su UID:GID` so the directory is created as the target user. +/// This handles NFS mounts where root is mapped to nobody. +/// 3. do_chown=false, running as non-root: create directly — we are already the target user. +bool createDirectoryAndChown(const std::string & dir, uid_t uid, gid_t gid, bool do_chown) +{ + if (dir.empty()) + return true; + + if (do_chown) + { + std::error_code ec; + fs::create_directories(dir, ec); + if (ec) + { + std::cerr << "docker-init: couldn't create directory " << dir << ": " << ec.message() << "\n"; + return false; + } + + /// Chown only if the owner needs to change (avoids slow recursive chown on already-correct dirs). + struct stat st{}; + if (stat(dir.c_str(), &st) == 0 && (st.st_uid != uid || st.st_gid != gid)) + recursiveChown(dir, uid, gid); + + return true; + } + + if (getuid() == 0) + { + /// Running as root with CLICKHOUSE_DO_NOT_CHOWN or CLICKHOUSE_RUN_AS_ROOT. + /// On NFS mounts root may be remapped to nobody, so create the directory as + /// the target user. Fork a child that drops privileges before calling + /// fs::create_directories — distroless has no mkdir binary. + pid_t pid = fork(); + if (pid < 0) + { + std::cerr << "docker-init: fork failed for directory creation: " << strerror(errno) << "\n"; // NOLINT(concurrency-mt-unsafe) + return false; + } + if (pid == 0) + { + if (setgroups(0, nullptr) < 0 || setgid(gid) < 0 || setuid(uid) < 0) + _exit(1); + std::error_code ec; + fs::create_directories(dir, ec); + _exit(ec ? 1 : 0); + } + int status = 0; + while (waitpid(pid, &status, 0) < 0 && errno == EINTR) {} + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) + { + /// Fallback: try direct creation (works when root is not remapped). + std::error_code ec; + fs::create_directories(dir, ec); + if (ec) + { + std::cerr << "docker-init: couldn't create directory " << dir << ": " << ec.message() << "\n"; + return false; + } + } + return true; + } + + /// Non-root: we are already running as UID:GID, so create the directory directly. + std::error_code ec; + fs::create_directories(dir, ec); + if (ec) + { + std::cerr << "docker-init: couldn't create directory " << dir << ": " << ec.message() << "\n"; + return false; + } + return true; +} + +/// Write the user management XML to `/etc/clickhouse-server/users.d/default-user.xml`. +/// Returns false if a user-requested setup (CLICKHOUSE_USER/PASSWORD/ACCESS_MANAGEMENT) +/// is invalid — caller should treat this as fatal. +bool manageClickHouseUser( + const std::string & config_file, + const std::string & clickhouse_user, + const std::string & clickhouse_password, + const std::string & access_management, + bool skip_user_setup) +{ + if (skip_user_setup) + { + std::cerr << "docker-init: explicitly skip changing user 'default'\n"; + return true; + } + + const std::string users_d_dir = "/etc/clickhouse-server/users.d"; + const std::string default_user_xml = users_d_dir + "/default-user.xml"; + + std::error_code ec; + fs::create_directories(users_d_dir, ec); + + /// Detect whether the default user was customised via a mounted config file. + bool clickhouse_default_changed = false; + std::string users_xml_path = extractConfigValue(config_file, "user_directories.users_xml.path"); + if (!users_xml_path.empty()) + { + std::string abs_users_xml = (users_xml_path[0] == '/') + ? users_xml_path + : (fs::path(config_file).parent_path() / users_xml_path).string(); + + auto join = [](const std::vector & v) + { + std::string s; + for (const auto & line : v) + s += line + "\n"; + return s; + }; + + auto [c1, original] = captureCommand({ + g_clickhouse_binary, "extract-from-config", + "--config-file", abs_users_xml, + "--key", "users.default", "--try", + }); + auto [c2, processed] = captureCommand({ + g_clickhouse_binary, "extract-from-config", + "--config-file", config_file, + "--users", "--key", "users.default", "--try", + }); + + if (c1 == 0 && c2 == 0 && join(original) != join(processed)) + clickhouse_default_changed = true; + } + + bool has_custom_user = !clickhouse_user.empty() && clickhouse_user != "default"; + bool has_password = !clickhouse_password.empty(); + bool has_access_mgmt = access_management != "0"; + + if (has_custom_user || has_password || has_access_mgmt) + { + if (access_management != "0" && access_management != "1") + { + std::cerr << "docker-init: error: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT must be '0' or '1', got '" + << access_management << "'\n"; + return false; + } + + if (!isValidIdentifier(clickhouse_user)) + { + std::cerr << "docker-init: error: CLICKHOUSE_USER '" << clickhouse_user + << "' contains characters not allowed in an XML element name; " + "use only alphanumeric characters and underscores\n"; + return false; + } + + std::cerr << "docker-init: create new user '" << clickhouse_user << "' instead 'default'\n"; + + /// Escape CDATA end marker: ]]> → ]]]]> + std::string escaped_password; + { + std::string_view src = clickhouse_password; + const std::string_view needle = "]]>"; + const std::string_view replacement = "]]]]>"; + size_t pos = 0; + size_t found; + while ((found = src.find(needle, pos)) != std::string_view::npos) + { + escaped_password.append(src, pos, found - pos); + escaped_password += replacement; + pos = found + needle.size(); + } + escaped_password.append(src, pos, src.size() - pos); + } + + { + std::ofstream f(default_user_xml); + f << "\n" + << " \n" + << " \n" + << " \n" + << " \n" + << " \n" + << "\n" + << " <" << clickhouse_user << ">\n" + << " default\n" + << " \n" + << " ::/0\n" + << " \n" + << " \n" + << " default\n" + << " " << access_management << "\n" + << " \n" + << " \n" + << "\n"; + if (!f.good()) + std::cerr << "docker-init: error: failed to write " << default_user_xml << "\n"; + } + } + else if (clickhouse_default_changed) + { + /// A mounted config already customised the user — leave it as-is. + } + else + { + std::cerr << "docker-init: neither CLICKHOUSE_USER nor CLICKHOUSE_PASSWORD is set, " + "disabling network access for user 'default'\n"; + { + std::ofstream f(default_user_xml); + f << "\n" + << " \n" + << " \n" + << " \n" + << " \n" + << " \n" + << " ::1\n" + << " 127.0.0.1\n" + << " \n" + << " \n" + << " \n" + << "\n"; + if (!f.good()) + std::cerr << "docker-init: error: failed to write " << default_user_xml << "\n"; + } + } + return true; +} + +/// Returns false on first error (fail-fast, matches shell `set -e`). +bool createClickHouseDatabase( + const std::vector & client_base, + const std::string & clickhouse_db) +{ + if (clickhouse_db.empty()) + return true; + if (!isValidIdentifier(clickhouse_db)) + { + std::cerr << "docker-init: error: CLICKHOUSE_DB '" << clickhouse_db + << "' contains characters not safe for use in SQL; " + "use only alphanumeric characters and underscores\n"; + return false; + } + std::cerr << "docker-init: create database '" << clickhouse_db << "'\n"; + std::vector args = client_base; + args.insert(args.end(), {"-q", "CREATE DATABASE IF NOT EXISTS " + clickhouse_db}); + if (runCommand(args) != 0) + { + std::cerr << "docker-init: error: failed to create database '" << clickhouse_db << "'\n"; + return false; + } + return true; +} + +/// Returns false on first script failure (fail-fast, matches shell `set -e`). +bool runInitScripts(const std::vector & client_base) +{ + std::error_code ec; + if (!fs::is_directory("/docker-entrypoint-initdb.d", ec)) + return true; + std::vector init_files; + for (const auto & entry : fs::directory_iterator("/docker-entrypoint-initdb.d", ec)) + init_files.push_back(entry.path()); + std::sort(init_files.begin(), init_files.end()); + + for (const auto & path : init_files) + { + std::string filename = path.filename().string(); + + if (filename.ends_with(".sql") && !filename.ends_with(".sql.gz")) + { + std::cerr << "docker-init: running " << path << "\n"; + std::vector args = client_base; + args.emplace_back("--queries-file"); + args.push_back(path.string()); + if (runCommand(args) != 0) + { + std::cerr << "docker-init: error: init script " << path << " failed\n"; + return false; + } + } + else if (filename.ends_with(".sql.gz")) + { + std::cerr << "docker-init: running " << path << " (decompressing)\n"; + /// Decompress via clickhouse-local (auto-detects .gz) and pipe to clickhouse-client. + /// Escape single quotes in the path to prevent SQL injection via crafted filenames. + std::string escaped_path; + for (char c : path.string()) + { + if (c == '\'') + escaped_path += "''"; + else + escaped_path += c; + } + std::vector decompress_args = { + g_clickhouse_binary, "local", + "--query", + "SELECT * FROM file('" + escaped_path + "', RawBLOB) FORMAT RawBLOB", + }; + if (runPipeline(decompress_args, client_base) != 0) + { + std::cerr << "docker-init: error: init script " << path << " failed\n"; + return false; + } + } + else if (filename.ends_with(".sh")) + { + std::cerr << "docker-init: WARNING: shell scripts cannot run in a distroless " + "environment, skipping " << path << "\n"; + } + else + { + std::cerr << "docker-init: ignoring " << path << "\n"; + } + } + return true; +} + +/// Start a temporary ClickHouse server, run init scripts, then stop it. +bool initClickHouseDB( + const std::string & config_file, + const std::string & data_dir, + const std::string & clickhouse_user, + const std::string & clickhouse_password, + uid_t run_uid, + gid_t run_gid, + const std::vector & extra_server_args, + bool always_run_initdb) +{ + /// Skip if data directory is already initialised and CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS is unset. + bool database_exists = fs::is_directory(data_dir + "/data"); + if (!always_run_initdb && database_exists) + { + std::cerr << "docker-init: ClickHouse data directory appears to contain a database; " + "skipping initialization\n"; + return true; + } + + std::string clickhouse_db = getEnv("CLICKHOUSE_DB"); + + /// Check whether /docker-entrypoint-initdb.d has any files. + std::error_code ec; + bool has_init_files = fs::is_directory("/docker-entrypoint-initdb.d", ec) + && fs::directory_iterator("/docker-entrypoint-initdb.d", ec) != fs::directory_iterator{}; + + if (!has_init_files && clickhouse_db.empty()) + return true; + + std::string native_port = extractConfigValue(config_file, "tcp_port"); + if (native_port.empty()) + native_port = "9000"; + + std::string run_as = std::to_string(run_uid) + ":" + std::to_string(run_gid); + + /// Start a temporary server bound only to localhost. + /// The "--" separator is required: positional arguments after "--" override config.xml + /// properties (e.g. --listen_host=127.0.0.1), while options before "--" are parsed by + /// Poco and must be registered. Without "--", Poco rejects "--listen_host" as unknown. + std::vector server_args = { + g_clickhouse_binary, "su", run_as, "clickhouse-server", + "--config-file=" + config_file, + "--", "--listen_host=127.0.0.1", + }; + for (const auto & arg : extra_server_args) + server_args.push_back(arg); + + if (g_shutdown_requested) + return true; + + pid_t server_pid = fork(); + if (server_pid < 0) + { + std::cerr << "docker-init: failed to fork temporary server\n"; + return false; + } + + if (server_pid == 0) + { + auto argv = buildArgv(server_args); + execvp(argv[0], argv.data()); + _exit(127); + } + + /// Allow the signal handler to forward SIGTERM to the temp server. + g_init_server_pid = server_pid; + + /// Poll until the server accepts connections. + /// This is a service-readiness wait, not a race condition workaround. + int tries = 1000; + { + std::string timeout_str = getEnv("CLICKHOUSE_INIT_TIMEOUT", "1000"); + try + { + tries = std::stoi(timeout_str); + } + catch (const std::exception &) + { + std::cerr << "docker-init: warning: invalid CLICKHOUSE_INIT_TIMEOUT '" + << timeout_str << "', using default 1000\n"; + } + } + bool server_ready = false; + + while (tries > 0 && !server_ready && !g_shutdown_requested) + { + pid_t check_pid = fork(); + if (check_pid < 0) + { + /// fork failed — skip this iteration and retry. + --tries; + sleep(1); // NOLINT(concurrency-mt-unsafe) + continue; + } + if (check_pid == 0) + { + int devnull = open("/dev/null", O_WRONLY); + if (devnull >= 0) + { + dup2(devnull, STDOUT_FILENO); + dup2(devnull, STDERR_FILENO); + close(devnull); + } + /// Keep args in a named variable so the strings outlive argv. + std::vector check_args = { + g_clickhouse_binary, "client", + "--host", "127.0.0.1", + "--port", native_port, + "-u", clickhouse_user, + "--password", clickhouse_password, + "--query", "SELECT 1", + }; + auto argv = buildArgv(check_args); + execvp(argv[0], argv.data()); + _exit(127); + } + + int check_status = 0; + while (waitpid(check_pid, &check_status, 0) < 0 && errno == EINTR) {} + if (WIFEXITED(check_status) && WEXITSTATUS(check_status) == 0) + { + server_ready = true; + } + else + { + --tries; + sleep(1); // NOLINT(concurrency-mt-unsafe) -- Wait between health-check retries — not a race condition fix. + } + } + + if (!server_ready) + { + if (g_shutdown_requested) + std::cerr << "docker-init: shutdown requested, stopping init server\n"; + else + std::cerr << "docker-init: ClickHouse init process timed out\n"; + kill(server_pid, SIGTERM); + while (waitpid(server_pid, nullptr, 0) < 0 && errno == EINTR) {} + g_init_server_pid = 0; + return false; + } + + std::vector client_base = { + g_clickhouse_binary, "client", + "--multiquery", + "--host", "127.0.0.1", + "--port", native_port, + "-u", clickhouse_user, + "--password", clickhouse_password, + }; + + const bool ok = createClickHouseDatabase(client_base, clickhouse_db) + && runInitScripts(client_base); + + /// Always stop the temporary server regardless of init result. + kill(server_pid, SIGTERM); + int server_status = 0; + while (waitpid(server_pid, &server_status, 0) < 0 && errno == EINTR) {} + g_init_server_pid = 0; + if (!WIFEXITED(server_status) || WEXITSTATUS(server_status) != 0) + std::cerr << "docker-init: warning: init server did not exit cleanly\n"; + return ok; +} + +} // anonymous namespace + + +int mainEntryClickHouseDockerInit(int argc, char ** argv) +{ + g_clickhouse_binary = (argc > 0 && argv[0][0] != '\0') ? argv[0] : "clickhouse"; + + bool keeper_mode = false; + bool show_help = false; + bool separator_seen = false; + std::vector extra_args; + + for (int i = 1; i < argc; ++i) + { + std::string_view arg = argv[i]; + + if (!separator_seen && (arg == "--help" || arg == "-h")) + { + show_help = true; + break; + } + else if (!separator_seen && arg == "--keeper") + keeper_mode = true; + else if (!separator_seen && arg == "--") + separator_seen = true; + else + extra_args.emplace_back(arg); + } + + if (show_help) + { + std::cout + << "Usage: clickhouse docker-init [--keeper] [-- ...]\n" + "Docker entrypoint for distroless ClickHouse images.\n" + "\nOptions:\n" + " --keeper Start ClickHouse Keeper instead of server\n" + " --help Show this help message\n" + "\nEnvironment variables (server mode):\n" + " CLICKHOUSE_CONFIG Path to config file " + "(default: /etc/clickhouse-server/config.xml)\n" + " CLICKHOUSE_RUN_AS_ROOT Run as root (0/1)\n" + " CLICKHOUSE_DO_NOT_CHOWN Skip chown operations (0/1)\n" + " CLICKHOUSE_UID Override UID to run as\n" + " CLICKHOUSE_GID Override GID to run as\n" + " CLICKHOUSE_USER Default user name (default: default)\n" + " CLICKHOUSE_PASSWORD Default user password\n" + " CLICKHOUSE_PASSWORD_FILE File containing password\n" + " CLICKHOUSE_DB Database to create on init\n" + " CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT Enable access management (0/1)\n" + " CLICKHOUSE_SKIP_USER_SETUP Skip user setup (0/1)\n" + " CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS Always run init scripts\n" + " CLICKHOUSE_INIT_TIMEOUT Max retries for server readiness (default: 1000)\n" + " CLICKHOUSE_WATCHDOG_ENABLE Enable watchdog (default: 0)\n" + "\nEnvironment variables (keeper mode):\n" + " KEEPER_CONFIG Path to keeper config file\n" + " CLICKHOUSE_DATA_DIR Data directory (default: /var/lib/clickhouse)\n" + " LOG_DIR Log directory (default: /var/log/clickhouse-keeper)\n"; + return 0; + } + + /// --- Passthrough mode --- + /// If the first extra argument does not start with '--', treat it as a command to exec + /// directly without server startup. This mirrors entrypoint.sh: + /// if [[ "$1" == "--"* ]]; then start server; fi; exec "$@" + /// + /// For recognized ClickHouse subcommand names (client, local, etc.) resolve the path + /// via bin_dir so that multi-tool dispatch (by argv[0] basename) works correctly. + /// For everything else (echo, date, bash, ...) let PATH resolution handle it. + if (!extra_args.empty() && !extra_args[0].starts_with("--")) + { + static constexpr std::array clickhouse_tools = { + "clickhouse-client", + "clickhouse-local", + "clickhouse-keeper-client", + "clickhouse-benchmark", + "clickhouse-format", + "clickhouse-compressor", + "clickhouse-obfuscator", + "clickhouse-extract-from-config", + "clickhouse-disks", + "client", + "local", + "keeper-client", + "benchmark", + "format", + "compressor", + "obfuscator", + "extract-from-config", + "disks", + }; + + std::string cmd = extra_args[0]; + if (std::find(clickhouse_tools.begin(), clickhouse_tools.end(), extra_args[0]) != clickhouse_tools.end()) + { + /// Build the full path to the symlink (e.g. /usr/bin/clickhouse-client). + /// The symlink points to the clickhouse binary; dispatching is done by argv[0]. + /// Short names like "client" must be resolved to "clickhouse-client" since the + /// distroless image only has "clickhouse-*" symlinks (not bare "client", "local", etc.). + fs::path bin_dir = fs::path(g_clickhouse_binary).parent_path(); + std::string link_name = extra_args[0]; + if (!link_name.starts_with("clickhouse-")) + link_name = "clickhouse-" + link_name; + cmd = (bin_dir / link_name).string(); + } + + std::vector exec_cmd = {cmd}; + for (std::size_t i = 1; i < extra_args.size(); ++i) + exec_cmd.push_back(extra_args[i]); + + auto exec_argv = buildArgv(exec_cmd); + execvp(exec_argv[0], exec_argv.data()); + std::cerr << "docker-init: failed to exec '" << extra_args[0] << "': " << strerror(errno) << "\n"; // NOLINT(concurrency-mt-unsafe) + return 1; + } + + /// --- Resolve identity --- + uid_t current_uid = getuid(); + uid_t run_uid; + gid_t run_gid; + bool do_chown = true; + + if (getEnv("CLICKHOUSE_RUN_AS_ROOT") == "1" || getEnv("CLICKHOUSE_DO_NOT_CHOWN") == "1") + do_chown = false; + + if (current_uid == 0) + { + if (getEnv("CLICKHOUSE_RUN_AS_ROOT") == "1") + { + run_uid = 0; + run_gid = 0; + } + else + { + /// Default to the `clickhouse` system user if it exists, otherwise fall back to UID 101. + uid_t default_uid = 101; + gid_t default_gid = 101; + const passwd * pw = getpwnam("clickhouse"); // NOLINT(concurrency-mt-unsafe) + if (pw) + { + default_uid = pw->pw_uid; + default_gid = pw->pw_gid; + } + + std::string uid_str = getEnv("CLICKHOUSE_UID"); + std::string gid_str = getEnv("CLICKHOUSE_GID"); + run_uid = default_uid; + run_gid = default_gid; + try + { + if (!uid_str.empty()) + run_uid = static_cast(std::stoul(uid_str)); + if (!gid_str.empty()) + run_gid = static_cast(std::stoul(gid_str)); + } + catch (const std::exception &) + { + std::cerr << "docker-init: warning: invalid CLICKHOUSE_UID/GID values, " + "using defaults\n"; + } + } + } + else + { + /// Non-root: cannot chown, run as current user. + run_uid = current_uid; + run_gid = getgid(); + do_chown = false; + } + + std::string run_as = std::to_string(run_uid) + ":" + std::to_string(run_gid); + + /// --- Keeper mode --- + if (keeper_mode) + { + std::string keeper_config = getEnv("KEEPER_CONFIG", "/etc/clickhouse-keeper/keeper_config.xml"); + std::string data_dir = getEnv("CLICKHOUSE_DATA_DIR", "/var/lib/clickhouse"); + std::string log_dir = getEnv("LOG_DIR", "/var/log/clickhouse-keeper"); + + for (const auto & dir : {data_dir, log_dir, + data_dir + "/coordination", + data_dir + "/coordination/log", + data_dir + "/coordination/snapshots"}) + { + if (!createDirectoryAndChown(dir, run_uid, run_gid, do_chown)) + return 1; + } + + /// Default to disabled so Ctrl+C works in Docker. Don't override if already set. + setenv("CLICKHOUSE_WATCHDOG_ENABLE", "0", 0); // NOLINT(concurrency-mt-unsafe) + + chdir(data_dir.c_str()); // NOLINT(bugprone-unused-return-value) + + std::vector exec_args = { + g_clickhouse_binary, "su", run_as, "clickhouse-keeper", + }; + + std::error_code ec; + if (!keeper_config.empty() && fs::exists(keeper_config, ec)) + exec_args.push_back("--config-file=" + keeper_config); + + for (const auto & arg : extra_args) + exec_args.push_back(arg); + + auto exec_argv = buildArgv(exec_args); + execvp(exec_argv[0], exec_argv.data()); + std::cerr << "docker-init: failed to exec clickhouse keeper: " << strerror(errno) << "\n"; // NOLINT(concurrency-mt-unsafe) + return 1; + } + + /// --- Server mode --- + std::string config_file = getEnv("CLICKHOUSE_CONFIG", "/etc/clickhouse-server/config.xml"); + + /// Extract all relevant paths from the config. + std::string data_dir = extractConfigValue(config_file, "path"); + std::string tmp_dir = extractConfigValue(config_file, "tmp_path"); + std::string user_files_path = extractConfigValue(config_file, "user_files_path"); + std::string format_schema_path = extractConfigValue(config_file, "format_schema_path"); + + std::string log_dir; + std::string log_path = extractConfigValue(config_file, "logger.log"); + if (!log_path.empty()) + log_dir = fs::path(log_path).parent_path().string(); + + std::string error_log_dir; + std::string error_log_path = extractConfigValue(config_file, "logger.errorlog"); + if (!error_log_path.empty()) + error_log_dir = fs::path(error_log_path).parent_path().string(); + + auto disk_paths = extractConfigValues(config_file, "storage_configuration.disks.*.path"); + auto disk_metadata_paths = extractConfigValues(config_file, "storage_configuration.disks.*.metadata_path"); + + /// Create and chown data directory first, then cd into it. + if (!data_dir.empty() && !createDirectoryAndChown(data_dir, run_uid, run_gid, do_chown)) + return 1; + + chdir(data_dir.empty() ? "/" : data_dir.c_str()); // NOLINT(bugprone-unused-return-value) + + for (const auto & dir : {error_log_dir, log_dir, tmp_dir, user_files_path, format_schema_path}) + { + if (!dir.empty() && !createDirectoryAndChown(dir, run_uid, run_gid, do_chown)) + return 1; + } + for (const auto & dir : disk_paths) + if (!createDirectoryAndChown(dir, run_uid, run_gid, do_chown)) + return 1; + for (const auto & dir : disk_metadata_paths) + if (!createDirectoryAndChown(dir, run_uid, run_gid, do_chown)) + return 1; + + /// Resolve password (from env or file). + std::string clickhouse_user = getEnv("CLICKHOUSE_USER", "default"); + std::string clickhouse_password = getEnv("CLICKHOUSE_PASSWORD"); + std::string password_file = getEnv("CLICKHOUSE_PASSWORD_FILE"); + if (!password_file.empty()) + { + std::ifstream pf(password_file); + if (pf.is_open()) + std::getline(pf, clickhouse_password); + else + std::cerr << "docker-init: warning: cannot read CLICKHOUSE_PASSWORD_FILE '" + << password_file << "': " << strerror(errno) << "\n"; // NOLINT(concurrency-mt-unsafe) + } + + std::string access_management = getEnv("CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT", "0"); + bool skip_user_setup = (getEnv("CLICKHOUSE_SKIP_USER_SETUP") == "1"); + + if (!manageClickHouseUser(config_file, clickhouse_user, clickhouse_password, access_management, skip_user_setup)) + return 1; + + /// Install signal handlers so `docker stop` during init triggers graceful shutdown. + /// As PID 1, signals without a handler are silently dropped by the kernel. + { + struct sigaction sa{}; + sa.sa_handler = shutdownHandler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; + sigaction(SIGTERM, &sa, nullptr); + sigaction(SIGINT, &sa, nullptr); + } + + bool always_run_initdb = !getEnv("CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS").empty(); + if (!initClickHouseDB(config_file, data_dir, clickhouse_user, clickhouse_password, + run_uid, run_gid, extra_args, always_run_initdb)) + return 1; + + if (g_shutdown_requested) + { + std::cerr << "docker-init: shutdown requested during initialization, exiting\n"; + return 1; + } + + /// Reset signal handlers before exec — the server handles its own signals. + signal(SIGTERM, SIG_DFL); // NOLINT(cert-err33-c) + signal(SIGINT, SIG_DFL); // NOLINT(cert-err33-c) + + /// Set watchdog env — default to disabled so Ctrl+C works in Docker. + if (std::getenv("CLICKHOUSE_WATCHDOG_ENABLE") == nullptr) // NOLINT(concurrency-mt-unsafe) + setenv("CLICKHOUSE_WATCHDOG_ENABLE", "0", 0); // NOLINT(concurrency-mt-unsafe) + + /// Replace this process with clickhouse-server via `clickhouse su`. + std::vector exec_args = { + g_clickhouse_binary, "su", run_as, "clickhouse-server", + "--config-file=" + config_file, + }; + for (const auto & arg : extra_args) + exec_args.push_back(arg); + + auto exec_argv = buildArgv(exec_args); + execvp(exec_argv[0], exec_argv.data()); + std::cerr << "docker-init: failed to exec clickhouse server: " << strerror(errno) << "\n"; // NOLINT(concurrency-mt-unsafe) + return 1; +} diff --git a/programs/main.cpp b/programs/main.cpp index c572436d1738..d7486d488908 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -78,6 +78,7 @@ int mainEntryClickHouseGitImport(int argc, char ** argv); int mainEntryClickHouseLocal(int argc, char ** argv); int mainEntryClickHouseObfuscator(int argc, char ** argv); int mainEntryClickHouseSU(int argc, char ** argv); +int mainEntryClickHouseDockerInit(int argc, char ** argv); int mainEntryClickHouseServer(int argc, char ** argv); int mainEntryClickHouseStaticFilesDiskUploader(int argc, char ** argv); int mainEntryClickHouseZooKeeperDumpTree(int argc, char ** argv); @@ -149,6 +150,7 @@ std::pair clickhouse_applications[] = {"git-import", mainEntryClickHouseGitImport}, {"static-files-disk-uploader", mainEntryClickHouseStaticFilesDiskUploader}, {"su", mainEntryClickHouseSU}, + {"docker-init", mainEntryClickHouseDockerInit}, {"hash-binary", mainEntryClickHouseHashBinary}, {"disks", mainEntryClickHouseDisks}, {"check-marks", mainEntryClickHouseCheckMarks}, diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 91b85cc30cca..78bfc892efe9 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1188,8 +1188,8 @@ try server_settings[ServerSetting::max_thread_pool_size], server_settings[ServerSetting::max_thread_pool_free_size], server_settings[ServerSetting::thread_pool_queue_size], - has_trace_collector ? server_settings[ServerSetting::global_profiler_real_time_period_ns] : 0, - has_trace_collector ? server_settings[ServerSetting::global_profiler_cpu_time_period_ns] : 0); + has_trace_collector ? server_settings[ServerSetting::global_profiler_real_time_period_ns].value : 0, + has_trace_collector ? server_settings[ServerSetting::global_profiler_cpu_time_period_ns].value : 0); if (has_trace_collector) { diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 979434eaa0d0..50901ce22154 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -562,12 +562,50 @@ struct AccessRights::Node friend bool operator!=(const Node & left, const Node & right) { return !(left == right); } + /// Checks whether `this` node's access rights are a superset of `other`'s bool contains(const Node & other) const { - Node tmp_node = *this; - tmp_node.makeIntersection(other); - /// If we get the same node after the intersection, our node is fully covered by the given one. - return tmp_node == other; + /// The check traverses children from both sides: + /// 1) For each child in `other`, find the matching node in `this` and verify containment. + /// 2) For each child in `this`, find the matching node in `other` and verify containment. + /// + /// The reverse traversal (step 2) is needed to handle partial revokes correctly. + /// Example: GRANT SELECT ON *.*, REVOKE SELECT ON foo.* + /// + /// this: other: + /// root (SELECT) root (SELECT) + /// | + /// "foo" (SELECT) + /// | + /// "" (leaf, USAGE) + /// + /// Step 1 alone would pass because `other` has no children to check against, but `this` does not contain `other` because + /// `this` has revoked SELECT on "foo". + + if (!flags.contains(other.flags)) + return false; + + if (other.children) + { + for (const auto & other_child : *other.children) + { + Node this_child = tryGetLeaf(other_child.node_name, other_child.level, !other_child.isLeaf()); + if (!this_child.contains(other_child)) + return false; + } + } + + if (children) + { + for (const auto & this_child : *children) + { + Node other_child = other.tryGetLeaf(this_child.node_name, this_child.level, !this_child.isLeaf()); + if (!this_child.contains(other_child)) + return false; + } + } + + return true; } void makeUnion(const Node & other) diff --git a/src/Access/tests/gtest_access_rights_ops.cpp b/src/Access/tests/gtest_access_rights_ops.cpp index b7d1c102085f..7e6f3362a902 100644 --- a/src/Access/tests/gtest_access_rights_ops.cpp +++ b/src/Access/tests/gtest_access_rights_ops.cpp @@ -924,6 +924,59 @@ TEST(AccessRights, ContainsWithWildcardsAndPartialRevokes) lhs.grantWildcard(AccessType::SELECT, "testing"); rhs.grantWildcard(AccessType::SELECT, "test"); ASSERT_FALSE(lhs.contains(rhs)); + + lhs = {}; + rhs = {}; + lhs.grantWithGrantOption(AccessType::SET_DEFINER); + lhs.revoke(AccessType::SET_DEFINER, "internal-user-1"); + rhs.grantWithGrantOption(AccessType::SET_DEFINER); + rhs.revoke(AccessType::SET_DEFINER, "internal-user-1"); + rhs.revoke(AccessType::SET_DEFINER, "internal-user-2"); + rhs.revoke(AccessType::SET_DEFINER, "internal-user-3"); + ASSERT_TRUE(lhs.contains(rhs)); + + lhs = {}; + rhs = {}; + rhs.grantWithGrantOption(AccessType::SET_DEFINER); + rhs.revoke(AccessType::SET_DEFINER, "internal-user-1"); + lhs.grantWithGrantOption(AccessType::SET_DEFINER); + lhs.revoke(AccessType::SET_DEFINER, "internal-user-1"); + lhs.revoke(AccessType::SET_DEFINER, "internal-user-2"); + lhs.revoke(AccessType::SET_DEFINER, "internal-user-3"); + ASSERT_FALSE(lhs.contains(rhs)); + + lhs = {}; + rhs = {}; + lhs.grantWithGrantOption(AccessType::SET_DEFINER); + lhs.revoke(AccessType::SET_DEFINER, "internal-user-1"); + lhs.revoke(AccessType::SET_DEFINER, "internal-user-2"); + rhs.grantWithGrantOption(AccessType::SET_DEFINER); + rhs.revoke(AccessType::SET_DEFINER, "internal-user-1"); + rhs.revoke(AccessType::SET_DEFINER, "internal-user-2"); + ASSERT_TRUE(lhs.contains(rhs)); + + lhs = {}; + rhs = {}; + lhs.grant(AccessType::CREATE_ROLE); + lhs.grant(AccessType::ROLE_ADMIN); + lhs.grantWithGrantOption(AccessType::SET_DEFINER); + lhs.revoke(AccessType::SET_DEFINER, "internal-user-1"); + rhs.grantWithGrantOption(AccessType::SET_DEFINER); + rhs.revoke(AccessType::SET_DEFINER, "internal-user-1"); + rhs.revoke(AccessType::SET_DEFINER, "internal-user-2"); + rhs.revoke(AccessType::SET_DEFINER, "internal-user-3"); + ASSERT_TRUE(lhs.contains(rhs)); + + lhs = {}; + rhs = {}; + lhs.grant(AccessType::SELECT); + lhs.revoke(AccessType::SELECT, "secret_db1"); + rhs.grant(AccessType::SELECT); + rhs.revoke(AccessType::SELECT, "secret_db1"); + rhs.revoke(AccessType::SELECT, "secret_db2"); + rhs.revoke(AccessType::SELECT, "secret_db3"); + ASSERT_TRUE(lhs.contains(rhs)); + ASSERT_FALSE(rhs.contains(lhs)); } TEST(AccessRights, ColumnLevelWildcardOperations) diff --git a/src/AggregateFunctions/AggregateFunctionGroupNumericIndexedVectorDataBSI.h b/src/AggregateFunctions/AggregateFunctionGroupNumericIndexedVectorDataBSI.h index a65a1fb1ca12..ffcf8816b915 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupNumericIndexedVectorDataBSI.h +++ b/src/AggregateFunctions/AggregateFunctionGroupNumericIndexedVectorDataBSI.h @@ -462,6 +462,19 @@ class BSINumericIndexedVector */ void pointwiseAddInplace(const BSINumericIndexedVector & rhs) { + /// Self-addition requires a deep copy because the full adder logic below + /// performs in-place XOR on shared bitmaps (`sum->rb_xor(*addend)` where + /// `sum` and `addend` alias the same Roaring bitmap via `shallowCopyFrom`), + /// which triggers an assertion in CRoaring (`assert(x1 != x2)`) and would + /// produce incorrect results (A XOR A = 0) in release builds. + if (this == &rhs) + { + BSINumericIndexedVector copy; + copy.deepCopyFrom(rhs); + pointwiseAddInplace(copy); + return; + } + if (isEmpty()) { deepCopyFrom(rhs); @@ -538,6 +551,16 @@ class BSINumericIndexedVector */ void pointwiseSubtractInplace(const BSINumericIndexedVector & rhs) { + /// Self-subtraction requires a deep copy for the same reason as + /// `pointwiseAddInplace`: in-place XOR on aliased bitmaps is undefined. + if (this == &rhs) + { + BSINumericIndexedVector copy; + copy.deepCopyFrom(rhs); + pointwiseSubtractInplace(copy); + return; + } + auto total_indexes = getAllIndex(); total_indexes->rb_or(*rhs.getAllIndex()); diff --git a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp index 0776099685b9..b246a7861553 100644 --- a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp +++ b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp @@ -152,8 +152,8 @@ class CrossToInnerJoinVisitor : public InDepthQueryTreeVisitorWithContext + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include + +#include + +namespace DB +{ + +namespace Setting +{ + +extern const SettingsBool enable_unaligned_array_join; + +} + +namespace +{ + +/// Per-expression usage state inside a single ARRAY JOIN node. +struct ExpressionUsage +{ + /// True when the expression is referenced directly (not via tupleElement), + /// meaning all subcolumns are needed, or it has no nested() inner function. + bool fully_used = false; + + /// Used subcolumn names (only meaningful for nested() expressions). + std::unordered_set used_subcolumns; + + /// Subcolumn names from the nested() first argument. Empty if not a nested() expression. + std::vector nested_subcolumn_names; + + bool hasNested() const { return !nested_subcolumn_names.empty(); } + + bool isUsed() const { return fully_used || !used_subcolumns.empty(); } +}; + +/// Key: (ArrayJoinNode raw ptr, column name) → ExpressionUsage. +using ArrayJoinUsageMap = std::unordered_map>; + +/// Set of ArrayJoinNode raw pointers that we are tracking. +using ArrayJoinNodeSet = std::unordered_set; + +/// Map: (ArrayJoinNode raw ptr, column name) → post-pruning column DataType. +using UpdatedTypeMap = std::unordered_map>; + +/// Visitor that marks which ARRAY JOIN expressions and nested subcolumns are used. +class MarkUsedArrayJoinColumnsVisitor : public InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + + MarkUsedArrayJoinColumnsVisitor(ContextPtr context_, ArrayJoinUsageMap & usage_map_, const ArrayJoinNodeSet & tracked_nodes_) + : Base(std::move(context_)) + , usage_map(usage_map_) + , tracked_nodes(tracked_nodes_) + { + } + + bool needChildVisit(QueryTreeNodePtr & parent, QueryTreeNodePtr & child) + { + /// Skip visiting the join expressions list of a tracked ARRAY JOIN node — + /// those are the expression definitions, not references. + if (tracked_nodes.contains(parent.get())) + { + auto * array_join_node = parent->as(); + if (array_join_node && child.get() == array_join_node->getJoinExpressionsNode().get()) + return false; + } + + /// If the parent is tupleElement whose first argument is a column from a tracked + /// ARRAY JOIN, skip visiting children — enterImpl already handles the tupleElement + /// by marking only the specific subcolumn. + auto * function_node = parent->as(); + if (function_node && function_node->getFunctionName() == "tupleElement") + { + const auto & arguments = function_node->getArguments().getNodes(); + if (arguments.size() >= 2) + { + if (auto * column_node = arguments[0]->as()) + { + auto source = column_node->getColumnSourceOrNull(); + if (source && tracked_nodes.contains(source.get())) + return false; + } + } + } + + return true; + } + + void enterImpl(const QueryTreeNodePtr & node) + { + /// Case 1: tupleElement(array_join_col, 'subcolumn_name') — mark specific subcolumn. + if (auto * function_node = node->as()) + { + if (function_node->getFunctionName() != "tupleElement") + return; + + const auto & arguments = function_node->getArguments().getNodes(); + if (arguments.size() < 2) + return; + + auto * column_node = arguments[0]->as(); + auto * constant_node = arguments[1]->as(); + if (!column_node || !constant_node) + return; + + auto source = column_node->getColumnSourceOrNull(); + if (!source || !tracked_nodes.contains(source.get())) + return; + + auto map_it = usage_map.find(source.get()); + if (map_it == usage_map.end()) + return; + + auto expr_it = map_it->second.find(column_node->getColumnName()); + if (expr_it == map_it->second.end() || expr_it->second.fully_used) + return; + + if (expr_it->second.hasNested()) + { + const auto & value = constant_node->getValue(); + if (value.getType() == Field::Types::String) + { + expr_it->second.used_subcolumns.insert(value.safeGet()); + } + else if (value.getType() == Field::Types::UInt64) + { + /// tupleElement uses 1-based indexing. + UInt64 index = value.safeGet(); + if (index >= 1 && index <= expr_it->second.nested_subcolumn_names.size()) + expr_it->second.used_subcolumns.insert(expr_it->second.nested_subcolumn_names[index - 1]); + else + expr_it->second.fully_used = true; + } + else + { + expr_it->second.fully_used = true; + } + } + else + expr_it->second.fully_used = true; + + return; + } + + /// Case 2: direct reference to an ARRAY JOIN column — mark fully used. + auto * column_node = node->as(); + if (!column_node) + return; + + auto source = column_node->getColumnSourceOrNull(); + if (!source || !tracked_nodes.contains(source.get())) + return; + + auto map_it = usage_map.find(source.get()); + if (map_it == usage_map.end()) + return; + + auto expr_it = map_it->second.find(column_node->getColumnName()); + if (expr_it != map_it->second.end()) + expr_it->second.fully_used = true; + } + +private: + ArrayJoinUsageMap & usage_map; + const ArrayJoinNodeSet & tracked_nodes; +}; + +void pruneNestedFunctionArguments( + ColumnNode & column_node, + FunctionNode & function_node, + const ExpressionUsage & expr_usage, + const ContextPtr & context) +{ + auto & nested_args = function_node.getArguments().getNodes(); + const auto & subcolumn_names = expr_usage.nested_subcolumn_names; + size_t num_subcolumns = subcolumn_names.size(); + + /// Find which indices to keep. + std::vector indices_to_keep; + for (size_t i = 0; i < num_subcolumns; ++i) + { + if (expr_usage.used_subcolumns.contains(subcolumn_names[i])) + indices_to_keep.push_back(i); + } + + /// Nothing to prune. + if (indices_to_keep.size() == num_subcolumns) + return; + + /// Keep at least one subcolumn so the expression remains valid. + if (indices_to_keep.empty()) + indices_to_keep.push_back(0); + + /// Build pruned names array and arguments. + Array pruned_names_array; + QueryTreeNodes pruned_args; + pruned_names_array.reserve(indices_to_keep.size()); + pruned_args.reserve(indices_to_keep.size() + 1); + + for (size_t idx : indices_to_keep) + pruned_names_array.push_back(subcolumn_names[idx]); + + auto pruned_names_type = std::make_shared(std::make_shared()); + pruned_args.push_back(std::make_shared(std::move(pruned_names_array), std::move(pruned_names_type))); + + for (size_t idx : indices_to_keep) + pruned_args.push_back(nested_args[idx + 1]); /// +1: first arg is the names array. + + nested_args = std::move(pruned_args); + + /// Re-resolve the function to update its return type. + auto nested_function = FunctionFactory::instance().get("nested", context); + function_node.resolveAsFunction(nested_function->build(function_node.getArgumentColumns())); + + /// Update the ARRAY JOIN column node's type to match the new result. + auto new_result_type = function_node.getResultType(); + auto new_column_type = assert_cast(*new_result_type).getNestedType(); + column_node.setColumnType(std::move(new_column_type)); +} + +/// Visitor that updates reference ColumnNode types and re-resolves tupleElement functions +/// after nested() arguments have been pruned. +class UpdateArrayJoinReferenceTypesVisitor : public InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + + UpdateArrayJoinReferenceTypesVisitor( + ContextPtr context_, + const UpdatedTypeMap & updated_types_, + const ArrayJoinNodeSet & tracked_nodes_) + : Base(std::move(context_)) + , updated_types(updated_types_) + , tracked_nodes(tracked_nodes_) + { + } + + void enterImpl(const QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + if (!function_node || function_node->getFunctionName() != "tupleElement") + return; + + const auto & arguments = function_node->getArguments().getNodes(); + if (arguments.size() < 2) + return; + + auto * column_node = arguments[0]->as(); + if (!column_node) + return; + + auto source = column_node->getColumnSourceOrNull(); + if (!source || !tracked_nodes.contains(source.get())) + return; + + auto node_it = updated_types.find(source.get()); + if (node_it == updated_types.end()) + return; + + auto type_it = node_it->second.find(column_node->getColumnName()); + if (type_it == node_it->second.end()) + return; + + const auto & new_type = type_it->second; + if (column_node->getColumnType()->equals(*new_type)) + return; + + column_node->setColumnType(new_type); + + auto tuple_element_function = FunctionFactory::instance().get("tupleElement", getContext()); + function_node->resolveAsFunction(tuple_element_function->build(function_node->getArgumentColumns())); + } + +private: + const UpdatedTypeMap & updated_types; + const ArrayJoinNodeSet & tracked_nodes; +}; + +} + +void PruneArrayJoinColumnsPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) +{ + auto * top_query_node = query_tree_node->as(); + if (!top_query_node) + return; + + const auto & settings = context->getSettingsRef(); + if (settings[Setting::enable_unaligned_array_join]) + return; + + /// Step 1: Find all ARRAY JOIN nodes and build the usage map. + ArrayJoinUsageMap usage_map; + ArrayJoinNodeSet tracked_nodes; + + auto table_expressions = extractTableExpressions(top_query_node->getJoinTree(), /*add_array_join=*/ true); + for (const auto & table_expr : table_expressions) + { + auto * array_join_node = table_expr->as(); + if (!array_join_node) + continue; + + auto & expressions_usage = usage_map[table_expr.get()]; + + for (const auto & join_expr : array_join_node->getJoinExpressions().getNodes()) + { + auto * column_node = join_expr->as(); + if (!column_node || !column_node->hasExpression()) + continue; + + ExpressionUsage expr_usage; + + auto * function_node = column_node->getExpression()->as(); + if (function_node && function_node->getFunctionName() == "nested") + { + const auto & args = function_node->getArguments().getNodes(); + if (args.size() >= 2) + { + if (auto * names_constant = args[0]->as()) + { + const auto & names_array = names_constant->getValue().safeGet(); + for (const auto & name : names_array) + expr_usage.nested_subcolumn_names.push_back(name.safeGet()); + } + } + } + + expressions_usage[column_node->getColumnName()] = std::move(expr_usage); + } + + if (!expressions_usage.empty()) + tracked_nodes.insert(table_expr.get()); + } + + if (tracked_nodes.empty()) + return; + + /// Step 2: Mark used expressions and subcolumns. + MarkUsedArrayJoinColumnsVisitor visitor(context, usage_map, tracked_nodes); + visitor.visit(query_tree_node); + + /// Step 3: Prune. + UpdatedTypeMap updated_types; + + for (auto & [node_ptr, expressions_usage] : usage_map) + { + /// Find the ArrayJoinNode among our table_expressions. + ArrayJoinNode * array_join_node = nullptr; + for (const auto & te : table_expressions) + { + if (te.get() == node_ptr) + { + array_join_node = te->as(); + break; + } + } + if (!array_join_node) + continue; + + auto & join_expressions = array_join_node->getJoinExpressions().getNodes(); + + /// 3a: Remove entire unused ARRAY JOIN expressions. + { + QueryTreeNodes kept; + kept.reserve(join_expressions.size()); + + for (auto & join_expr : join_expressions) + { + auto * column_node = join_expr->as(); + if (!column_node) + { + kept.push_back(std::move(join_expr)); + continue; + } + + auto expr_it = expressions_usage.find(column_node->getColumnName()); + if (expr_it == expressions_usage.end() || expr_it->second.isUsed()) + kept.push_back(std::move(join_expr)); + } + + /// Keep at least one expression to preserve row multiplication. + if (kept.empty() && !join_expressions.empty()) + kept.push_back(std::move(join_expressions[0])); + + join_expressions = std::move(kept); + } + + /// 3b: Prune unused nested() subcolumn arguments. + for (auto & join_expr : join_expressions) + { + auto * column_node = join_expr->as(); + if (!column_node || !column_node->hasExpression()) + continue; + + auto expr_it = expressions_usage.find(column_node->getColumnName()); + if (expr_it == expressions_usage.end()) + continue; + + auto & expr_usage = expr_it->second; + if (expr_usage.fully_used || !expr_usage.hasNested()) + continue; + + auto * function_node = column_node->getExpression()->as(); + if (!function_node) + continue; + + pruneNestedFunctionArguments(*column_node, *function_node, expr_usage, context); + } + + /// Collect post-pruning types for step 3c. + auto & type_map = updated_types[node_ptr]; + for (const auto & join_expr : join_expressions) + { + auto * col_node = join_expr->as(); + if (!col_node) + continue; + type_map[col_node->getColumnName()] = col_node->getColumnType(); + } + } + + /// 3c: Update types of reference ColumnNodes and re-resolve tupleElement functions. + UpdateArrayJoinReferenceTypesVisitor type_updater(context, updated_types, tracked_nodes); + type_updater.visit(query_tree_node); +} + +} diff --git a/src/Analyzer/Passes/PruneArrayJoinColumnsPass.h b/src/Analyzer/Passes/PruneArrayJoinColumnsPass.h new file mode 100644 index 000000000000..f265f2743346 --- /dev/null +++ b/src/Analyzer/Passes/PruneArrayJoinColumnsPass.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +namespace DB +{ + +/** Prune unused ARRAY JOIN expressions and unused subcolumns from `nested()` functions. + * + * 1. If ARRAY JOIN has multiple expressions and some are not referenced + * anywhere in the query, remove the unused expressions. + * + * 2. When a Nested column is used in ARRAY JOIN, the analyzer creates a `nested()` + * function with ALL subcolumns as arguments. This pass removes arguments that + * are not referenced, so that only the needed subcolumns are read from storage. + * + * Example 1: SELECT b FROM t ARRAY JOIN a, b => ARRAY JOIN b + * + * Example 2: Table has n.a, n.b, n.c. + * SELECT n.a FROM t ARRAY JOIN n + * Before: ARRAY JOIN nested(['a','b','c'], n.a, n.b, n.c) AS n + * After: ARRAY JOIN nested(['a'], n.a) AS n + */ +class PruneArrayJoinColumnsPass final : public IQueryTreePass +{ +public: + String getName() override { return "PruneArrayJoinColumns"; } + + String getDescription() override { return "Prune unused ARRAY JOIN expressions and nested() subcolumns"; } + + void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; +}; + +} diff --git a/src/Analyzer/Passes/RegexpFunctionRewritePass.cpp b/src/Analyzer/Passes/RegexpFunctionRewritePass.cpp index 860c3fb71d29..e898105c69f3 100644 --- a/src/Analyzer/Passes/RegexpFunctionRewritePass.cpp +++ b/src/Analyzer/Passes/RegexpFunctionRewritePass.cpp @@ -1,13 +1,12 @@ #include -#include -#include #include #include #include #include #include #include +#include #include #include #include @@ -35,28 +34,13 @@ class RegexpFunctionRewriteVisitor : public InDepthQueryTreeVisitorWithContextas(); - if (!function_node || !function_node->isOrdinaryFunction() || !isString(function_node->getResultType())) + if (!function_node || !function_node->isOrdinaryFunction() || !isString(removeNullable(function_node->getResultType()))) return; /// If a regular expression without alternatives starts with ^ or ends with an unescaped $, rewrite /// replaceRegexpAll with replaceRegexpOne. if (function_node->getFunctionName() == "replaceRegexpAll" || Poco::toLower(function_node->getFunctionName()) == "regexp_replace") - { - if (!handleReplaceRegexpAll(*function_node)) - return; - - /// After optimization, function_node might now be "replaceRegexpOne", so continue processing - } - - /// If a replaceRegexpOne function has a regexp that matches entire haystack, and a replacement of nothing other - /// than \1 and some subpatterns in the regexp, or \0 and no subpatterns in the regexp, rewrite it with extract. - if (function_node->getFunctionName() == "replaceRegexpOne") - { - if (!handleReplaceRegexpOne(*function_node)) - return; - - /// After optimization, function_node might now be "extract", so continue processing - } + handleReplaceRegexpAll(*function_node); /// If an extract function has a regexp with some subpatterns and the regexp starts with ^.* or ending with an /// unescaped .*$, remove this prefix and/or suffix. @@ -114,62 +98,6 @@ class RegexpFunctionRewriteVisitor : public InDepthQueryTreeVisitorWithContextas(); - if (!constant_node) - return false; - - if (auto constant_type = constant_node->getResultType(); !isString(constant_type)) - return false; - - String replacement = constant_node->getValue().safeGet(); - bool replacement_zero = replacement == "\\0"; - bool replacement_one = replacement == "\\1"; - if (!replacement_zero && !replacement_one) - return false; - - const auto * regexp_node = function_node_arguments_nodes[1]->as(); - if (!regexp_node) - return false; - - if (auto regexp_type = regexp_node->getResultType(); !isString(regexp_type)) - return false; - - String regexp = regexp_node->getValue().safeGet(); - - /// Currently only look for ^...$ patterns without alternatives. - bool starts_with_caret = regexp.front() == '^'; - if (!starts_with_caret) - return false; - - bool ends_with_unescaped_dollar = false; - if (!regexp.empty() && regexp.back() == '$') - ends_with_unescaped_dollar = isUnescaped(regexp, regexp.size() - 1); - - if (!ends_with_unescaped_dollar) - return false; - - /// Analyze the regular expression to detect presence of alternatives (e.g., 'a|b'). If any alternatives are - /// found, return false to indicate the regexp is not suitable for optimization. - RegexpAnalysisResult result = OptimizedRegularExpression::analyze(regexp); - if (!result.alternatives.empty()) - return false; - - if ((replacement_one && result.has_capture) || (replacement_zero && !result.has_capture)) - { - function_node_arguments_nodes.resize(2); - resolveOrdinaryFunctionNodeByName(function_node, "extract", getContext()); - return true; - } - - return false; - } - void handleExtract(FunctionNode & function_node) { auto & function_node_arguments_nodes = function_node.getArguments().getNodes(); diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index a818ad348020..c510d1e84270 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -260,6 +261,7 @@ void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze) /// This pass should be run for the secondary queries /// to ensure that the only required columns are read from VIEWs on the shards. manager.addPass(std::make_unique()); + manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index 216d5a90bf24..745f9c974ab1 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -997,8 +997,17 @@ IdentifierResolveResult IdentifierResolver::tryResolveIdentifierFromJoin(const I { auto & resolved_column = resolved_identifier_candidate->as(); auto using_column_node_it = using_column_name_to_column_node.find(resolved_column.getColumnName()); + if (using_column_node_it == using_column_name_to_column_node.end()) + return; + + const auto & using_column_list = using_column_node_it->second->as().getExpressionOrThrow()->as(); + auto matches_using_column = [&](const auto & node) { return node->isEqual(*resolved_identifier_candidate); }; + if (std::ranges::none_of(using_column_list.getNodes(), matches_using_column)) + return; + if (using_column_node_it != using_column_name_to_column_node.end() && !using_column_node_it->second->getColumnType()->equals(*resolved_column.getColumnType())) + { // std::cerr << "... fixing type for " << resolved_column.dumpTree() << std::endl; auto resolved_column_clone = std::static_pointer_cast(resolved_column.clone()); diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 26d34be495f3..00d004e33426 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -5385,56 +5385,88 @@ void QueryAnalyzer::resolveCrossJoin(QueryTreeNodePtr & cross_join_node, Identif } } -static NameSet getColumnsFromTableExpression(const QueryTreeNodePtr & table_expression) +static NameSet getColumnsFromTableExpression(const QueryTreeNodePtr & root_table_expression) { NameSet existing_columns; - switch (table_expression->getNodeType()) + std::stack nodes_to_process; + nodes_to_process.push(root_table_expression.get()); + + while (!nodes_to_process.empty()) { - case QueryTreeNodeType::TABLE: { - const auto * table_node = table_expression->as(); - chassert(table_node); + const auto * table_expression = nodes_to_process.top(); + nodes_to_process.pop(); - auto get_column_options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withSubcolumns(); - for (const auto & column : table_node->getStorageSnapshot()->getColumns(get_column_options)) - existing_columns.insert(column.name); + switch (table_expression->getNodeType()) + { + case QueryTreeNodeType::TABLE: + { + const auto * table_node = table_expression->as(); + chassert(table_node); - return existing_columns; - } - case QueryTreeNodeType::TABLE_FUNCTION: { - const auto * table_function_node = table_expression->as(); - chassert(table_function_node); + auto get_column_options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withSubcolumns(); + for (const auto & column : table_node->getStorageSnapshot()->getColumns(get_column_options)) + existing_columns.insert(column.name); - auto get_column_options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withSubcolumns(); - for (const auto & column : table_function_node->getStorageSnapshot()->getColumns(get_column_options)) - existing_columns.insert(column.name); + break; + } + case QueryTreeNodeType::TABLE_FUNCTION: + { + const auto * table_function_node = table_expression->as(); + chassert(table_function_node); - return existing_columns; - } - case QueryTreeNodeType::QUERY: { - const auto * query_node = table_expression->as(); - chassert(query_node); + auto get_column_options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withSubcolumns(); + for (const auto & column : table_function_node->getStorageSnapshot()->getColumns(get_column_options)) + existing_columns.insert(column.name); - for (const auto & column : query_node->getProjectionColumns()) - existing_columns.insert(column.name); + break; + } + case QueryTreeNodeType::QUERY: + { + const auto * query_node = table_expression->as(); + chassert(query_node); - return existing_columns; - } - case QueryTreeNodeType::UNION: { - const auto * union_node = table_expression->as(); - chassert(union_node); + for (const auto & column : query_node->getProjectionColumns()) + existing_columns.insert(column.name); - for (const auto & column : union_node->computeProjectionColumns()) - existing_columns.insert(column.name); + break; + } + case QueryTreeNodeType::UNION: + { + const auto * union_node = table_expression->as(); + chassert(union_node); - return existing_columns; + for (const auto & column : union_node->computeProjectionColumns()) + existing_columns.insert(column.name); + break; + } + case QueryTreeNodeType::JOIN: + { + const auto * join_node = table_expression->as(); + chassert(join_node); + + nodes_to_process.push(join_node->getLeftTableExpression().get()); + nodes_to_process.push(join_node->getRightTableExpression().get()); + break; + } + case QueryTreeNodeType::CROSS_JOIN: + { + const auto * cross_join_node = table_expression->as(); + chassert(cross_join_node); + for (const auto & table_expr : cross_join_node->getTableExpressions()) + nodes_to_process.push(table_expr.get()); + break; + } + default: + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected TableNode, TableFunctionNode, QueryNode or UnionNode, got {}: {}", + table_expression->getNodeTypeName(), + table_expression->formatASTForErrorMessage()); + } } - default: - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Expected TableNode, TableFunctionNode, QueryNode or UnionNode, got {}: {}", - table_expression->getNodeTypeName(), - table_expression->formatASTForErrorMessage()); } + return existing_columns; } /// Resolve join node in scope @@ -5528,8 +5560,14 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS while (existing_columns.contains(column_name_type.name)) column_name_type.name = "_" + column_name_type.name; + auto [expression_source, is_single_source] = getExpressionSource(resolved_nodes.front()); + /// Do not support `SELECT t1.a + t2.a AS id ... USING id` + if (!is_single_source) + return nullptr; + /// Create ColumnNode with expression from parent projection - return std::make_shared(std::move(column_name_type), resolved_nodes.front(), left_table_expression); + return std::make_shared(std::move(column_name_type), resolved_nodes.front(), + expression_source ? expression_source : left_table_expression); } } } diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index a873091f7960..7c2142c90f4c 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -104,7 +104,7 @@ bool isStorageUsedInTree(const StoragePtr & storage, const IQueryTreeNode * root if (table_node || table_function_node) { const auto & table_storage = table_node ? table_node->getStorage() : table_function_node->getStorage(); - if (table_storage->getStorageID() == storage->getStorageID()) + if (table_storage && table_storage->getStorageID() == storage->getStorageID()) return true; } @@ -975,12 +975,7 @@ void resolveAggregateFunctionNodeByName(FunctionNode & function_node, const Stri function_node.resolveAsAggregateFunction(std::move(aggregate_function)); } -/** Returns: - * {_, false} - multiple sources - * {nullptr, true} - no sources (for constants) - * {source, true} - single source - */ -std::pair getExpressionSourceImpl(const QueryTreeNodePtr & node) +std::pair getExpressionSource(const QueryTreeNodePtr & node) { if (const auto * column = node->as()) { @@ -996,7 +991,7 @@ std::pair getExpressionSourceImpl(const QueryTreeNodePtr const auto & args = func->getArguments().getNodes(); for (const auto & arg : args) { - auto [arg_source, is_ok] = getExpressionSourceImpl(arg); + auto [arg_source, is_ok] = getExpressionSource(arg); if (!is_ok) return {nullptr, false}; @@ -1015,14 +1010,6 @@ std::pair getExpressionSourceImpl(const QueryTreeNodePtr return {nullptr, false}; } -QueryTreeNodePtr getExpressionSource(const QueryTreeNodePtr & node) -{ - auto [source, is_ok] = getExpressionSourceImpl(node); - if (!is_ok) - return nullptr; - return source; -} - /** There are no limits on the maximum size of the result for the subquery. * Since the result of the query is not the result of the entire query. */ diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index 215bc816cccc..9a19af2b4e0d 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -157,8 +157,10 @@ void resolveOrdinaryFunctionNodeByName(FunctionNode & function_node, const Strin /// Arguments and parameters are taken from the node. void resolveAggregateFunctionNodeByName(FunctionNode & function_node, const String & function_name); -/// Checks that node has only one source and returns it -QueryTreeNodePtr getExpressionSource(const QueryTreeNodePtr & node); +/// Returns single source of expression node. +/// First element of pair is source node, can be nullptr if there are no sources or multiple sources. +/// Second element of pair is true if there is at most one source, false if there are multiple sources. +std::pair getExpressionSource(const QueryTreeNodePtr & node); /// Update mutable context for subquery execution void updateContextForSubqueryExecution(ContextMutablePtr & mutable_context); diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 8009f0038843..bed961f7c10f 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -26,6 +26,8 @@ #include #include +#include + namespace ProfileEvents { @@ -54,8 +56,11 @@ namespace ErrorCodes extern const int CANNOT_RESTORE_TO_NONENCRYPTED_DISK; extern const int FAILED_TO_SYNC_BACKUP_OR_RESTORE; extern const int LOGICAL_ERROR; + extern const int INSECURE_PATH; } +namespace fs = std::filesystem; + namespace { const int INITIAL_BACKUP_VERSION = 1; @@ -89,6 +94,43 @@ namespace return path.substr(1); return path; } + + /// Validate that a file name from a backup does not contain path traversal sequences. + /// This prevents a corrupted or tampered backup from accessing files outside the intended directories during restore. + void validateFileNameFromBackup(const String & file_name, const String & field_name, const String & backup_name_for_logging) + { + fs::path path(file_name); + + /// Reject absolute or rooted paths. + if (path.is_absolute() || path.has_root_name() || path.has_root_directory()) + throw Exception( + ErrorCodes::INSECURE_PATH, + "Backup {}: <{}> {} is an absolute path, which is not allowed", + backup_name_for_logging, + field_name, + quoteString(file_name)); + + /// Normalize the path and check that it does not escape the backup root. + auto normalized = path.lexically_normal(); + + /// Reject empty or degenerate paths. + if (normalized.empty() || normalized == fs::path(".")) + throw Exception( + ErrorCodes::BACKUP_DAMAGED, + "Backup {}: <{}> {} is empty or invalid", + backup_name_for_logging, + field_name, + quoteString(file_name)); + + /// After normalization, a path that escapes the root starts with "..". + if (*normalized.begin() == "..") + throw Exception( + ErrorCodes::INSECURE_PATH, + "Backup {}: <{}> {} resolves to a path outside the backup, which is not allowed", + backup_name_for_logging, + field_name, + quoteString(file_name)); + } } @@ -501,6 +543,7 @@ void BackupImpl::readBackupMetadata() const Poco::XML::Node * file_config = child; BackupFileInfo info; info.file_name = getString(file_config, "name"); + validateFileNameFromBackup(info.file_name, "name", backup_name_for_logging); info.object_key = getString(file_config, "object_key", ""); info.size = getUInt64(file_config, "size"); if (info.size) @@ -532,6 +575,8 @@ void BackupImpl::readBackupMetadata() if (info.size > info.base_size) { info.data_file_name = getString(file_config, "data_file", info.file_name); + if (info.data_file_name != info.file_name) + validateFileNameFromBackup(info.data_file_name, "data_file", backup_name_for_logging); } info.encrypted_by_disk = getBool(file_config, "encrypted_by_disk", false); } diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index ba0e6718d6f1..48b1fcd96f50 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -833,6 +833,11 @@ void RestorerFromBackup::createDatabase(const String & database_name) const auto create_query_context = Context::createCopy(query_context); create_query_context->setSetting("allow_deprecated_database_ordinary", 1); + /// We shouldn't use the progress callback copied from the `query_context` because it was set in a protocol handler (e.g. HTTPHandler) + /// for the "RESTORE ASYNC" query which could have already finished (the restore process is working in the background). + /// TODO: Get rid of using `query_context` in class RestorerFromBackup. + create_query_context->setProgressCallback(nullptr); + #if CLICKHOUSE_CLOUD if (shared_catalog && SharedDatabaseCatalog::instance().shouldRestoreDatabase(create_database_query)) { @@ -1075,6 +1080,11 @@ void RestorerFromBackup::createTable(const QualifiedTableName & table_name) create_query_context->setUnderRestore(true); + /// We shouldn't use the progress callback copied from the `query_context` because it was set in a protocol handler (e.g. HTTPHandler) + /// for the "RESTORE ASYNC" query which could have already finished (the restore process is working in the background). + /// TODO: Get rid of using `query_context` in class RestorerFromBackup. + create_query_context->setProgressCallback(nullptr); + /// Execute CREATE TABLE query (we call IDatabase::createTableRestoredFromBackup() to allow the database to do some /// database-specific things). database->createTableRestoredFromBackup( diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 8fe062107fe7..6ced3cd4114c 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -3418,7 +3418,10 @@ void ClientBase::runInteractive() initQueryIdFormats(); #if USE_CLIENT_AI - initAIProvider(); + /// AI SQL generation is disabled for the embedded client (SSH and WebSocket protocols) + /// because it accesses the environment (API keys) which could be a security concern. + if (!isEmbeeddedClient()) + initAIProvider(); #endif /// Initialize DateLUT here to avoid counting time spent here as query execution time. @@ -3669,7 +3672,8 @@ void ClientBase::runNonInteractive() initQueryIdFormats(); #if USE_CLIENT_AI - initAIProvider(); + if (!isEmbeeddedClient()) + initAIProvider(); #endif if (!buzz_house && !queries_files.empty()) diff --git a/src/Columns/ColumnDynamic.h b/src/Columns/ColumnDynamic.h index 4e1bbe3232f1..710be4059b83 100644 --- a/src/Columns/ColumnDynamic.h +++ b/src/Columns/ColumnDynamic.h @@ -212,7 +212,7 @@ class ColumnDynamic final : public COWHelper, Colum ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override { - return create(variant_column_ptr->filter(filt, result_size_hint), variant_info, max_dynamic_types, global_max_dynamic_types); + return create(variant_column_ptr->filter(filt, result_size_hint), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } void expand(const Filter & mask, bool inverted) override @@ -222,17 +222,17 @@ class ColumnDynamic final : public COWHelper, Colum ColumnPtr permute(const Permutation & perm, size_t limit) const override { - return create(variant_column_ptr->permute(perm, limit), variant_info, max_dynamic_types, global_max_dynamic_types); + return create(variant_column_ptr->permute(perm, limit), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } ColumnPtr index(const IColumn & indexes, size_t limit) const override { - return create(variant_column_ptr->index(indexes, limit), variant_info, max_dynamic_types, global_max_dynamic_types); + return create(variant_column_ptr->index(indexes, limit), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } ColumnPtr replicate(const Offsets & replicate_offsets) const override { - return create(variant_column_ptr->replicate(replicate_offsets), variant_info, max_dynamic_types, global_max_dynamic_types); + return create(variant_column_ptr->replicate(replicate_offsets), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override @@ -241,7 +241,7 @@ class ColumnDynamic final : public COWHelper, Colum MutableColumns scattered_columns; scattered_columns.reserve(num_columns); for (auto & scattered_variant_column : scattered_variant_columns) - scattered_columns.emplace_back(create(std::move(scattered_variant_column), variant_info, max_dynamic_types, global_max_dynamic_types)); + scattered_columns.emplace_back(create(std::move(scattered_variant_column), variant_info, max_dynamic_types, global_max_dynamic_types, statistics)); return scattered_columns; } diff --git a/src/Columns/ColumnFunction.cpp b/src/Columns/ColumnFunction.cpp index 5e50ee2a954a..1719bc09c43f 100644 --- a/src/Columns/ColumnFunction.cpp +++ b/src/Columns/ColumnFunction.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -291,6 +292,28 @@ size_t ColumnFunction::allocatedBytes() const return total_size; } +void ColumnFunction::updateHashWithValue(size_t n, SipHash & hash) const +{ + hash.update(function->getName()); + for (const auto & column : captured_columns) + column.column->updateHashWithValue(n, hash); +} + +WeakHash32 ColumnFunction::getWeakHash32() const +{ + WeakHash32 hash(elements_size); + for (const auto & column : captured_columns) + hash.update(column.column->getWeakHash32()); + return hash; +} + +void ColumnFunction::updateHashFast(SipHash & hash) const +{ + hash.update(function->getName()); + for (const auto & column : captured_columns) + column.column->updateHashFast(hash); +} + void ColumnFunction::appendArguments(const ColumnsWithTypeAndName & columns) { auto args = function->getArgumentTypes().size(); diff --git a/src/Columns/ColumnFunction.h b/src/Columns/ColumnFunction.h index 0e3335332dee..0f4316caea7d 100644 --- a/src/Columns/ColumnFunction.h +++ b/src/Columns/ColumnFunction.h @@ -121,20 +121,9 @@ class ColumnFunction final : public COWHelper, Col throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot skip serialized {}", getName()); } - void updateHashWithValue(size_t, SipHash &) const override - { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "updateHashWithValue is not implemented for {}", getName()); - } - - WeakHash32 getWeakHash32() const override - { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getWeakHash32 is not implemented for {}", getName()); - } - - void updateHashFast(SipHash &) const override - { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "updateHashFast is not implemented for {}", getName()); - } + void updateHashWithValue(size_t n, SipHash & hash) const override; + WeakHash32 getWeakHash32() const override; + void updateHashFast(SipHash & hash) const override; void popBack(size_t) override { diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 7ca7701a1e83..7e6a1e356c19 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -1149,7 +1149,7 @@ ColumnPtr ColumnObject::filter(const Filter & filt, ssize_t result_size_hint) co filtered_dynamic_paths[path] = column->filter(filt, result_size_hint); auto filtered_shared_data = shared_data->filter(filt, result_size_hint); - return ColumnObject::create(filtered_typed_paths, filtered_dynamic_paths, filtered_shared_data, max_dynamic_paths, global_max_dynamic_paths, max_dynamic_types); + return ColumnObject::create(filtered_typed_paths, filtered_dynamic_paths, filtered_shared_data, max_dynamic_paths, global_max_dynamic_paths, max_dynamic_types, statistics); } void ColumnObject::expand(const Filter & mask, bool inverted) @@ -1174,7 +1174,7 @@ ColumnPtr ColumnObject::permute(const Permutation & perm, size_t limit) const permuted_dynamic_paths[path] = column->permute(perm, limit); auto permuted_shared_data = shared_data->permute(perm, limit); - return ColumnObject::create(permuted_typed_paths, permuted_dynamic_paths, permuted_shared_data, max_dynamic_paths, global_max_dynamic_paths, max_dynamic_types); + return ColumnObject::create(permuted_typed_paths, permuted_dynamic_paths, permuted_shared_data, max_dynamic_paths, global_max_dynamic_paths, max_dynamic_types, statistics); } ColumnPtr ColumnObject::index(const IColumn & indexes, size_t limit) const @@ -1190,7 +1190,7 @@ ColumnPtr ColumnObject::index(const IColumn & indexes, size_t limit) const indexed_dynamic_paths[path] = column->index(indexes, limit); auto indexed_shared_data = shared_data->index(indexes, limit); - return ColumnObject::create(indexed_typed_paths, indexed_dynamic_paths, indexed_shared_data, max_dynamic_paths, global_max_dynamic_paths, max_dynamic_types); + return ColumnObject::create(indexed_typed_paths, indexed_dynamic_paths, indexed_shared_data, max_dynamic_paths, global_max_dynamic_paths, max_dynamic_types, statistics); } ColumnPtr ColumnObject::replicate(const Offsets & replicate_offsets) const @@ -1206,7 +1206,7 @@ ColumnPtr ColumnObject::replicate(const Offsets & replicate_offsets) const replicated_dynamic_paths[path] = column->replicate(replicate_offsets); auto replicated_shared_data = shared_data->replicate(replicate_offsets); - return ColumnObject::create(replicated_typed_paths, replicated_dynamic_paths, replicated_shared_data, max_dynamic_paths, global_max_dynamic_paths, max_dynamic_types); + return ColumnObject::create(replicated_typed_paths, replicated_dynamic_paths, replicated_shared_data, max_dynamic_paths, global_max_dynamic_paths, max_dynamic_types, statistics); } MutableColumns ColumnObject::scatter(ColumnIndex num_columns, const Selector & selector) const @@ -1237,7 +1237,7 @@ MutableColumns ColumnObject::scatter(ColumnIndex num_columns, const Selector & s MutableColumns result_columns; result_columns.reserve(num_columns); for (size_t i = 0; i != num_columns; ++i) - result_columns.emplace_back(ColumnObject::create(std::move(scattered_typed_paths[i]), std::move(scattered_dynamic_paths[i]), std::move(scattered_shared_data_columns[i]), max_dynamic_paths, global_max_dynamic_paths, max_dynamic_types)); + result_columns.emplace_back(ColumnObject::create(std::move(scattered_typed_paths[i]), std::move(scattered_dynamic_paths[i]), std::move(scattered_shared_data_columns[i]), max_dynamic_paths, global_max_dynamic_paths, max_dynamic_types, statistics)); return result_columns; } @@ -1505,16 +1505,26 @@ bool ColumnObject::isFinalized() const void ColumnObject::getExtremes(DB::Field & min, DB::Field & max) const { + min = Object(); + max = Object(); + if (empty()) + return; + + size_t min_idx = 0; + size_t max_idx = 0; + + size_t end = size(); + for (size_t i = 1; i < end; ++i) { - min = Object(); - max = Object(); - } - else - { - get(0, min); - get(0, max); + if (compareAt(i, min_idx, *this, /* nan_direction_hint = */ 1) < 0) + min_idx = i; + else if (compareAt(i, max_idx, *this, /* nan_direction_hint = */ -1) > 0) + max_idx = i; } + + get(min_idx, min); + get(max_idx, max); } void ColumnObject::prepareForSquashing(const std::vector & source_columns, size_t factor) diff --git a/src/Columns/ColumnVariant.cpp b/src/Columns/ColumnVariant.cpp index fc3f00915eb8..bd6dfbcde261 100644 --- a/src/Columns/ColumnVariant.cpp +++ b/src/Columns/ColumnVariant.cpp @@ -203,6 +203,8 @@ ColumnVariant::ColumnVariant(DB::MutableColumnPtr local_discriminators_, DB::Mut global_to_local_discriminators[local_to_global_discriminators[i]] = i; } } + + validateState(); } namespace @@ -915,7 +917,10 @@ ColumnPtr ColumnVariant::filter(const Filter & filt, ssize_t result_size_hint) c /// If we have only NULLs, just filter local_discriminators column. if (hasOnlyNulls()) { - Columns new_variants(variants.begin(), variants.end()); + Columns new_variants; + new_variants.reserve(variants.size()); + for (const auto & variant : variants) + new_variants.emplace_back(variant->cloneEmpty()); auto new_discriminators = local_discriminators->filter(filt, result_size_hint); /// In case of all NULL values offsets doesn't contain any useful values, just resize it. ColumnPtr new_offsets = offsets->cloneResized(new_discriminators->size()); @@ -1759,5 +1764,35 @@ void ColumnVariant::fixDynamicStructure() variant->fixDynamicStructure(); } +void ColumnVariant::validateState() const +{ + const auto & local_discriminators_data = getLocalDiscriminators(); + const auto & offsets_data = getOffsets(); + if (local_discriminators_data.size() != offsets_data.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of discriminators and offsets should be equal, but {} and {} were given", local_discriminators_data.size(), offsets_data.size()); + + std::vector actual_variant_sizes(variants.size()); + for (size_t i = 0; i != variants.size(); ++i) + actual_variant_sizes[i] = variants[i]->size(); + + std::vector expected_variant_sizes(variants.size(), 0); + for (size_t i = 0; i != local_discriminators_data.size(); ++i) + { + auto local_discr = local_discriminators_data[i]; + if (local_discr != NULL_DISCRIMINATOR) + { + ++expected_variant_sizes[local_discr]; + if (offsets_data[i] >= actual_variant_sizes[local_discr]) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Offset at position {} is {}, but variant {} ({}) has size {}", i, offsets_data[i], static_cast(local_discr), variants[local_discr]->getName(), variants[local_discr]->size()); + } + } + + for (size_t i = 0; i != variants.size(); ++i) + { + if (variants[i]->size() != expected_variant_sizes[i]) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Variant {} ({}) has size {}, but expected {}", i, variants[i]->getName(), variants[i]->size(), expected_variant_sizes[i]); + } +} + } diff --git a/src/Columns/ColumnVariant.h b/src/Columns/ColumnVariant.h index a1bdbe7d4039..c537397d46cb 100644 --- a/src/Columns/ColumnVariant.h +++ b/src/Columns/ColumnVariant.h @@ -347,6 +347,8 @@ class ColumnVariant final : public COWHelper, Colum void takeDynamicStructureFromColumn(const ColumnPtr & source_column) override; void fixDynamicStructure() override; + void validateState() const; + private: void insertFromImpl(const IColumn & src_, size_t n, const std::vector * global_discriminators_mapping); void insertRangeFromImpl(const IColumn & src_, size_t start, size_t length, const std::vector * global_discriminators_mapping, const Discriminator * skip_discriminator); diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 476738471079..d467b17a2e3c 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -129,9 +129,11 @@ static struct InitFiu ONCE(database_iceberg_gcs) \ REGULAR(rmt_delay_execute_drop_range) \ REGULAR(rmt_delay_commit_part) \ + REGULAR(patch_parts_reverse_column_order) \ ONCE(smt_commit_exception_before_op) \ ONCE(backup_add_empty_memory_table) \ - REGULAR(refresh_task_stop_racing_for_running_refresh) + REGULAR(refresh_task_stop_racing_for_running_refresh) \ + REGULAR(wide_part_writer_fail_in_add_streams) namespace FailPoints diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index 241d6d218515..38adbce5367a 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -38,15 +38,13 @@ namespace ErrorCodes namespace { -using Metrics = std::map; - /// Format is /// kernel 5 /// rss 15 /// [...] -Metrics readAllMetricsFromStatFile(ReadBufferFromFile & buf) +std::map readAllMetricsFromStatFile(ReadBufferFromFile & buf) { - Metrics metrics; + std::map metrics; while (!buf.eof()) { std::string current_key; @@ -64,10 +62,21 @@ Metrics readAllMetricsFromStatFile(ReadBufferFromFile & buf) return metrics; } -uint64_t readMetricsFromStatFile(ReadBufferFromFile & buf, std::initializer_list keys, std::initializer_list optional_keys, bool * warnings_printed) +using Metrics = std::map; + +void readMetricsFromStatFile( + ReadBufferFromFile & buf, + Metrics & metrics, + std::initializer_list keys, + bool * warnings_printed) { - uint64_t sum = 0; - uint64_t found_mask = 0; + /// Zero out existing values; keeps map nodes allocated for reuse. + for (auto & [_, v] : metrics) + v = 0; + + /// Track which keys were actually seen in this pass. + uint64_t seen_mask = 0; + bool print_warnings = !*warnings_printed; while (!buf.eof()) { @@ -79,36 +88,42 @@ uint64_t readMetricsFromStatFile(ReadBufferFromFile & buf, std::initializer_list { std::string dummy; readStringUntilNewlineInto(dummy, buf); - buf.tryIgnore(1); /// skip EOL (if not EOF) + buf.tryIgnore(1); continue; } - if (print_warnings && (found_mask & (1l << (it - keys.begin())))) - { - *warnings_printed = true; - LOG_ERROR(getLogger("CgroupsReader"), "Duplicate key '{}' in '{}'", current_key, buf.getFileName()); - } - found_mask |= 1ll << (it - keys.begin()); - assertChar(' ', buf); uint64_t value = 0; readIntText(value, buf); - sum += value; - buf.tryIgnore(1); /// skip EOL (if not EOF) + buf.tryIgnore(1); + + uint64_t key_bit = 1ull << (it - keys.begin()); + if (seen_mask & key_bit) + { + if (print_warnings) + { + *warnings_printed = true; + LOG_ERROR(getLogger("CgroupsReader"), "Duplicate key '{}' in '{}'", current_key, buf.getFileName()); + } + } + seen_mask |= key_bit; + + /// Use the string_view from keys (string literals) as map key. + metrics[*it] = value; } - /// Did we see all keys? - for (const auto * it = keys.begin(); it != keys.end(); ++it) + if (print_warnings) { - if (print_warnings - && !(found_mask & (1l << (it - keys.begin()))) - && std::find(optional_keys.begin(), optional_keys.end(), *it) == optional_keys.end()) + for (const auto * it = keys.begin(); it != keys.end(); ++it) { - *warnings_printed = true; - LOG_ERROR(getLogger("CgroupsReader"), "Cannot find '{}' in '{}'", *it, buf.getFileName()); + uint64_t key_bit = 1ull << (it - keys.begin()); + if (!(seen_mask & key_bit)) + { + *warnings_printed = true; + LOG_ERROR(getLogger("CgroupsReader"), "Cannot find '{}' in '{}'", *it, buf.getFileName()); + } } } - return sum; } struct CgroupsV1Reader : ICgroupsReader @@ -119,7 +134,9 @@ struct CgroupsV1Reader : ICgroupsReader { std::lock_guard lock(mutex); buf.rewind(); - return readMetricsFromStatFile(buf, {"rss"}, {}, &warnings_printed); + readMetricsFromStatFile(buf, metrics, {"rss"}, &warnings_printed); + auto it = metrics.find("rss"); + return it != metrics.end() ? it->second : 0; } std::string dumpAllStats() override @@ -132,6 +149,7 @@ struct CgroupsV1Reader : ICgroupsReader private: std::mutex mutex; ReadBufferFromFile buf TSA_GUARDED_BY(mutex); + Metrics metrics TSA_GUARDED_BY(mutex); bool warnings_printed TSA_GUARDED_BY(mutex) = false; }; @@ -143,7 +161,25 @@ struct CgroupsV2Reader : ICgroupsReader { std::lock_guard lock(mutex); stat_buf.rewind(); - return readMetricsFromStatFile(stat_buf, {"anon", "sock", "kernel"}, {"kernel"}, &warnings_printed); + readMetricsFromStatFile( + stat_buf, metrics, {"anon", "sock", "kernel", "slab_reclaimable"}, &warnings_printed); + + auto get = [](const Metrics & m, std::string_view key) -> uint64_t + { + auto it = m.find(key); + return it != m.end() ? it->second : 0; + }; + + /// anon + sock: actual process memory. + /// kernel - slab_reclaimable: non-reclaimable kernel memory (pagetables, kernel_stack, slab_unreclaimable). + /// slab_reclaimable is excluded because the kernel reclaims it synchronously under memory pressure + /// before invoking the OOM killer, so it should not count against the application's memory budget. + uint64_t usage = get(metrics, "anon") + get(metrics, "sock"); + uint64_t kernel = get(metrics, "kernel"); + uint64_t slab_reclaimable = get(metrics, "slab_reclaimable"); + if (kernel > slab_reclaimable) + usage += kernel - slab_reclaimable; + return usage; } std::string dumpAllStats() override @@ -156,6 +192,7 @@ struct CgroupsV2Reader : ICgroupsReader private: std::mutex mutex; ReadBufferFromFile stat_buf TSA_GUARDED_BY(mutex); + Metrics metrics TSA_GUARDED_BY(mutex); bool warnings_printed TSA_GUARDED_BY(mutex) = false; }; diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 6e98ca6b7f60..ddf2efa2b823 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -674,6 +674,7 @@ The server successfully detected this situation and will download merged part fr M(CachedReadBufferReadFromCacheHits, "Number of times the read from filesystem cache hit the cache.", ValueType::Number) \ M(CachedReadBufferReadFromCacheMisses, "Number of times the read from filesystem cache miss the cache.", ValueType::Number) \ M(CachedReadBufferReadFromSourceMicroseconds, "Time reading from filesystem cache source (from remote filesystem, etc)", ValueType::Microseconds) \ + M(CachedReadBufferWaitReadBufferMicroseconds, "Time spend waiting for internal read buffer (includes cache waiting)", ValueType::Microseconds) \ M(CachedReadBufferReadFromCacheMicroseconds, "Time reading from filesystem cache", ValueType::Microseconds) \ M(CachedReadBufferReadFromSourceBytes, "Bytes read from filesystem cache source (from remote fs, etc)", ValueType::Bytes) \ M(CachedReadBufferReadFromCacheBytes, "Bytes read from filesystem cache", ValueType::Bytes) \ @@ -744,12 +745,6 @@ The server successfully detected this situation and will download merged part fr M(ThreadpoolReaderSubmitLookupInCacheMicroseconds, "How much time we spent checking if content is cached", ValueType::Microseconds) \ M(AsynchronousReaderIgnoredBytes, "Number of bytes ignored during asynchronous reading", ValueType::Bytes) \ \ - M(FileSegmentWaitReadBufferMicroseconds, "Metric per file segment. Time spend waiting for internal read buffer (includes cache waiting)", ValueType::Microseconds) \ - M(FileSegmentReadMicroseconds, "Metric per file segment. Time spend reading from file", ValueType::Microseconds) \ - M(FileSegmentCacheWriteMicroseconds, "Metric per file segment. Time spend writing data to cache", ValueType::Microseconds) \ - M(FileSegmentPredownloadMicroseconds, "Metric per file segment. Time spent pre-downloading data to cache (pre-downloading - finishing file segment download (after someone who failed to do that) up to the point current thread was requested to do)", ValueType::Microseconds) \ - M(FileSegmentUsedBytes, "Metric per file segment. How many bytes were actually used from current file segment", ValueType::Bytes) \ - \ M(ReadBufferSeekCancelConnection, "Number of seeks which lead to new connection (s3, http)", ValueType::Number) \ \ M(SleepFunctionCalls, "Number of times a sleep function (sleep, sleepEachRow) has been called.", ValueType::Number) \ diff --git a/src/Common/Scheduler/CPULeaseAllocation.cpp b/src/Common/Scheduler/CPULeaseAllocation.cpp index 4fab4be028b8..7a761e1bbdd8 100644 --- a/src/Common/Scheduler/CPULeaseAllocation.cpp +++ b/src/Common/Scheduler/CPULeaseAllocation.cpp @@ -201,6 +201,13 @@ CPULeaseAllocation::CPULeaseAllocation(SlotCount max_threads_, ResourceLink mast , scheduled_increment(CurrentMetrics::ConcurrencyControlScheduled, 0) , lease_id(lease_counter.fetch_add(1, std::memory_order_relaxed)) { + // Capture query-level counters (ThreadGroup) that outlive all worker threads. + // Cannot use CurrentThread::getProfileEvents() in schedule() — it returns the calling + // thread's counters, which may be destroyed before the timer is flushed (UAF). + wait_thread_group = CurrentThread::getGroup(); + if (wait_thread_group) + wait_counters = &wait_thread_group->performance_counters; + std::unique_lock lock{mutex}; if (!schedule(lock)) grantImpl(lock); @@ -220,6 +227,7 @@ void CPULeaseAllocation::free() shutdown = true; acquirable.store(false, std::memory_order_relaxed); + wait_timer.reset(); // Wake up all preempted threads while (true) @@ -593,7 +601,7 @@ bool CPULeaseAllocation::schedule(std::unique_lock &) if (requests.enqueue(cost, requested_ns)) { scheduled_increment.add(); - wait_timer.emplace(CurrentThread::getProfileEvents().timer(ProfileEvents::ConcurrencyControlWaitMicroseconds)); + wait_timer.emplace(wait_counters->timer(ProfileEvents::ConcurrencyControlWaitMicroseconds)); LOG_EVENT(E); return true; } diff --git a/src/Common/Scheduler/CPULeaseAllocation.h b/src/Common/Scheduler/CPULeaseAllocation.h index 687b5709f724..bd2dd59b483f 100644 --- a/src/Common/Scheduler/CPULeaseAllocation.h +++ b/src/Common/Scheduler/CPULeaseAllocation.h @@ -20,6 +20,8 @@ namespace DB { +class ThreadGroup; + struct CPULeaseSettings { static constexpr ResourceCost default_quantum_ns = 10'000'000; @@ -314,6 +316,13 @@ class CPULeaseAllocation final : public ISlotAllocation /// Introspection CurrentMetrics::Increment acquired_increment; CurrentMetrics::Increment scheduled_increment; + /// Stable counters for wait_timer. We cannot use CurrentThread::getProfileEvents() in + /// schedule() because it returns the calling thread's counters, which may be destroyed + /// before the timer is flushed — storing a Timer with a dangling Counters& causes UAF. + /// The ThreadGroupPtr keeps the ThreadGroup (and its performance_counters) alive. + /// Declared before wait_timer so the owner outlives the timer during member destruction. + std::shared_ptr wait_thread_group; + ProfileEvents::Counters * wait_counters = &ProfileEvents::global_counters; std::optional wait_timer; const size_t lease_id; /// Unique identifier for this lease allocation, used for tracing static std::atomic lease_counter; diff --git a/src/Common/Scheduler/Nodes/FifoQueue.h b/src/Common/Scheduler/Nodes/FifoQueue.h index d7fa5e90bf4f..fb72b0a345b6 100644 --- a/src/Common/Scheduler/Nodes/FifoQueue.h +++ b/src/Common/Scheduler/Nodes/FifoQueue.h @@ -9,6 +9,7 @@ #include #include +#include namespace DB @@ -119,16 +120,25 @@ class FifoQueue final : public ISchedulerQueue void purgeQueue() override { - std::lock_guard lock(mutex); - is_not_usable = true; - while (!requests.empty()) + // Collect requests to fail while holding the lock, but call failed() outside the lock + // to avoid potential deadlock with CPULeaseAllocation::mutex (lock order inversion) + std::vector requests_to_fail; { - ResourceRequest * request = &requests.front(); - requests.pop_front(); - request->failed(std::make_exception_ptr( - Exception(ErrorCodes::INVALID_SCHEDULER_NODE, "Scheduler queue with resource request is about to be destructed"))); + std::lock_guard lock(mutex); + is_not_usable = true; + while (!requests.empty()) + { + ResourceRequest * request = &requests.front(); + requests.pop_front(); + requests_to_fail.push_back(request); + } + event_queue->cancelActivation(this); } - event_queue->cancelActivation(this); + // Now notify all collected requests about the failure without holding the mutex + auto exception = std::make_exception_ptr( + Exception(ErrorCodes::INVALID_SCHEDULER_NODE, "Scheduler queue with resource request is about to be destructed")); + for (ResourceRequest * request : requests_to_fail) + request->failed(exception); } bool isActive() override diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 55501ea1d568..20db52cca1f2 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -789,6 +789,15 @@ void ZooKeeper::sendThread() /// After we popped element from the queue, we must register callbacks (even in the case when expired == true right now), /// because they must not be lost (callbacks must be called because the user will wait for them). + if (info.watch) + info.request->has_watch = true; + + if (info.request->add_root_path) + info.request->addRootPath(args.chroot); + + /// Insert into operations AFTER mutating the request (has_watch, addRootPath) + /// to avoid a data race: receiveThread reads from operations concurrently, + /// and the request object is shared via shared_ptr. if (info.request->xid != close_xid) { CurrentMetrics::add(CurrentMetrics::ZooKeeperRequest); @@ -796,19 +805,11 @@ void ZooKeeper::sendThread() operations[info.request->xid] = info; } - if (info.watch) - { - info.request->has_watch = true; - } - if (requests_queue.isFinished()) { break; } - if (info.request->add_root_path) - info.request->addRootPath(args.chroot); - info.request->probably_sent = true; info.request->write(getWriteBuffer(), use_xid_64); flushWriteBuffer(); diff --git a/src/Common/tests/gtest_cgroups_reader.cpp b/src/Common/tests/gtest_cgroups_reader.cpp index d512aefe973d..06da5018bec9 100644 --- a/src/Common/tests/gtest_cgroups_reader.cpp +++ b/src/Common/tests/gtest_cgroups_reader.cpp @@ -160,7 +160,7 @@ TEST_P(CgroupsMemoryUsageObserverFixture, ReadMemoryUsageTest) ASSERT_EQ( reader->readMemoryUsage(), version == ICgroupsReader::CgroupsVersion::V1 ? /* rss from memory.stat */ 2232029184 - : /* anon+sock+kernel from memory.stat */ 11967193184); + : /* anon+sock+kernel-slab_reclaimable from memory.stat */ 10506210680); } @@ -177,4 +177,29 @@ INSTANTIATE_TEST_SUITE_P( CgroupsMemoryUsageObserverFixture, ::testing::Values(ICgroupsReader::CgroupsVersion::V1, ICgroupsReader::CgroupsVersion::V2)); + +/// Test cgroupv2 memory.stat without kernel/slab_reclaimable (older kernels). +/// Result should be just anon + sock. +TEST(CgroupsV2NoKernel, ReadMemoryUsageTest) +{ + std::string tmp_dir = "./test_cgroups_v2_no_kernel"; + fs::create_directories(tmp_dir); + + auto stat_file = WriteBufferFromFile(tmp_dir + "/memory.stat"); + std::string content = R"(anon 5000000000 +file 1000000000 +sock 1000 +inactive_anon 0 +active_anon 5000000000 +)"; + stat_file.write(content.data(), content.size()); + stat_file.finalize(); + stat_file.sync(); + + auto reader = ICgroupsReader::createCgroupsReader(ICgroupsReader::CgroupsVersion::V2, tmp_dir); + ASSERT_EQ(reader->readMemoryUsage(), /* anon + sock */ 5000001000); + + fs::remove_all(tmp_dir); +} + #endif diff --git a/src/Coordination/CoordinationSettings.cpp b/src/Coordination/CoordinationSettings.cpp index 172844e8ea5b..9268bf8497e5 100644 --- a/src/Coordination/CoordinationSettings.cpp +++ b/src/Coordination/CoordinationSettings.cpp @@ -46,7 +46,7 @@ namespace ErrorCodes DECLARE(UInt64, max_requests_batch_bytes_size, 100*1024, "Max size in bytes of batch of requests that can be sent to RAFT", 0) \ DECLARE(UInt64, max_requests_append_size, 100, "Max size of batch of requests that can be sent to replica in append request", 0) \ DECLARE(UInt64, max_flush_batch_size, 1000, "Max size of batch of requests that can be flushed together", 0) \ - DECLARE(UInt64, max_requests_quick_batch_size, 100, "Max size of batch of requests to try to get before proceeding with RAFT. Keeper will not wait for requests but take only requests that are already in queue" , 0) \ + DECLARE(UInt64, max_requests_quick_batch_size, 100, "Obsolete setting, does nothing." , SettingsTierType::OBSOLETE) \ DECLARE(Bool, quorum_reads, false, "Execute read requests as writes through whole RAFT consesus with similar speed", 0) \ DECLARE(Bool, force_sync, true, "Call fsync on each change in RAFT changelog", 0) \ DECLARE(Bool, compress_logs, false, "Write compressed coordination logs in ZSTD format", 0) \ diff --git a/src/Coordination/KeeperConstants.cpp b/src/Coordination/KeeperConstants.cpp index d15c142df6f0..5da770f67566 100644 --- a/src/Coordination/KeeperConstants.cpp +++ b/src/Coordination/KeeperConstants.cpp @@ -209,12 +209,6 @@ M(ThreadpoolReaderSubmitReadSynchronouslyMicroseconds) \ M(ThreadpoolReaderSubmitLookupInCacheMicroseconds) \ M(AsynchronousReaderIgnoredBytes) \ -\ - M(FileSegmentWaitReadBufferMicroseconds) \ - M(FileSegmentReadMicroseconds) \ - M(FileSegmentCacheWriteMicroseconds) \ - M(FileSegmentPredownloadMicroseconds) \ - M(FileSegmentUsedBytes) \ \ M(ReadBufferSeekCancelConnection) \ \ diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 13aebbcd4de0..9d18d525ff06 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -175,7 +175,7 @@ void KeeperDispatcher::requestThread() ReadableSize(total_memory_tracker.get()), ReadableSize(total_memory_tracker.getRSS()), request.request->getOpNum()); - addErrorResponses({request}, Coordination::Error::ZOUTOFMEMORY); + addErrorResponses({request}, Coordination::Error::ZOUTOFMEMORY, /*may_have_dependent_reads=*/ false); continue; } @@ -205,7 +205,7 @@ void KeeperDispatcher::requestThread() { const auto & last_request = current_batch.back(); std::lock_guard lock(read_request_queue_mutex); - read_request_queue[last_request.session_id][last_request.request->xid].push_back(request); + read_request_queue[{last_request.session_id, last_request.request->xid}].push_back(request); } else if (request.request->getOpNum() == Coordination::OpNum::Reconfig) { @@ -294,7 +294,24 @@ void KeeperDispatcher::requestThread() /// which always returns nullptr /// in that case we don't have to do manual wait because are already sure that the batch was committed when we get /// the result back - /// otherwise, we need to manually wait until the batch is committed + /// otherwise, we need to manually wait until the batch is committed. + /// TODO: there are a few problems: + /// * There can be multiple forceWaitAndProcessResult calls for different + /// batches between waitCommittedUpto calls. + /// In such case, the addErrorResponses below would apply only to the + /// latest of those batches, but they may all be failed. + /// * Of those multiple forceWaitAndProcessResult calls, it's possible that an + /// earlier one succeeds but a later one fails. Then we won't call + /// waitCommittedUpto on the log_idx from the earlier batch, so a subsequent + /// read may happen before that write is committed, violating + /// read-after-write consistency. + /// * With async replication, it's possible for requests to fail even after + /// their forceWaitAndProcessResult call succeeds, if the leader died after + /// accepting the requests for processing (and returning log_idx) but before + /// sending them to a majority of followers. In such case we'll never send + /// a response to the client for those requests. And we may execute + /// subsequent requests from the same session and send responses for those, + /// violating ordering of responses. if (result_buf) { nuraft::buffer_serializer bs(result_buf); @@ -479,24 +496,19 @@ void KeeperDispatcher::initialize(const Poco::Util::AbstractConfiguration & conf { { /// check if we have queue of read requests depending on this request to be committed + SessionAndXID key(request_for_session.session_id, request_for_session.request->xid); std::lock_guard lock(read_request_queue_mutex); - if (auto it = read_request_queue.find(request_for_session.session_id); it != read_request_queue.end()) + if (auto it = read_request_queue.find(key); it != read_request_queue.end()) { - auto & xid_to_request_queue = it->second; - - if (auto request_queue_it = xid_to_request_queue.find(request_for_session.request->xid); - request_queue_it != xid_to_request_queue.end()) + for (const auto & read_request : it->second) { - for (const auto & read_request : request_queue_it->second) - { - if (server->isLeaderAlive()) - server->putLocalReadRequest(read_request); - else - addErrorResponses({read_request}, Coordination::Error::ZCONNECTIONLOSS); - } - - xid_to_request_queue.erase(request_queue_it); + if (server->isLeaderAlive()) + server->putLocalReadRequest(read_request); + else + addErrorResponses({read_request}, Coordination::Error::ZCONNECTIONLOSS, /*may_have_dependent_reads=*/ false); } + + read_request_queue.erase(it); } } }); @@ -732,14 +744,12 @@ void KeeperDispatcher::finishSession(int64_t session_id) CurrentMetrics::sub(CurrentMetrics::KeeperAliveConnections); } } - { - std::lock_guard lock(read_request_queue_mutex); - read_request_queue.erase(session_id); - } } -void KeeperDispatcher::addErrorResponses(const KeeperRequestsForSessions & requests_for_sessions, Coordination::Error error) +void KeeperDispatcher::addErrorResponses(const KeeperRequestsForSessions & requests_for_sessions, Coordination::Error error, bool may_have_dependent_reads) { + KeeperRequestsForSessions dependent_reads; + for (const auto & request_for_session : requests_for_sessions) { KeeperResponsesForSessions responses; @@ -753,7 +763,25 @@ void KeeperDispatcher::addErrorResponses(const KeeperRequestsForSessions & reque response->xid, response->zxid, error); + + if (may_have_dependent_reads) + { + SessionAndXID key(request_for_session.session_id, request_for_session.request->xid); + std::lock_guard lock(read_request_queue_mutex); + if (auto it = read_request_queue.find(key); it != read_request_queue.end()) + { + dependent_reads.insert(dependent_reads.end(), std::move_iterator(it->second.begin()), std::move_iterator(it->second.end())); + read_request_queue.erase(it); + } + } } + + /// Cancel reads that we piggy-backed to the request that failed. They're innocent bystanders + /// that could otherwise succeed, but we don't have a simple way to run these reads correctly + /// in this situation. In particular, there may be later write requests from their sessions that + /// already completed; in that case we can't do the read at all, our committed state is too new. + if (!dependent_reads.empty()) + addErrorResponses(dependent_reads, error, /*may_have_dependent_reads=*/ false); } nuraft::ptr KeeperDispatcher::forceWaitAndProcessResult( @@ -1023,6 +1051,11 @@ Keeper4LWInfo KeeperDispatcher::getKeeper4LWInfo() const return result; } +uint64_t KeeperDispatcher::SessionAndXIDHash::operator()(std::pair p) const +{ + return CityHash_v1_0_2::Hash128to64({uint64_t(p.first), uint64_t(p.second)}); +} + void KeeperDispatcher::cleanResources() { #if USE_JEMALLOC diff --git a/src/Coordination/KeeperDispatcher.h b/src/Coordination/KeeperDispatcher.h index f2cf75d22a48..0ced1dd4a5e3 100644 --- a/src/Coordination/KeeperDispatcher.h +++ b/src/Coordination/KeeperDispatcher.h @@ -93,7 +93,9 @@ class KeeperDispatcher /// Add error responses for requests to responses queue. /// Clears requests. - void addErrorResponses(const KeeperRequestsForSessions & requests_for_sessions, Coordination::Error error); + /// If may_have_dependent_reads is true, also looks at read_request_queue and adds error + /// responses for any reads that were piggy-backed to these requests. + void addErrorResponses(const KeeperRequestsForSessions & requests_for_sessions, Coordination::Error error, bool may_have_dependent_reads = true); /// Forcefully wait for result and sets errors if something when wrong. /// Clears both arguments @@ -101,10 +103,21 @@ class KeeperDispatcher RaftAppendResult & result, KeeperRequestsForSessions & requests_for_sessions, bool clear_requests_on_success); public: + using SessionAndXID = std::pair; + + struct SessionAndXIDHash + { + uint64_t operator()(std::pair) const; + }; + std::mutex read_request_queue_mutex; - /// queue of read requests that can be processed after a request with specific session ID and XID is committed - std::unordered_map> read_request_queue; + /// Local read requests that are piggy-backed to other raft requests. + /// Map: raft request -> read requests. + /// The read must be executed immediately after the corresponding raft request is committed. + /// Note that the read may belong to a different session than the raft request. + /// (So e.g. we can't remove session ID from this map when the session is closed.) + std::unordered_map read_request_queue; /// Just allocate some objects, real initialization is done by `intialize method` KeeperDispatcher(); diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index cdac684740aa..11be234d7584 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -559,6 +559,37 @@ struct KeeperStorageBase::Delta Operation operation; }; +std::string_view deltaTypeToString(const Operation & operation) +{ + /// Using std::visit ensures compile-time exhaustiveness checking - + /// adding a new type to Operation will cause a compilation error until handled here + return std::visit([](const T &) -> std::string_view + { + if constexpr (std::is_same_v) + return "CreateNodeDelta"; + else if constexpr (std::is_same_v) + return "RemoveNodeDelta"; + else if constexpr (std::is_same_v) + return "UpdateNodeStatDelta"; + else if constexpr (std::is_same_v) + return "UpdateNodeDataDelta"; + else if constexpr (std::is_same_v) + return "SetACLDelta"; + else if constexpr (std::is_same_v) + return "AddAuthDelta"; + else if constexpr (std::is_same_v) + return "ErrorDelta"; + else if constexpr (std::is_same_v) + return "SubDeltaEnd"; + else if constexpr (std::is_same_v) + return "FailedMultiDelta"; + else if constexpr (std::is_same_v) + return "CloseSessionDelta"; + else + static_assert(sizeof(T) == 0, "Unhandled Operation type in deltaTypeToString"); + }, operation); +} + KeeperStorageBase::DeltaIterator KeeperStorageBase::DeltaRange::begin() const { return begin_it; @@ -702,7 +733,7 @@ void KeeperStorage::UncommittedState::UncommittedNode::materializeACL template void KeeperStorage::UncommittedState::applyDelta(const Delta & delta, uint64_t * digest) { - chassert(!delta.path.empty()); + chassert(!delta.path.empty(), fmt::format("Path is empty for delta of type '{}'", deltaTypeToString(delta.operation))); UncommittedNode * uncommitted_node = nullptr; auto node_it = nodes.end(); @@ -822,7 +853,7 @@ bool KeeperStorage::UncommittedState::hasACL(int64_t session_id, bool template void KeeperStorage::UncommittedState::rollbackDelta(const Delta & delta) { - chassert(!delta.path.empty()); + chassert(!delta.path.empty(), fmt::format("Path is empty for delta of type '{}'", deltaTypeToString(delta.operation))); std::visit( [&](const DeltaType & operation) @@ -1195,16 +1226,17 @@ void KeeperStorage::applyUncommittedState(KeeperStorage & other, int6 zxids_to_apply.insert(transaction.zxid); } - auto it = uncommitted_state.deltas.begin(); - - for (; it != uncommitted_state.deltas.end(); ++it) + std::list uncommitted_deltas_to_apply; + for (const auto & uncommitted_delta : uncommitted_state.deltas) { - if (!zxids_to_apply.contains(it->zxid)) + if (!zxids_to_apply.contains(uncommitted_delta.zxid)) continue; - other.uncommitted_state.applyDelta(*it, /*digest=*/nullptr); - other.uncommitted_state.deltas.push_back(*it); + uncommitted_deltas_to_apply.push_back(uncommitted_delta); } + + other.uncommitted_state.applyDeltas(uncommitted_deltas_to_apply, /*digest=*/nullptr); + other.uncommitted_state.addDeltas(std::move(uncommitted_deltas_to_apply)); } template diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index 3f9e2ed2c010..39c54ecac96c 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -532,6 +532,10 @@ void BaseSettings::readBinary(ReadBuffer & in) size_t index = accessor.find(name); std::ignore = BaseSettingsHelpers::readFlags(in); + + if (index == static_cast(-1)) + BaseSettingsHelpers::throwSettingNotFound(name); + accessor.readBinary(*this, index, in); } } diff --git a/src/DataTypes/DataTypeFunction.cpp b/src/DataTypes/DataTypeFunction.cpp index 51eb20f023b5..6c2386610cfd 100644 --- a/src/DataTypes/DataTypeFunction.cpp +++ b/src/DataTypes/DataTypeFunction.cpp @@ -36,10 +36,16 @@ bool DataTypeFunction::equals(const IDataType & rhs) const void DataTypeFunction::updateHashImpl(SipHash & hash) const { + /// Argument types and return type can be nullptr when the lambda is not yet resolved. hash.update(argument_types.size()); for (const auto & arg_type : argument_types) - arg_type->updateHash(hash); + { + hash.update(arg_type != nullptr); + if (arg_type) + arg_type->updateHash(hash); + } + hash.update(return_type != nullptr); if (return_type) return_type->updateHash(hash); } diff --git a/src/DataTypes/Serializations/SerializationDynamic.cpp b/src/DataTypes/Serializations/SerializationDynamic.cpp index 5864b6a6e585..e21836f408bf 100644 --- a/src/DataTypes/Serializations/SerializationDynamic.cpp +++ b/src/DataTypes/Serializations/SerializationDynamic.cpp @@ -160,7 +160,7 @@ void SerializationDynamic::serializeBinaryBulkStatePrefix( for (const auto & type : flattened_column.types) { if (settings.native_format && settings.format_settings && settings.format_settings->native.encode_types_in_binary_format) - encodeDataType(type); + encodeDataType(type, *stream); else writeStringBinary(type->getName(), *stream); } diff --git a/src/DataTypes/Serializations/SerializationDynamicHelpers.cpp b/src/DataTypes/Serializations/SerializationDynamicHelpers.cpp index 7736a875540d..8c4f0dd67f1b 100644 --- a/src/DataTypes/Serializations/SerializationDynamicHelpers.cpp +++ b/src/DataTypes/Serializations/SerializationDynamicHelpers.cpp @@ -14,6 +14,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; } namespace @@ -157,6 +158,17 @@ void fillDynamicColumn( for (size_t i = 0; i != indexes_data.size(); ++i) { auto index = indexes_data[i]; + if (index > null_index) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Incorrect index {} in indexes column of flattened Dynamic column at row {}: " + "the index should be in range [0, {}] (there are {} types, index {} is reserved for NULL values)", + static_cast(index), + i, + null_index, + flattened_column.types.size(), + null_index); + if (index == null_index) { local_discriminators.push_back(ColumnVariant::NULL_DISCRIMINATOR); diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index e164e10d6bd1..d597256f993d 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -217,8 +217,10 @@ void SerializationObject::enumerateStreams(EnumerateStreamsSettings & settings, { shared_data_serialization_version = SerializationObjectSharedData::SerializationVersion(settings.object_shared_data_serialization_version); /// Avoid creating buckets in shared data for Wide part if shared data is empty. - if (settings.data_part_type != MergeTreeDataPartType::Wide || !column_object->getStatistics() || !column_object->getStatistics()->shared_data_paths_statistics.empty()) + if (settings.data_part_type != MergeTreeDataPartType::Wide || !column_object->getStatistics() + || !column_object->getStatistics()->shared_data_paths_statistics.empty()) num_buckets = settings.object_shared_data_buckets; + } shared_data_serialization = std::make_shared(shared_data_serialization_version, dynamic_type, num_buckets); @@ -569,6 +571,16 @@ void SerializationObject::deserializeBinaryBulkStatePrefix( }; size_t task_size = std::max(structure_state_concrete->sorted_dynamic_paths->size() / num_tasks, 1ul); + + /// Ensure all already-scheduled tasks are drained on any exit path (including exceptions), + /// so pool threads do not dereference dangling references to stack locals. + SCOPE_EXIT( + for (const auto & task : tasks) + task->tryExecute(); + for (const auto & task : tasks) + task->wait(); + ); + for (size_t i = 0; i != num_tasks; ++i) { auto cache_copy = cache ? std::make_unique(*cache) : nullptr; diff --git a/src/DataTypes/Serializations/SerializationObjectSharedData.cpp b/src/DataTypes/Serializations/SerializationObjectSharedData.cpp index 52676c4c09d2..78aa333eef86 100644 --- a/src/DataTypes/Serializations/SerializationObjectSharedData.cpp +++ b/src/DataTypes/Serializations/SerializationObjectSharedData.cpp @@ -682,10 +682,10 @@ std::shared_ptr SerializationO structure_state.last_granule_structure.clear(); size_t rows_to_read = limit + rows_offset; - StructureGranule current_granule; - std::swap(structure_state.last_granule_structure, current_granule); while (rows_to_read != 0) { + auto & current_granule = structure_state.last_granule_structure; + /// Calculate remaining rows in current granule that can be read. size_t remaining_rows_in_granule = current_granule.num_rows - current_granule.limit - current_granule.offset; @@ -736,12 +736,7 @@ std::shared_ptr SerializationO } result->push_back(current_granule); - current_granule.clear(); } - - /// Remember the state of the last read granule because it can be partially read. - if (!result->empty()) - structure_state.last_granule_structure = result->back(); } /// Add deserialized data into cache. diff --git a/src/DataTypes/Serializations/SerializationSparse.cpp b/src/DataTypes/Serializations/SerializationSparse.cpp index 5e6638a7df4b..2e8e8559e005 100644 --- a/src/DataTypes/Serializations/SerializationSparse.cpp +++ b/src/DataTypes/Serializations/SerializationSparse.cpp @@ -68,7 +68,10 @@ size_t deserializeOffsets(IColumn::Offsets & offsets, skipped_values_rows = 0; size_t max_rows_to_read = offset + limit; - if (max_rows_to_read && state.num_trailing_defaults >= max_rows_to_read) + if (max_rows_to_read == 0) + return 0; + + if (state.num_trailing_defaults >= max_rows_to_read) { state.num_trailing_defaults -= max_rows_to_read; return limit; @@ -111,7 +114,7 @@ size_t deserializeOffsets(IColumn::Offsets & offsets, size_t next_total_rows = total_rows + group_size; group_size += state.num_trailing_defaults; - if (max_rows_to_read && next_total_rows >= max_rows_to_read) + if (next_total_rows >= max_rows_to_read) { /// If it was not last group in granule, /// we have to add current non-default value at further reads. diff --git a/src/DataTypes/Serializations/SerializationVariant.cpp b/src/DataTypes/Serializations/SerializationVariant.cpp index 8bf1a829abe2..31dec1e5ab09 100644 --- a/src/DataTypes/Serializations/SerializationVariant.cpp +++ b/src/DataTypes/Serializations/SerializationVariant.cpp @@ -257,6 +257,9 @@ void SerializationVariant::serializeBinaryBulkWithMultipleStreamsAndUpdateVarian size_t & total_size_of_variants) const { const ColumnVariant & col = assert_cast(column); + if (offset == 0) + col.validateState(); + if (const size_t size = col.size(); limit == 0 || offset + limit > size) limit = size - offset; @@ -315,9 +318,17 @@ void SerializationVariant::serializeBinaryBulkWithMultipleStreamsAndUpdateVarian addVariantElementToPath(settings.path, i); /// We can use the same offset/limit as for whole Variant column if (i == non_empty_global_discr) - variant_serializations[i]->serializeBinaryBulkWithMultipleStreams(col.getVariantByGlobalDiscriminator(i), offset, limit, settings, variant_state->variant_states[i]); + { + const auto & variant_column = col.getVariantByGlobalDiscriminator(i); + if (variant_column.size() < offset + limit) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Variant {} has less rows ({}) than expected rows to serialize ({})", variant_names[i], variant_column.size(), offset + limit); + + variant_serializations[i]->serializeBinaryBulkWithMultipleStreams(variant_column, offset, limit, settings, variant_state->variant_states[i]); + } else + { variant_serializations[i]->serializeBinaryBulkWithMultipleStreams(col.getVariantByGlobalDiscriminator(i), col.getVariantByGlobalDiscriminator(i).size(), 0, settings, variant_state->variant_states[i]); + } settings.path.pop_back(); } variants_statistics[variant_names[non_empty_global_discr]] += limit; @@ -442,6 +453,10 @@ void SerializationVariant::serializeBinaryBulkWithMultipleStreamsAndUpdateVarian settings.path.push_back(Substream::VariantElements); for (size_t i = 0; i != variant_serializations.size(); ++i) { + const auto & variant_column = col.getVariantByGlobalDiscriminator(i); + if (variant_column.size() < variant_offsets_and_limits[i].first + variant_offsets_and_limits[i].second) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Variant {} has less rows ({}) than expected rows to serialize ({})", variant_names[i], variant_column.size(), variant_offsets_and_limits[i].first + variant_offsets_and_limits[i].second); + addVariantElementToPath(settings.path, i); variant_serializations[i]->serializeBinaryBulkWithMultipleStreams( col.getVariantByGlobalDiscriminator(i), @@ -639,6 +654,9 @@ void SerializationVariant::deserializeBinaryBulkWithMultipleStreams( last_non_empty_discr = i; } + if (col.getVariantByLocalDiscriminator(i).size() < variant_limits[i]) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of variant {} is expected to be not less than {} according to discriminators, but it is {}", variant_names[i], variant_limits[i], col.getVariantByLocalDiscriminator(i).size()); + variant_offsets.push_back(col.getVariantByLocalDiscriminator(i).size() - variant_limits[i]); } @@ -676,6 +694,8 @@ void SerializationVariant::deserializeBinaryBulkWithMultipleStreams( addColumnWithNumReadRowsToSubstreamsCache(cache, settings.path, col.getOffsetsPtr(), col.getOffsetsPtr()->size() - prev_size); } settings.path.pop_back(); + + col.validateState(); } std::pair, std::vector> SerializationVariant::deserializeCompactDiscriminators( diff --git a/src/Databases/DataLake/DataLakeConstants.h b/src/Databases/DataLake/DataLakeConstants.h index eaa8f5a276e6..0b228bf310ec 100644 --- a/src/Databases/DataLake/DataLakeConstants.h +++ b/src/Databases/DataLake/DataLakeConstants.h @@ -21,9 +21,19 @@ static constexpr auto DEFAULT_MASKING_RULE = [](const DB::Field &){ return "'[HI using ValueMaskingFunc = std::function; static inline std::unordered_map SETTINGS_TO_HIDE = { + /// Catalog credentials {"catalog_credential", DEFAULT_MASKING_RULE}, {"auth_header", DEFAULT_MASKING_RULE}, + /// AWS credentials {"aws_access_key_id", DEFAULT_MASKING_RULE}, {"aws_secret_access_key", DEFAULT_MASKING_RULE}, + /// OneLake credentials + {"onelake_client_secret", DEFAULT_MASKING_RULE}, + /// Google credentials + {"google_adc_client_secret", DEFAULT_MASKING_RULE}, + {"google_adc_refresh_token", DEFAULT_MASKING_RULE}, + /// DLF credentials + {"dlf_access_key_id", DEFAULT_MASKING_RULE}, + {"dlf_access_key_secret", DEFAULT_MASKING_RULE}, }; } diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 2679ae0de56c..6f022b0da6a4 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -522,8 +522,12 @@ DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIterator( if (filter_by_table_name && !filter_by_table_name(table_name)) continue; - [[maybe_unused]] bool inserted = tables.emplace(table_name, futures[future_index].get()).second; - chassert(inserted); + auto table_ptr = futures[future_index].get(); + if (table_ptr) + { + [[maybe_unused]] bool inserted = tables.emplace(table_name, table_ptr).second; + chassert(inserted); + } future_index++; } return std::make_unique(tables, getDatabaseName()); diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 1775113d1bd1..3ffd3781d86a 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -19,12 +19,7 @@ namespace ProfileEvents { -extern const Event FileSegmentWaitReadBufferMicroseconds; -extern const Event FileSegmentReadMicroseconds; -extern const Event FileSegmentCacheWriteMicroseconds; -extern const Event FileSegmentPredownloadMicroseconds; -extern const Event FileSegmentUsedBytes; - +extern const Event CachedReadBufferWaitReadBufferMicroseconds; extern const Event CachedReadBufferReadFromSourceMicroseconds; extern const Event CachedReadBufferReadFromCacheMicroseconds; extern const Event CachedReadBufferCacheWriteMicroseconds; @@ -72,7 +67,7 @@ CachedOnDiskReadBufferFromFile::CachedOnDiskReadBufferFromFile( , cache(cache_) , settings(settings_) , read_until_position(read_until_position_ ? *read_until_position_ : file_size_) - , implementation_buffer_creator(implementation_buffer_creator_) + , implementation_buffer_creator(std::move(implementation_buffer_creator_)) , query_id(query_id_) , current_buffer_id(getRandomASCIIString(8)) , user(user_) @@ -106,13 +101,9 @@ void CachedOnDiskReadBufferFromFile::appendFilesystemCacheLog( .file_segment_size = range.size(), .read_from_cache_attempted = true, .read_buffer_id = current_buffer_id, - .profile_counters = std::make_shared( - current_file_segment_counters.getPartiallyAtomicSnapshot()), .user_id = user.user_id, }; - current_file_segment_counters.reset(); - switch (type) { case CachedOnDiskReadBufferFromFile::ReadType::CACHED: @@ -443,10 +434,7 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegment & file_segme read_buffer_for_file_segment->getFileOffsetOfBufferEnd(), file_segment.getInfoForLog()); - current_file_segment_counters.increment( - ProfileEvents::FileSegmentWaitReadBufferMicroseconds, watch.elapsedMicroseconds()); - - ProfileEvents::increment(ProfileEvents::FileSegmentWaitReadBufferMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::CachedReadBufferWaitReadBufferMicroseconds, watch.elapsedMicroseconds()); [[maybe_unused]] auto download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; chassert(download_current_segment == file_segment.isDownloader()); @@ -582,8 +570,6 @@ bool CachedOnDiskReadBufferFromFile::predownload(FileSegment & file_segment) Stopwatch predownload_watch(CLOCK_MONOTONIC); SCOPE_EXIT({ predownload_watch.stop(); - current_file_segment_counters.increment( - ProfileEvents::FileSegmentPredownloadMicroseconds, predownload_watch.elapsedMicroseconds()); }); OpenTelemetry::SpanHolder span("CachedOnDiskReadBufferFromFile::predownload"); @@ -615,7 +601,6 @@ bool CachedOnDiskReadBufferFromFile::predownload(FileSegment & file_segment) watch.stop(); auto elapsed = watch.elapsedMicroseconds(); - current_file_segment_counters.increment(ProfileEvents::FileSegmentReadMicroseconds, elapsed); ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceMicroseconds, elapsed); } @@ -801,7 +786,6 @@ bool CachedOnDiskReadBufferFromFile::writeCache(char * data, size_t size, size_t watch.stop(); auto elapsed = watch.elapsedMicroseconds(); - current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, elapsed); ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteMicroseconds, elapsed); ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, size); @@ -1013,7 +997,6 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() watch.stop(); auto elapsed = watch.elapsedMicroseconds(); - current_file_segment_counters.increment(ProfileEvents::FileSegmentReadMicroseconds, elapsed); // We don't support implementation_buffer implementations that use nextimpl_working_buffer_offset. chassert(implementation_buffer->position() == implementation_buffer->buffer().begin()); @@ -1117,8 +1100,6 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() swap.reset(); - current_file_segment_counters.increment(ProfileEvents::FileSegmentUsedBytes, available()); - if (size == 0 && file_offset_of_buffer_end < read_until_position) { size_t cache_file_size = getFileSizeFromReadBuffer(*implementation_buffer); @@ -1178,6 +1159,8 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() file_segment.getInfoForLog()); } + swap.reset(); + // No necessary because of the SCOPE_EXIT above, but useful for logging below. if (download_current_segment) file_segment.completePartAndResetDownloader(); diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h index 4f26a94f91f8..455b454fa39c 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h @@ -142,9 +142,8 @@ class CachedOnDiskReadBufferFromFile : public ReadBufferFromFileBase FileCacheUserInfo user; bool allow_seeks_after_first_read; - [[maybe_unused]]bool use_external_buffer; + bool use_external_buffer; CurrentMetrics::Increment metric_increment{CurrentMetrics::FilesystemCacheReadBuffers}; - ProfileEvents::Counters current_file_segment_counters; FileCacheQueryLimit::QueryContextHolderPtr query_context_holder; diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp index 566a57a0871d..0d7adc934dc2 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp @@ -215,7 +215,6 @@ void FileSegmentRangeWriter::appendFilesystemCacheLog(const FileSegment & file_s .file_segment_size = file_segment_range.size(), .read_from_cache_attempted = false, .read_buffer_id = {}, - .profile_counters = nullptr, }; cache_log->add(std::move(elem)); diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp index 8675555f2668..61641f167c7f 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp @@ -85,7 +85,7 @@ std::unique_ptr CachedObjectStorage::readObject( /// NOL auto global_context = Context::getGlobalContextInstance(); auto modified_read_settings = read_settings.withNestedBuffer(); - auto read_buffer_creator = [=, this]() + auto read_buffer_creator = [this, object, read_settings, read_hint, file_size]() { return object_storage->readObject(object, patchSettings(read_settings), read_hint, file_size); }; diff --git a/src/Formats/JSONExtractTree.cpp b/src/Formats/JSONExtractTree.cpp index ae60f9ce7e0a..e77fa113fa4f 100644 --- a/src/Formats/JSONExtractTree.cpp +++ b/src/Formats/JSONExtractTree.cpp @@ -1996,7 +1996,7 @@ class ObjectJSONNode : public JSONExtractTreeNode if (!sorted_paths_to_skip.empty()) { auto it = std::lower_bound(sorted_paths_to_skip.begin(), sorted_paths_to_skip.end(), path); - if (it != sorted_paths_to_skip.begin() && path.starts_with(*std::prev(it))) + if (it != sorted_paths_to_skip.begin() && path.starts_with(*std::prev(it) + ".")) return true; } diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 745ac3650300..20d096330965 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -396,6 +396,7 @@ struct ToYearWeekImpl return yw.first * 100 + yw.second; } + static constexpr bool hasMonotonicity() { return true; } using FactorTransform = ZeroTransform; }; @@ -431,6 +432,7 @@ struct ToStartOfWeekImpl return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode); } + static constexpr bool hasMonotonicity() { return true; } using FactorTransform = ZeroTransform; }; @@ -464,6 +466,7 @@ struct ToLastDayOfWeekImpl return time_zone.toLastDayNumOfWeek(ExtendedDayNum(d), week_mode); } + static constexpr bool hasMonotonicity() { return true; } using FactorTransform = ZeroTransform; }; @@ -494,6 +497,11 @@ struct ToWeekImpl return yw.second; } + /// toWeek() is not monotonic because week numbers can wrap at year boundaries + /// (e.g. ISO week 52 -> week 1 in late December), depending on the week_mode. + /// See https://github.com/ClickHouse/ClickHouse/issues/90240 + static constexpr bool hasMonotonicity() { return false; } + using FactorTransform = ToStartOfYearImpl; }; @@ -1591,6 +1599,7 @@ struct ToDayOfWeekImpl return time_zone.toDayOfWeek(DayNum(d), mode); } + static constexpr bool hasMonotonicity() { return true; } using FactorTransform = ToMondayImpl; }; diff --git a/src/Functions/FunctionFile.cpp b/src/Functions/FunctionFile.cpp index 1b3fc680171d..648b8b97d124 100644 --- a/src/Functions/FunctionFile.cpp +++ b/src/Functions/FunctionFile.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,14 @@ class FunctionFile : public IFunction, WithContext { public: static constexpr auto name = "file"; - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + + static FunctionPtr create(ContextPtr context_) + { + if (context_ && context_->getApplicationType() != Context::ApplicationType::LOCAL) + context_->checkAccess(AccessType::READ, toStringSource(AccessTypeObjects::Source::FILE)); + + return std::make_shared(context_); + } explicit FunctionFile(ContextPtr context_) : WithContext(context_) {} bool isVariadic() const override { return true; } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index ec7e23d9f587..ba33c9e5abe2 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -3588,13 +3588,13 @@ struct ToDateMonotonicity } else if ( ((left.getType() == Field::Types::UInt64 || left.isNull()) && (right.getType() == Field::Types::UInt64 || right.isNull()) - && ((left.isNull() || left.safeGet() < 0xFFFF) && (right.isNull() || right.safeGet() >= 0xFFFF))) + && ((left.isNull() || left.safeGet() <= DATE_LUT_MAX_DAY_NUM) && (right.isNull() || right.safeGet() > DATE_LUT_MAX_DAY_NUM))) || ((left.getType() == Field::Types::Int64 || left.isNull()) && (right.getType() == Field::Types::Int64 || right.isNull()) - && ((left.isNull() || left.safeGet() < 0xFFFF) && (right.isNull() || right.safeGet() >= 0xFFFF))) + && ((left.isNull() || left.safeGet() <= DATE_LUT_MAX_DAY_NUM) && (right.isNull() || right.safeGet() > DATE_LUT_MAX_DAY_NUM))) || (( (left.getType() == Field::Types::Float64 || left.isNull()) && (right.getType() == Field::Types::Float64 || right.isNull()) - && ((left.isNull() || left.safeGet() < 0xFFFF) && (right.isNull() || right.safeGet() >= 0xFFFF)))) + && ((left.isNull() || left.safeGet() <= DATE_LUT_MAX_DAY_NUM) && (right.isNull() || right.safeGet() > DATE_LUT_MAX_DAY_NUM)))) || !isNativeNumber(type)) { return {}; diff --git a/src/Functions/IFunctionCustomWeek.h b/src/Functions/IFunctionCustomWeek.h index 99941e5c186c..9978586506c0 100644 --- a/src/Functions/IFunctionCustomWeek.h +++ b/src/Functions/IFunctionCustomWeek.h @@ -41,7 +41,7 @@ class IFunctionCustomWeek : public IFunction return true; } - bool hasInformationAboutMonotonicity() const override { return true; } + bool hasInformationAboutMonotonicity() const override { return Transform::hasMonotonicity(); } Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override { diff --git a/src/Functions/ULIDStringToDateTime.cpp b/src/Functions/ULIDStringToDateTime.cpp index 3c6797c9297c..53d1ae06ebdc 100644 --- a/src/Functions/ULIDStringToDateTime.cpp +++ b/src/Functions/ULIDStringToDateTime.cpp @@ -147,6 +147,18 @@ class FunctionULIDStringToDateTime : public IFunction static DateTime64 decode(const UInt8 * data) { + /// Validate that all bytes are ASCII before passing to ulid_decode, + /// which uses char values as array indices. Signed chars with values + /// >= 128 would produce negative indices and out-of-bounds reads. + for (size_t i = 0; i < ULID_LENGTH; ++i) + { + if (data[i] >= 128) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot parse ULID: non-ASCII character at position {}", + i); + } + unsigned char buffer[16]; int ret = ulid_decode(buffer, reinterpret_cast(data)); if (ret != 0) diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index d829240946a5..162fd90d7bec 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -125,9 +125,10 @@ struct Main return left[i] >= right; } - static constexpr bool lessOrEqual(const IColumn & left, const Result & right, size_t i, size_t) noexcept { return left[i] >= right; } + static bool lessOrEqual(const IColumn & left, const Result & right, size_t i, size_t) { return left[i] >= right; } - static constexpr bool lessOrEqual(const Array& arr, const Field& rhs, size_t pos, size_t) noexcept { + static bool lessOrEqual(const Array & arr, const Field & rhs, size_t pos, size_t) + { return accurateLessOrEqual(rhs, arr[pos]); } @@ -666,7 +667,7 @@ class FunctionArrayIndex : public IFunction * @return {nullptr, null_map_item} if there are four arguments but the third is missing. * @return {null_map_data, null_map_item} if there are four arguments. */ - static NullMaps getNullMaps(const ColumnsWithTypeAndName & arguments) noexcept + static NullMaps getNullMaps(const ColumnsWithTypeAndName & arguments) { if (arguments.size() < 3) return {nullptr, nullptr}; diff --git a/src/Functions/bech32.cpp b/src/Functions/bech32.cpp index 3391d164b5e2..b469d2615e20 100644 --- a/src/Functions/bech32.cpp +++ b/src/Functions/bech32.cpp @@ -99,6 +99,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; } /// Encode string to Bech32 or Bech32m address @@ -278,6 +279,24 @@ class EncodeToBech32Representation : public IFunction uint8_t witness_version = have_witness_version ? witness_version_col->getUInt(i) : default_witness_version; + /** Witness version is a versioning mechanism for Bitcoin SegWit addresses: + * - Version 0: Original SegWit (BIP-141, BIP-173), uses Bech32 encoding + * - Version 1: Taproot (BIP-341, BIP-350), uses Bech32m encoding + * - Versions 2-16: Reserved for future protocol upgrades + * + * The witness version must be in range [0, 16] per the SegWit specification. + * It also must fit in the bech32 charset which is 5 bits (0-31), otherwise + * indexing into the CHARSET array in bech32::encode will cause a buffer overflow. + */ + if (witness_version > 16) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Invalid witness version {} for function {}, expected value in range [0, 16]", + witness_version, + name); + } + bech32_data input_5bit; input_5bit.push_back(witness_version); convertbits<8, 5, true>(input_5bit, input); /// squash input from 8-bit -> 5-bit bytes diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index f4fc2ba6de96..be288c8c4b3b 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -603,7 +603,9 @@ class FunctionFormatDateTimeImpl : public IFunction for (UInt32 i = scale; i > 0; --i) { - dest[i - 1] += fractional_second % 10; + /// Use assignment instead of `+=` to avoid reading uninitialized memory + /// when the output buffer is not pre-filled with the template (variable-width formatters path). + dest[i - 1] = '0' + (fractional_second % 10); fractional_second /= 10; } return scale; @@ -617,7 +619,9 @@ class FunctionFormatDateTimeImpl : public IFunction for (UInt32 i = scale; i > 0; --i) { - dest[i - 1] += fractional_second % 10; + /// Use assignment instead of `+=` to avoid reading uninitialized memory + /// when the output buffer is not pre-filled with the template (variable-width formatters path). + dest[i - 1] = '0' + (fractional_second % 10); fractional_second /= 10; } return scale; @@ -819,6 +823,11 @@ class FunctionFormatDateTimeImpl : public IFunction return min_represent_digits; } auto str = toString(fractional_second); + /// Left-pad with zeros to `scale` digits, because `toString` does not preserve leading zeros + /// (e.g. fractional_second=5, scale=3 gives "5" but we need "005"). + /// Without this, the buffer would be left partially uninitialized. + if (str.size() < scale) + str.insert(0, scale - str.size(), '0'); if (min_represent_digits > scale) { for (UInt64 i = 0; i < min_represent_digits - scale; ++i) @@ -851,7 +860,7 @@ class FunctionFormatDateTimeImpl : public IFunction static bool containsOnlyFixedWidthMySQLFormatters(std::string_view format, bool mysql_M_is_month_name, bool mysql_format_ckl_without_leading_zeros, bool mysql_e_with_space_padding) { static constexpr std::array variable_width_formatter = {'W'}; - static constexpr std::array variable_width_formatter_M_is_month_name = {'W', 'M'}; + static constexpr std::array variable_width_formatter_M_is_month_name = {'M'}; static constexpr std::array variable_width_formatter_leading_zeros = {'c', 'l', 'k'}; static constexpr std::array variable_width_formatter_e_with_space_padding = {'e'}; @@ -862,6 +871,12 @@ class FunctionFormatDateTimeImpl : public IFunction case '%': if (i + 1 >= format.size()) throwLastCharacterIsPercentException(); + + if (std::any_of( + variable_width_formatter.begin(), variable_width_formatter.end(), + [&](char c){ return c == format[i + 1]; })) + return false; + if (mysql_M_is_month_name) { if (std::any_of( @@ -883,13 +898,7 @@ class FunctionFormatDateTimeImpl : public IFunction [&](char c){ return c == format[i + 1]; })) return false; } - else - { - if (std::any_of( - variable_width_formatter.begin(), variable_width_formatter.end(), - [&](char c){ return c == format[i + 1]; })) - return false; - } + i += 1; continue; default: diff --git a/src/Functions/reverseUTF8.cpp b/src/Functions/reverseUTF8.cpp index 15deed86c256..00d400b11f0a 100644 --- a/src/Functions/reverseUTF8.cpp +++ b/src/Functions/reverseUTF8.cpp @@ -46,26 +46,31 @@ struct ReverseUTF8Impl ColumnString::Offset j = prev_offset; while (j < offsets[i]) { + size_t remaining = offsets[i] - j; + + unsigned int char_len; if (data[j] < 0xC0) - { - res_data[offsets[i] + prev_offset - 1 - j] = data[j]; - j += 1; - } + char_len = 1; else if (data[j] < 0xE0) - { - memcpy(&res_data[offsets[i] + prev_offset - 1 - j - 1], &data[j], 2); - j += 2; - } + char_len = 2; else if (data[j] < 0xF0) + char_len = 3; + else + char_len = 4; + + /// If not enough bytes remaining, treat as single byte (invalid UTF-8). + if (char_len > remaining) + char_len = 1; + + if (char_len == 1) { - memcpy(&res_data[offsets[i] + prev_offset - 1 - j - 2], &data[j], 3); - j += 3; + res_data[offsets[i] + prev_offset - 1 - j] = data[j]; } else { - memcpy(&res_data[offsets[i] + prev_offset - 1 - j - 3], &data[j], 4); - j += 4; + memcpy(&res_data[offsets[i] + prev_offset - j - char_len], &data[j], char_len); } + j += char_len; } prev_offset = offsets[i]; diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp index c3ffa173cac3..ce8d3aa7091a 100644 --- a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp +++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp @@ -209,7 +209,7 @@ off_t AsynchronousReadBufferFromFileDescriptor::seek(off_t offset, int whence) } else if (whence == SEEK_CUR) { - new_pos = file_offset_of_buffer_end - (working_buffer.end() - pos) + offset; + new_pos = static_cast(getPosition()) + offset; } else { @@ -217,13 +217,15 @@ off_t AsynchronousReadBufferFromFileDescriptor::seek(off_t offset, int whence) } /// Position is unchanged. - if (new_pos + (working_buffer.end() - pos) == file_offset_of_buffer_end) + if (new_pos == static_cast(getPosition())) return new_pos; bool read_from_prefetch = false; while (true) { - if (file_offset_of_buffer_end - working_buffer.size() <= new_pos && new_pos <= file_offset_of_buffer_end) + if (bytes_to_ignore == 0 + && file_offset_of_buffer_end - working_buffer.size() <= new_pos + && new_pos <= file_offset_of_buffer_end) { /// Position is still inside the buffer. /// Probably it is at the end of the buffer - then we will load data on the following 'next' call. @@ -289,6 +291,7 @@ void AsynchronousReadBufferFromFileDescriptor::rewind() working_buffer.resize(0); pos = working_buffer.begin(); file_offset_of_buffer_end = 0; + bytes_to_ignore = 0; } std::optional AsynchronousReadBufferFromFileDescriptor::tryGetFileSize() diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.h b/src/IO/AsynchronousReadBufferFromFileDescriptor.h index e15a41474256..fe4fa55d886f 100644 --- a/src/IO/AsynchronousReadBufferFromFileDescriptor.h +++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.h @@ -62,7 +62,7 @@ class AsynchronousReadBufferFromFileDescriptor : public ReadBufferFromFileBase off_t getPosition() override { - return file_offset_of_buffer_end - (working_buffer.end() - pos); + return file_offset_of_buffer_end - (working_buffer.end() - pos) + bytes_to_ignore; } /// If 'offset' is small enough to stay in buffer after seek, then true seek in file does not happen. diff --git a/src/IO/S3/AWSLogger.cpp b/src/IO/S3/AWSLogger.cpp index 254b7aef3d21..71c2ac3f936d 100644 --- a/src/IO/S3/AWSLogger.cpp +++ b/src/IO/S3/AWSLogger.cpp @@ -121,6 +121,13 @@ void AWSLogger::callLogImpl(Aws::Utils::Logging::LogLevel log_level, const char LOG_IMPL(default_logger, level, prio, "{}: {}", tag, message); } +void AWSLogger::vaLog(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * format_str, va_list) +{ + if (is404Muted(format_str)) + return; + callLogImpl(log_level, tag, format_str); /// FIXME. Variadic arguments? +} + } #endif diff --git a/src/IO/S3/AWSLogger.h b/src/IO/S3/AWSLogger.h index a4987f17c0dd..5bcb1db458f8 100644 --- a/src/IO/S3/AWSLogger.h +++ b/src/IO/S3/AWSLogger.h @@ -29,6 +29,8 @@ class AWSLogger final : public Aws::Utils::Logging::LogSystemInterface void Flush() final {} + void vaLog(Aws::Utils::Logging::LogLevel log_level, const char * tag, const char * format_str, va_list args) final; + private: LoggerPtr default_logger; bool enable_s3_requests_logging; diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index c46d1456c417..e4f8757d4968 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 7b36c5195a34..a002d298710d 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -133,6 +133,11 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( LOG_INFO(getLogger("PocoHTTPClientConfiguration"), "Jitter factor for the retry strategy must be within the [0, 1], clamping"); retry_strategy.jitter_factor = std::clamp(retry_strategy.jitter_factor, 0.0, 1.0); } + + /// NOTE: Without these settings AWS SDK enable transfer-encoding: chunked and content-encoding: aws-chunked + /// We don't use them and MinIO server doesn't support them. + checksumConfig.requestChecksumCalculation = Aws::Client::RequestChecksumCalculation::WHEN_REQUIRED; + checksumConfig.responseChecksumValidation = Aws::Client::ResponseChecksumValidation::WHEN_REQUIRED; } void PocoHTTPClientConfiguration::updateSchemeAndRegion() @@ -272,6 +277,9 @@ PocoHTTPClient::S3MetricKind PocoHTTPClient::getMetricKind(const Aws::Http::Http { case Aws::Http::HttpMethod::HTTP_GET: case Aws::Http::HttpMethod::HTTP_HEAD: + case Aws::Http::HttpMethod::HTTP_TRACE: + case Aws::Http::HttpMethod::HTTP_OPTIONS: + case Aws::Http::HttpMethod::HTTP_CONNECT: return S3MetricKind::Read; case Aws::Http::HttpMethod::HTTP_POST: case Aws::Http::HttpMethod::HTTP_DELETE: @@ -351,12 +359,15 @@ void PocoHTTPClient::observeLatency(const Aws::Http::HttpRequest & request, S3La { switch (m) { - case Aws::Http::HttpMethod::HTTP_GET: return "GET"; - case Aws::Http::HttpMethod::HTTP_HEAD: return "HEAD"; - case Aws::Http::HttpMethod::HTTP_POST: return "POST"; - case Aws::Http::HttpMethod::HTTP_DELETE: return "DELETE"; - case Aws::Http::HttpMethod::HTTP_PUT: return "PUT"; - case Aws::Http::HttpMethod::HTTP_PATCH: return "PATCH"; + case Aws::Http::HttpMethod::HTTP_GET: return "GET"; + case Aws::Http::HttpMethod::HTTP_HEAD: return "HEAD"; + case Aws::Http::HttpMethod::HTTP_POST: return "POST"; + case Aws::Http::HttpMethod::HTTP_DELETE: return "DELETE"; + case Aws::Http::HttpMethod::HTTP_PUT: return "PUT"; + case Aws::Http::HttpMethod::HTTP_PATCH: return "PATCH"; + case Aws::Http::HttpMethod::HTTP_CONNECT: return "CONNECT"; + case Aws::Http::HttpMethod::HTTP_TRACE: return "TRACE"; + case Aws::Http::HttpMethod::HTTP_OPTIONS: return "OPTIONS"; } }(request.GetMethod()); @@ -417,6 +428,12 @@ String getMethod(const Aws::Http::HttpRequest & request) return Poco::Net::HTTPRequest::HTTP_HEAD; case Aws::Http::HttpMethod::HTTP_PATCH: return Poco::Net::HTTPRequest::HTTP_PATCH; + case Aws::Http::HttpMethod::HTTP_CONNECT: + return Poco::Net::HTTPRequest::HTTP_CONNECT; + case Aws::Http::HttpMethod::HTTP_TRACE: + return Poco::Net::HTTPRequest::HTTP_TRACE; + case Aws::Http::HttpMethod::HTTP_OPTIONS: + return Poco::Net::HTTPRequest::HTTP_OPTIONS; } } @@ -463,6 +480,9 @@ void PocoHTTPClient::makeRequestInternalImpl( { case Aws::Http::HttpMethod::HTTP_GET: case Aws::Http::HttpMethod::HTTP_HEAD: + case Aws::Http::HttpMethod::HTTP_TRACE: + case Aws::Http::HttpMethod::HTTP_OPTIONS: + case Aws::Http::HttpMethod::HTTP_CONNECT: if (get_request_throttler) { Stopwatch sleep_watch; diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h index 6685a1694077..2a6737d227cb 100644 --- a/src/IO/S3/Requests.h +++ b/src/IO/S3/Requests.h @@ -92,6 +92,12 @@ class ExtendedRequest : public BaseRequest return BaseRequest::GetChecksumAlgorithmName(); } + /// TODO Understand what is it. Maybe we need it... + bool IsStreaming() const override + { + return false; + } + std::string getRegionOverride() const { return region_override; @@ -149,6 +155,15 @@ class UploadPartRequest : public ExtendedRequest { public: void SetAdditionalCustomHeaderValue(const Aws::String& headerName, const Aws::String& headerValue) override; + bool RequestChecksumRequired() const override { return is_s3express_bucket; } + bool ShouldComputeContentMd5() const override { return !is_s3express_bucket && checksum; } +}; + +class PutObjectRequest : public ExtendedRequest +{ +public: + bool RequestChecksumRequired() const override { return is_s3express_bucket; } + bool ShouldComputeContentMd5() const override { return !is_s3express_bucket && checksum; } }; class CompleteMultipartUploadRequest : public ExtendedRequest @@ -161,10 +176,19 @@ using CreateMultipartUploadRequest = ExtendedRequest; using UploadPartCopyRequest = ExtendedRequest; -using PutObjectRequest = ExtendedRequest; -using DeleteObjectRequest = ExtendedRequest; -using DeleteObjectsRequest = ExtendedRequest; +class DeleteObjectRequest : public ExtendedRequest +{ +public: + bool RequestChecksumRequired() const override { return is_s3express_bucket; } + bool ShouldComputeContentMd5() const override { return !is_s3express_bucket && checksum; } +}; +class DeleteObjectsRequest : public ExtendedRequest +{ +public: + bool RequestChecksumRequired() const override { return is_s3express_bucket; } + bool ShouldComputeContentMd5() const override { return !is_s3express_bucket && checksum; } +}; class ComposeObjectRequest : public ExtendedRequest { diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index 5b2c7ed5a9c7..694dca66a262 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -180,7 +180,7 @@ ReturnType parseDateTimeBestEffortImpl( } if (num_digits == 10 && !year && !has_time) { - if (strict) + if constexpr (strict) return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Strict best effort parsing doesn't allow timestamps"); /// This is unix timestamp. @@ -188,14 +188,18 @@ ReturnType parseDateTimeBestEffortImpl( if (fractional && !in.eof() && *in.position() == '.') { ++in.position(); - fractional->digits = readDigits(digits, sizeof(digits), in); + // Prevent numeric overflow + using FractionalType = typename std::decay_tvalue)>; + fractional->digits = static_cast(std::min( + static_cast(std::numeric_limits::digits10), + readDigits(digits, sizeof(digits), in))); readDecimalNumber(fractional->value, fractional->digits, digits); } return ReturnType(true); } if (num_digits == 9 && !year && !has_time) { - if (strict) + if constexpr (strict) return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Strict best effort parsing doesn't allow timestamps"); /// This is unix timestamp. @@ -203,14 +207,18 @@ ReturnType parseDateTimeBestEffortImpl( if (fractional && !in.eof() && *in.position() == '.') { ++in.position(); - fractional->digits = readDigits(digits, sizeof(digits), in); + // Prevent numeric overflow + using FractionalType = typename std::decay_tvalue)>; + fractional->digits = static_cast(std::min( + static_cast(std::numeric_limits::digits10), + readDigits(digits, sizeof(digits), in))); readDecimalNumber(fractional->value, fractional->digits, digits); } return ReturnType(true); } if (num_digits == 14 && !year && !has_time) { - if (strict) + if constexpr (strict) return on_error( ErrorCodes::CANNOT_PARSE_DATETIME, "Strict best effort parsing doesn't allow date times without separators"); @@ -225,7 +233,7 @@ ReturnType parseDateTimeBestEffortImpl( } else if (num_digits == 8 && !year) { - if (strict) + if constexpr (strict) return on_error( ErrorCodes::CANNOT_PARSE_DATETIME, "Strict best effort parsing doesn't allow date times without separators"); @@ -236,7 +244,7 @@ ReturnType parseDateTimeBestEffortImpl( } else if (num_digits == 6) { - if (strict) + if constexpr (strict) return on_error( ErrorCodes::CANNOT_PARSE_DATETIME, "Strict best effort parsing doesn't allow date times without separators"); @@ -474,8 +482,9 @@ ReturnType parseDateTimeBestEffortImpl( { if (day_of_month) { - if (strict && hour) - return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: hour component is duplicated"); + if constexpr (strict) + if (hour) + return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: hour component is duplicated"); hour = hour_or_day_of_month_or_month; } @@ -516,7 +525,7 @@ ReturnType parseDateTimeBestEffortImpl( if (fractional) { using FractionalType = typename std::decay_tvalue)>; - // Reading more decimal digits than fits into FractionalType would case an + // Reading more decimal digits than fits into FractionalType would cause an // overflow, so it is better to skip all digits from the right side that do not // fit into result type. To provide less precise value rather than bogus one. num_digits = std::min(static_cast(std::numeric_limits::digits10), num_digits); @@ -524,7 +533,7 @@ ReturnType parseDateTimeBestEffortImpl( fractional->digits = num_digits; readDecimalNumber(fractional->value, num_digits, digits); } - else if (strict) + else if constexpr (strict) { /// Fractional part is not allowed. return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: unexpected fractional part"); @@ -784,6 +793,15 @@ ReturnType parseDateTimeBestEffortImpl( } res = *res_maybe; adjust_time_zone(); + + /// After timezone adjustment, the value may have shifted outside the valid range. + /// For example, "2106-02-07 06:28:15-01:00" is within range before adjustment, + /// but after converting to UTC it exceeds UINT32_MAX. + if constexpr (!is_64) + { + if (res < 0 || static_cast(res) > UINT32_MAX) + return false; + } } else { diff --git a/src/Interpreters/Access/InterpreterGrantQuery.cpp b/src/Interpreters/Access/InterpreterGrantQuery.cpp index 2b5381e96400..1b43109a05f3 100644 --- a/src/Interpreters/Access/InterpreterGrantQuery.cpp +++ b/src/Interpreters/Access/InterpreterGrantQuery.cpp @@ -319,9 +319,20 @@ namespace if (!roles_to_revoke.empty()) { if (admin_option) + { grantee.granted_roles.revokeAdminOption(grantee.granted_roles.findGrantedWithAdminOption(roles_to_revoke)); + } else - grantee.granted_roles.revoke(grantee.granted_roles.findGranted(roles_to_revoke)); + { + auto found_roles_to_revoke = grantee.granted_roles.findGranted(roles_to_revoke); + grantee.granted_roles.revoke(found_roles_to_revoke); + + if constexpr (std::is_same_v) + { + for (const auto & id : found_roles_to_revoke) + grantee.default_roles.ids.erase(id); + } + } } if (!roles_to_grant.empty()) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 300a43f3d779..417eb055dc9b 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -163,8 +163,17 @@ void ActionsDAG::Node::updateHash(SipHash & hash_state) const hash_state.update(is_deterministic_constant); if (column) + { hash_state.update(column->getName()); + /// We must also hash the actual constant value, not just the column type name. + /// Otherwise, two different constants with the same type and the same expression-based + /// result_name (e.g. from CTE constant folding) would produce identical hashes, + /// leading to query condition cache collisions and incorrect results. + if (isColumnConst(*column)) + column->updateHashWithValue(0, hash_state); + } + for (const auto & child : children) child->updateHash(hash_state); } @@ -842,6 +851,9 @@ static ColumnWithTypeAndName executeActionForPartialResult(const ActionsDAG::Nod case ActionsDAG::ActionType::ARRAY_JOIN: { auto key = arguments.at(0); + if (!key.column) + break; + key.column = key.column->convertToFullColumnIfConst(); const auto * array = getArrayJoinColumnRawPtr(key.column); diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp index 1d037a5f14bf..5812ae193ca8 100644 --- a/src/Interpreters/CancellationChecker.cpp +++ b/src/Interpreters/CancellationChecker.cpp @@ -10,6 +10,14 @@ namespace DB { +/// Align all timeouts to a grid to allow batching of timeout processing. +/// Tasks may be cancelled slightly later than their exact timeout, but never before. +static constexpr UInt64 CANCELLATION_GRID_MS = 100; + +/// Maximum allowed timeout is 1 year in milliseconds. +/// This prevents overflow in chrono calculations and ensures reasonable behavior. +static constexpr Int64 MAX_TIMEOUT_MS = 365LL * 24 * 60 * 60 * 1000; + struct CancellationChecker::QueryToTrack { QueryToTrack(QueryStatusPtr query_, UInt64 timeout_, UInt64 endtime_, OverflowMode overflow_mode_) @@ -63,38 +71,46 @@ void CancellationChecker::terminateThread() cond_var.notify_all(); } -bool CancellationChecker::removeQueryFromSet(QueryStatusPtr query) -{ - auto it = std::ranges::find(query_set, query, &QueryToTrack::query); - - if (it == query_set.end()) - return false; - - LOG_TEST(log, "Removing query {} from done tasks", query->getClientInfo().current_query_id); - query_set.erase(it); - return true; -} - -void CancellationChecker::appendTask(const QueryStatusPtr & query, const Int64 timeout, OverflowMode overflow_mode) +bool CancellationChecker::appendTask(const QueryStatusPtr & query, const Int64 timeout, OverflowMode overflow_mode) { if (timeout <= 0) // Avoid cases when the timeout is less or equal zero { LOG_TEST(log, "Did not add the task because the timeout is 0, query_id: {}", query->getClientInfo().current_query_id); - return; + return false; } + + /// Cap timeout to 1 year to prevent overflow in chrono calculations. + /// std::condition_variable::wait_for converts milliseconds to nanoseconds internally + /// (multiplying by 1,000,000), which overflows for values close to INT64_MAX. + const Int64 capped_timeout = std::min(timeout, MAX_TIMEOUT_MS); + std::unique_lock lock(m); - LOG_TEST(log, "Added to set. query: {}, timeout: {} milliseconds", query->getInfo().query, timeout); + LOG_TEST(log, "Added to set. query: {}, timeout: {} milliseconds", query->getInfo().query, capped_timeout); const auto now = std::chrono::steady_clock::now(); - const UInt64 end_time = std::chrono::duration_cast(now.time_since_epoch()).count() + timeout; - query_set.emplace(query, timeout, end_time, overflow_mode); - cond_var.notify_all(); + const UInt64 now_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + /// Round up to the next grid boundary to enable batching of timeout checks. + /// This ensures tasks are never cancelled before their timeout, only slightly after. + const UInt64 end_time = ((now_ms + capped_timeout + CANCELLATION_GRID_MS - 1) / CANCELLATION_GRID_MS) * CANCELLATION_GRID_MS; + auto iter = query_set.emplace(query, capped_timeout, end_time, overflow_mode); + if (iter == query_set.begin()) // Only notify if the new task is the earliest one + cond_var.notify_all(); + return true; } void CancellationChecker::appendDoneTasks(const QueryStatusPtr & query) { - std::unique_lock lock(m); - removeQueryFromSet(query); - cond_var.notify_all(); + std::unique_lock lock(m); + + auto it = std::ranges::find(query_set, query, &QueryToTrack::query); + if (it == query_set.end()) + return; + + LOG_TEST(log, "Removing query {} from done tasks", query->getClientInfo().current_query_id); + query_set.erase(it); + + // Note that there is no need to notify the worker thread here. Even if we have just removed the earliest task, + // it will wake up before the next task anyway and fix its timeout to a proper value on wake-up. + // This optimization avoids unnecessary contention on the mutex. } void CancellationChecker::workerFunction() @@ -107,20 +123,19 @@ void CancellationChecker::workerFunction() while (!stop_thread) { UInt64 now_ms = 0; - std::chrono::steady_clock::duration duration_milliseconds = std::chrono::milliseconds(0); - if (!query_set.empty()) { - const auto next_task_it = query_set.begin(); - - // Convert UInt64 timeout to std::chrono::steady_clock::time_point - duration_milliseconds = std::chrono::milliseconds(next_task_it->timeout); - - auto end_time_ms = next_task_it->endtime; auto now = std::chrono::steady_clock::now(); now_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); - if ((end_time_ms <= now_ms && duration_milliseconds.count() != 0)) + + /// Batch all tasks that have reached their deadline. + /// Since deadlines are aligned to a grid, multiple tasks often expire together. + while (!query_set.empty()) { + auto next_task_it = query_set.begin(); + if (next_task_it->endtime > now_ms || next_task_it->timeout == 0) + break; + LOG_DEBUG( log, "Cancelling the task because of the timeout: {} ms, query_id: {}", @@ -129,7 +144,6 @@ void CancellationChecker::workerFunction() tasks_to_cancel.push_back(*next_task_it); query_set.erase(next_task_it); - continue; } } @@ -142,20 +156,24 @@ void CancellationChecker::workerFunction() continue; } - /// if last time we checked there were no queries, + /// if there are no queries, /// wakeup on first query that was added so we can setup /// proper timeout for waking up the thread - if (!now_ms) + if (query_set.empty()) { cond_var.wait(lock, [&] { return stop_thread || !query_set.empty(); }); } else { - chassert(duration_milliseconds.count()); + chassert(!query_set.empty()); cond_var.wait_for( lock, - duration_milliseconds, - [&, now_ms] { return stop_thread || (!query_set.empty() && query_set.begin()->endtime < now_ms); }); + std::chrono::milliseconds(query_set.begin()->endtime - now_ms), + [&] { + /// Use fresh time to avoid spinning when the predicate is re-evaluated after spurious wakeups. + UInt64 fresh_now_ms = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + return stop_thread || (!query_set.empty() && query_set.begin()->endtime <= fresh_now_ms); + }); } } } diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index fcc9d43f1543..7a4f0e089de6 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -42,9 +42,6 @@ class CancellationChecker std::mutex m; std::condition_variable cond_var; - // Function to execute when a task's endTime is reached - bool removeQueryFromSet(QueryStatusPtr query); - static void cancelTask(CancellationChecker::QueryToTrack task); const LoggerPtr log; @@ -59,8 +56,8 @@ class CancellationChecker void terminateThread(); - // Method to add a new task to the multiset - void appendTask(const QueryStatusPtr & query, Int64 timeout, OverflowMode overflow_mode); + // Method to add a new task to the multiset. Returns true if the task was added. + [[nodiscard]] bool appendTask(const QueryStatusPtr & query, Int64 timeout, OverflowMode overflow_mode); // Used when some task is done void appendDoneTasks(const QueryStatusPtr & query); diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index a3f456faca7c..4c50f2f352e6 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -1429,10 +1429,23 @@ void DDLWorker::markReplicasActive(bool reinitialized) zookeeper->deleteEphemeralNodeIfContentMatches(active_path, active_id); } Coordination::Requests ops; + Coordination::Responses res; ops.emplace_back(zkutil::makeCreateRequest(active_path, active_id, zkutil::CreateMode::Ephemeral)); /// To bump node mtime ops.emplace_back(zkutil::makeSetRequest(fs::path(replicas_dir) / host_id, "", -1)); - zookeeper->multi(ops); + auto code = zookeeper->tryMulti(ops, res); + + /// We have this tryMulti for a very weird edge case when it's related to localhost. + /// Each replica may have a localhost as hostid and if we configured multiple replicas to add their + /// localhosts to some clusters multiple of them may think that they must mark it as active. + if (code != Coordination::Error::ZOK) + { + LOG_WARNING(log, "Cannot mark a replica active: active_path={}, active_id={}, code={}", active_path, active_id, Coordination::errorMessage(code)); + } + else + { + LOG_INFO(log, "Marked a replica active: active_path={}, active_id={}", active_path, active_id); + } auto active_node_holder_zookeeper = zookeeper; auto active_node_holder = zkutil::EphemeralNodeHolder::existing(active_path, *active_node_holder_zookeeper); diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index be910cf3de23..62d4b07b2242 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -89,6 +89,7 @@ namespace ErrorCodes namespace Setting { extern const SettingsBool fsync_metadata; + extern const SettingsBool allow_experimental_analyzer; } namespace MergeTreeSetting @@ -383,6 +384,7 @@ DatabaseAndTable DatabaseCatalog::getTableImpl( return {}; } + bool analyzer = context_->getSettingsRef()[Setting::allow_experimental_analyzer]; if (table_id.hasUUID()) { /// Shortcut for tables which have persistent UUID @@ -401,7 +403,8 @@ DatabaseAndTable DatabaseCatalog::getTableImpl( } return {}; } - else + /// In old analyzer resolving done in multiple places, so we ignore TABLE_UUID_MISMATCH error. + else if (analyzer) { const auto & table_storage_id = db_and_table.second->getStorageID(); if (db_and_table.first->getDatabaseName() != table_id.database_name || diff --git a/src/Interpreters/FilesystemCacheLog.cpp b/src/Interpreters/FilesystemCacheLog.cpp index f518ef46ba37..d88168f16617 100644 --- a/src/Interpreters/FilesystemCacheLog.cpp +++ b/src/Interpreters/FilesystemCacheLog.cpp @@ -42,7 +42,6 @@ ColumnsDescription FilesystemCacheLogElement::getColumnsDescription() {"size", std::make_shared(), "Read size"}, {"read_type", std::make_shared(), "Read type: READ_FROM_CACHE, READ_FROM_FS_AND_DOWNLOADED_TO_CACHE, READ_FROM_FS_BYPASSING_CACHE"}, {"read_from_cache_attempted", std::make_shared(), "Whether reading from cache was attempted"}, - {"ProfileEvents", std::make_shared(low_cardinality_string, std::make_shared()), "Profile events collected while reading this file segment"}, {"read_buffer_id", std::make_shared(), "Internal implementation read buffer id"}, {"user_id", std::make_shared(), "User id of the user which created the file segment"}, }; @@ -66,17 +65,6 @@ void FilesystemCacheLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(file_segment_size); columns[i++]->insert(typeToString(cache_type)); columns[i++]->insert(read_from_cache_attempted); - - if (profile_counters) - { - auto * column = columns[i++].get(); - ProfileEvents::dumpToMapColumn(*profile_counters, column, true); - } - else - { - columns[i++]->insertDefault(); - } - columns[i++]->insert(read_buffer_id); columns[i++]->insert(user_id); } diff --git a/src/Interpreters/FilesystemCacheLog.h b/src/Interpreters/FilesystemCacheLog.h index 94c3986a1ab2..3d5393f15ab5 100644 --- a/src/Interpreters/FilesystemCacheLog.h +++ b/src/Interpreters/FilesystemCacheLog.h @@ -35,7 +35,6 @@ struct FilesystemCacheLogElement size_t file_segment_size = 0; bool read_from_cache_attempted; String read_buffer_id{}; - std::shared_ptr profile_counters = nullptr; String user_id{}; static std::string name() { return "FilesystemCacheLog"; } diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp index f957b218c4c9..231c7ec02dd6 100644 --- a/src/Interpreters/GraceHashJoin.cpp +++ b/src/Interpreters/GraceHashJoin.cpp @@ -528,6 +528,17 @@ class GraceHashJoin::DelayedBlocks : public IBlocksStream if (not_processed) { auto res = not_processed->next(); + if (res.is_last && res.next_block) + { + res.next_block->filterBySelector(); + auto next_block = std::move(*res.next_block).getSourceBlock(); + if (next_block.rows() > 0) + { + auto new_res = hash_join->joinBlock(std::move(next_block)); + std::lock_guard lock(extra_block_mutex); + not_processed_results.emplace_back(std::move(new_res)); + } + } if (!res.is_last) { std::lock_guard lock(extra_block_mutex); @@ -602,6 +613,17 @@ class GraceHashJoin::DelayedBlocks : public IBlocksStream auto res = hash_join->joinBlock(block); auto next = res->next(); + if (next.is_last && next.next_block) + { + next.next_block->filterBySelector(); + auto next_block = std::move(*next.next_block).getSourceBlock(); + if (next_block.rows() > 0) + { + auto new_res = hash_join->joinBlock(std::move(next_block)); + std::lock_guard lock(extra_block_mutex); + not_processed_results.emplace_back(std::move(new_res)); + } + } if (!next.is_last) { std::lock_guard lock(extra_block_mutex); diff --git a/src/Interpreters/HashJoin/HashJoinMethods.h b/src/Interpreters/HashJoin/HashJoinMethods.h index 4241c4e129ee..ffec6ef4910d 100644 --- a/src/Interpreters/HashJoin/HashJoinMethods.h +++ b/src/Interpreters/HashJoin/HashJoinMethods.h @@ -193,7 +193,7 @@ class HashJoinMethods /// First to collect all matched rows refs by join keys, then filter out rows which are not true in additional filter expression. template - static size_t joinRightColumnsWithAddtitionalFilter( + static size_t joinRightColumnsWithAdditionalFilter( std::vector && key_getter_vector, const std::vector & mapv, AddedColumns & added_columns, diff --git a/src/Interpreters/HashJoin/HashJoinMethodsImpl.h b/src/Interpreters/HashJoin/HashJoinMethodsImpl.h index ee4e90f8da04..13e7a07aed93 100644 --- a/src/Interpreters/HashJoin/HashJoinMethodsImpl.h +++ b/src/Interpreters/HashJoin/HashJoinMethodsImpl.h @@ -306,7 +306,7 @@ size_t HashJoinMethods::joinRightColumnsSwitchMu if (added_columns.additional_filter_expression) { const bool mark_per_row_used = join_features.right || join_features.full || mapv.size() > 1; - return joinRightColumnsWithAddtitionalFilter( + return joinRightColumnsWithAdditionalFilter( std::forward>(key_getter_vector), mapv, added_columns, @@ -815,7 +815,7 @@ static ColumnPtr buildAdditionalFilter( template template -size_t HashJoinMethods::joinRightColumnsWithAddtitionalFilter( +size_t HashJoinMethods::joinRightColumnsWithAdditionalFilter( std::vector && key_getter_vector, const std::vector & mapv, AddedColumns & added_columns, diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 374d74325416..a1405381a05c 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1518,7 +1518,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) create.sql_security = std::make_shared(); if (create.sql_security) - processSQLSecurityOption(getContext(), create.sql_security->as(), create.is_materialized_view, /* skip_check_permissions= */ mode >= LoadingStrictnessLevel::SECONDARY_CREATE); + processSQLSecurityOption(getContext(), create.sql_security->as(), create.is_materialized_view, mode); DDLGuardPtr ddl_guard; @@ -2454,7 +2454,7 @@ void InterpreterCreateQuery::addColumnsDescriptionToCreateQueryIfNecessary(ASTCr } } -void InterpreterCreateQuery::processSQLSecurityOption(ContextMutablePtr context_, ASTSQLSecurity & sql_security, bool is_materialized_view, bool skip_check_permissions) +void InterpreterCreateQuery::processSQLSecurityOption(ContextMutablePtr context_, ASTSQLSecurity & sql_security, bool is_materialized_view, LoadingStrictnessLevel mode) { /// If no SQL security is specified, apply default from default_*_view_sql_security setting. if (!sql_security.type) @@ -2495,27 +2495,30 @@ void InterpreterCreateQuery::processSQLSecurityOption(ContextMutablePtr context_ } /// Checks the permissions for the specified definer user. - if (sql_security.definer && !skip_check_permissions) + if (sql_security.definer) { auto definer_name = sql_security.definer->toString(); if (definer_name != current_user_name) context_->checkAccess(AccessType::SET_DEFINER, definer_name); - auto & access_control = context_->getAccessControl(); - const auto user = access_control.read(definer_name); - if (access_control.isEphemeral(access_control.getID(definer_name))) + if (mode <= LoadingStrictnessLevel::CREATE) { - definer_name = user->getName() + ":definer"; - sql_security.definer = std::make_shared(definer_name); - auto new_user = typeid_cast>(user->clone()); - new_user->setName(definer_name); - new_user->authentication_methods.clear(); - new_user->authentication_methods.emplace_back(AuthenticationType::NO_AUTHENTICATION); - access_control.insertOrReplace(new_user); + auto & access_control = context_->getAccessControl(); + const auto user = access_control.read(definer_name); + if (access_control.isEphemeral(access_control.getID(definer_name))) + { + definer_name = user->getName() + ":definer"; + sql_security.definer = std::make_shared(definer_name); + auto new_user = typeid_cast>(user->clone()); + new_user->setName(definer_name); + new_user->authentication_methods.clear(); + new_user->authentication_methods.emplace_back(AuthenticationType::NO_AUTHENTICATION); + access_control.insertOrReplace(new_user); + } } } - if (sql_security.type == SQLSecurityType::NONE && !skip_check_permissions) + if (sql_security.type == SQLSecurityType::NONE) context_->checkAccess(AccessType::ALLOW_SQL_SECURITY_NONE); } diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 65e7ac5962a2..ab5f77a9fd36 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -84,8 +84,7 @@ class InterpreterCreateQuery : public IInterpreter, WithMutableContext void extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr & ast, ContextPtr) const override; /// Check access right, validate definer statement and replace `CURRENT USER` with actual name. - static void processSQLSecurityOption( - ContextMutablePtr context_, ASTSQLSecurity & sql_security, bool is_materialized_view = false, bool skip_check_permissions = false); + static void processSQLSecurityOption(ContextMutablePtr context_, ASTSQLSecurity & sql_security, bool is_materialized_view = false, LoadingStrictnessLevel mode = LoadingStrictnessLevel::CREATE); private: struct TableProperties diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 32c5ecd0a164..be8912cc183e 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -329,11 +330,11 @@ ProcessList::EntryPtr ProcessList::insert( processes.end(), query); - CancellationChecker::getInstance().appendTask(query, query_context->getSettingsRef()[Setting::max_execution_time].totalMilliseconds(), query_context->getSettingsRef()[Setting::timeout_overflow_mode]); + bool registered_in_cancellation_checker = CancellationChecker::getInstance().appendTask(query, query_context->getSettingsRef()[Setting::max_execution_time].totalMilliseconds(), query_context->getSettingsRef()[Setting::timeout_overflow_mode]); increaseQueryKindAmount(query_kind); - res = std::make_shared(*this, process_it); + res = std::make_shared(*this, process_it, registered_in_cancellation_checker); (*process_it)->setUserProcessList(&user_process_list); (*process_it)->setProcessListEntry(res); @@ -367,6 +368,14 @@ ProcessList::EntryPtr ProcessList::insert( ProcessListEntry::~ProcessListEntry() { + if (registered_in_cancellation_checker) + { + /// We need to block the overcommit tracker here to avoid lock inversion because OvercommitTracker takes a lock on the ProcessList::mutex. + /// When task is added, we lock the ProcessList::mutex, and then the CancellationChecker mutex. + OvercommitTrackerBlockerInThread blocker; + CancellationChecker::getInstance().appendDoneTasks(*it); + } + LockAndOverCommitTrackerBlocker lock(parent.getMutex()); String user = (*it)->getClientInfo().current_user; @@ -401,8 +410,6 @@ ProcessListEntry::~ProcessListEntry() if (auto query_user = parent.queries_to_user.find(query_id); query_user != parent.queries_to_user.end()) parent.queries_to_user.erase(query_user); - CancellationChecker::getInstance().appendDoneTasks(*it); - /// This removes the memory_tracker of one request. parent.processes.erase(it); diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 8b6cd93946c1..645acf0f67ea 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -343,10 +343,11 @@ class ProcessListEntry ProcessList & parent; Container::iterator it; + bool registered_in_cancellation_checker = false; public: - ProcessListEntry(ProcessList & parent_, Container::iterator it_) - : parent(parent_), it(it_) {} + ProcessListEntry(ProcessList & parent_, Container::iterator it_, bool registered_in_cancellation_checker_) + : parent(parent_), it(it_), registered_in_cancellation_checker(registered_in_cancellation_checker_) {} ~ProcessListEntry(); diff --git a/src/Interpreters/QueryNormalizer.cpp b/src/Interpreters/QueryNormalizer.cpp index 55938aab62dd..6c08016e7296 100644 --- a/src/Interpreters/QueryNormalizer.cpp +++ b/src/Interpreters/QueryNormalizer.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace DB @@ -186,7 +187,19 @@ void QueryNormalizer::visit(ASTTablesInSelectQueryElement & node, const ASTPtr & static bool needVisitChild(const ASTPtr & child) { /// exclude interpolate elements - they are not subject for normalization and will be processed in filling transform - return !(child->as() || child->as() || child->as()); + if (child->as() || child->as() || child->as()) + return false; + + /// Column transformer children (EXCEPT, REPLACE, APPLY) contain column name references + /// that must not be substituted with alias expressions. For example, in + /// `SELECT * EXCEPT (Budget), toFloat64(Budget) AS Budget FROM t`, the `Budget` inside + /// EXCEPT refers to a column name to exclude, not to the alias `Budget`. + /// If the normalizer replaces it, the downstream ASTColumnsExceptTransformer::transform + /// will encounter a non-ASTIdentifier child and throw a LOGICAL_ERROR. + if (child->as()) + return false; + + return true; } /// special visitChildren() for ASTSelectQuery diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index 374d29264984..de0b10dde1f2 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -267,7 +267,7 @@ void Set::appendSetElements(SetKeyColumns & holder) { auto filtered_column = holder.key_columns[i]->filter(holder.filter->getData(), rows); if (set_elements[i]->empty()) - set_elements[i] = filtered_column; + set_elements[i] = IColumn::mutate(std::move(filtered_column)); else set_elements[i]->insertRangeFrom(*filtered_column, 0, filtered_column->size()); if (transform_null_in && holder.null_map_holder) @@ -281,6 +281,16 @@ void Set::checkIsCreated() const throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to use set before it has been built."); } +Columns Set::getSetElements() const +{ + checkIsCreated(); + Columns result; + result.reserve(set_elements.size()); + for (const auto & col : set_elements) + result.push_back(col->getPtr()); + return result; +} + ColumnUInt8::Ptr checkDateTimePrecision(const ColumnWithTypeAndName & column_to_cast) { // Handle nullable columns diff --git a/src/Interpreters/Set.h b/src/Interpreters/Set.h index cc282c7c42ea..da617b2f2d8d 100644 --- a/src/Interpreters/Set.h +++ b/src/Interpreters/Set.h @@ -81,7 +81,7 @@ class Set bool hasExplicitSetElements() const { return fill_set_elements || (!set_elements.empty() && set_elements.front()->size() == data.getTotalRowCount()); } bool hasSetElements() const { return !set_elements.empty(); } - Columns getSetElements() const { checkIsCreated(); return { set_elements.begin(), set_elements.end() }; } + Columns getSetElements() const; void checkColumnsNumber(size_t num_key_columns) const; bool areTypesEqual(size_t set_type_idx, const DataTypePtr & other_type) const; @@ -143,7 +143,7 @@ class Set /// Collected elements of `Set`. /// It is necessary for the index to work on the primary key in the IN statement. - std::vector set_elements; + MutableColumns set_elements; /** Protects work with the set in the functions `insertFromBlock` and `execute`. * These functions can be called simultaneously from different threads only when using StorageSet, diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 05768ab19284..fe68def3d353 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -1362,7 +1362,14 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( result.analyzed_join = std::make_shared(); if (remove_duplicates) + { + Aliases aliases; + NameSet name_set; + + normalize(query, aliases, name_set, select_options.ignore_alias, settings, /* allow_self_aliases = */ true, getContext(), select_options.is_create_parameterized_view); renameDuplicatedColumns(select_query); + } + /// Perform it before analyzing JOINs, because it may change number of columns with names unique and break some logic inside JOINs if (settings[Setting::optimize_normalize_count_variants]) diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index 44279c876b05..2696b413fa1b 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -228,7 +228,12 @@ void ASTColumnsExceptTransformer::transform(ASTs & nodes) const if (!pattern) { for (const auto & child : children) - expected_columns.insert(child->as().name()); + { + if (const auto * identifier = child->as()) + expected_columns.insert(identifier->name()); + else + expected_columns.insert(child->getAliasOrColumnName()); + } for (auto * it = nodes.begin(); it != nodes.end();) { diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index 9d580827fc76..998765b665b4 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -148,6 +148,13 @@ class FunctionSecretArgumentsFinder { findYTsaurusStorageTableEngineSecretArguments(); } + else if ((function->name() == "jdbc") || (function->name() == "odbc")) + { + /// jdbc('DSN', schema, table) or jdbc('DSN', table) + /// odbc('DSN', schema, table) or odbc('DSN', table) + /// The DSN (connection string) may contain credentials. + findXDBCSecretArguments(); + } } void findMySQLFunctionSecretArguments() @@ -216,6 +223,76 @@ class FunctionSecretArgumentsFinder } } + void findXDBCSecretArguments() + { + if (isNamedCollectionName(0)) + { + /// jdbc(named_collection, ..., datasource = 'DSN', ...) + /// odbc(named_collection, ..., connection_settings = 'DSN', ...) + /// `datasource` and `connection_settings` are mutually exclusive aliases. + /// If the value is a URI, mask only the password; otherwise hide the whole value. + /// If somehow both are present (invalid query), hide all named arguments. + ssize_t ds_idx = findNamedArgument(nullptr, "datasource", 1); + ssize_t cs_idx = findNamedArgument(nullptr, "connection_settings", 1); + + if (ds_idx >= 0 && cs_idx >= 0) + { + /// Both present — hide all named arguments starting from index 1. + result.start = 1; + result.count = function->arguments->size() - 1; + result.are_named = true; + } + else if (ds_idx >= 0) + maskXDBCSecretNamedArgument("datasource", 1); + else if (cs_idx >= 0) + maskXDBCSecretNamedArgument("connection_settings", 1); + } + else + { + /// jdbc('DSN', schema, table) / jdbc('DSN', table) + /// odbc('DSN', schema, table) / odbc('DSN', table) + /// JDBC('DSN', database, table) / ODBC('DSN', database, table) + /// The connection string may be a URI with credentials embedded, + /// e.g. scheme://username:password@host:port/dbname + /// If so, mask only the password part; otherwise hide the whole argument. + String uri; + if (tryGetStringFromArgument(0, &uri)) + { + if (maskURIPassword(&uri)) + { + chassert(result.count == 0); + result.start = 0; + result.count = 1; + result.replacement = std::move(uri); + return; + } + } + markSecretArgument(0, false); + } + } + + /// Similar to `findSecretNamedArgument`, but if the value is a URI with credentials, + /// masks only the password part instead of hiding the entire value. + void maskXDBCSecretNamedArgument(std::string_view key, size_t start) + { + String value; + ssize_t arg_idx = findNamedArgument(&value, key, start); + if (arg_idx < 0) + return; + + if (!value.empty() && maskURIPassword(&value)) + { + result.are_named = true; + result.start = arg_idx; + result.count = 1; + result.replacement = std::move(value); + } + else + { + markSecretArgument(arg_idx, /* argument_is_named= */ true); + } + } + /// Returns the number of arguments excluding "headers" and "extra_credentials" (which should /// always be at the end). Marks "headers" as secret, if found. size_t excludeS3OrURLNestedMaps() @@ -540,6 +617,13 @@ class FunctionSecretArgumentsFinder { findYTsaurusStorageTableEngineSecretArguments(); } + else if ((engine_name == "JDBC") || (engine_name == "ODBC")) + { + /// JDBC('DSN', database, table) + /// ODBC('DSN', database, table) + /// The DSN (connection string) may contain credentials. + findXDBCSecretArguments(); + } } void findExternalDistributedTableEngineSecretArguments() diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 38d0b4c19efd..243fe3fcba0b 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -1706,6 +1706,10 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (!sql_security) sql_security_p.parse(pos, sql_security, expected); + /// Accept COMMENT before AS SELECT for forward compatibility with newer versions + /// that may format views as: CREATE VIEW ... COMMENT 'text' AS SELECT ... + auto comment = parseComment(pos, expected); + /// AS SELECT ... if (!s_as.ignore(pos, expected)) return false; @@ -1713,7 +1717,8 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (!select_p.parse(pos, select, expected)) return false; - auto comment = parseComment(pos, expected); + if (!comment) + comment = parseComment(pos, expected); auto query = std::make_shared(); node = query; diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index bc1f16d8d18a..2bdb0651f230 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -368,6 +369,47 @@ std::optional analyzeWindow( } } + /// When `group_by_use_nulls = 1` with CUBE/ROLLUP/GROUPING SETS, GROUP BY keys become Nullable + /// in the data flowing into window functions. But the aggregate function was created during analysis + /// with the original (non-nullable) argument types. We need to re-create the aggregate function + /// with the actual (nullable) argument types so that the Null combinator is properly applied. + for (auto & window_description : window_descriptions) + { + for (auto & window_function : window_description.window_functions) + { + bool types_changed = false; + DataTypes actual_argument_types; + actual_argument_types.reserve(window_function.argument_names.size()); + + for (size_t i = 0; i < window_function.argument_names.size(); ++i) + { + const auto * dag_node = before_window_actions->dag.tryFindInOutputs(window_function.argument_names[i]); + if (dag_node && !window_function.argument_types[i]->equals(*dag_node->result_type)) + { + actual_argument_types.push_back(dag_node->result_type); + types_changed = true; + } + else + { + actual_argument_types.push_back(window_function.argument_types[i]); + } + } + + if (types_changed) + { + AggregateFunctionProperties properties; + auto new_function = AggregateFunctionFactory::instance().get( + window_function.aggregate_function->getName(), + NullsAction::EMPTY, + actual_argument_types, + window_function.function_parameters, + properties); + window_function.aggregate_function = std::move(new_function); + window_function.argument_types = std::move(actual_argument_types); + } + } + } + ColumnsWithTypeAndName window_functions_additional_columns; for (auto & window_description : window_descriptions) diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index f9e1d0bbf4d9..783e946240bf 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -486,6 +487,12 @@ FilterDAGInfo buildFilterInfo(ASTPtr filter_expression, QueryAnalysisPass query_analysis_pass(table_expression); query_analysis_pass.run(filter_query_tree, query_context); + /// Optimize logical expressions in the filter, e.g. convert OR-chains of + /// equalities into IN (important for row policies that produce many + /// permissive conditions like `x = 1 OR x = 2 OR ... OR x = N`). + LogicalExpressionOptimizerPass logical_expression_optimizer_pass; + logical_expression_optimizer_pass.run(filter_query_tree, query_context); + return buildFilterInfo(std::move(filter_query_tree), table_expression, planner_context, std::move(table_expression_required_names_without_filter)); } diff --git a/src/Processors/Formats/Impl/Parquet/Write.cpp b/src/Processors/Formats/Impl/Parquet/Write.cpp index 2102f50f879d..352131c7ff20 100644 --- a/src/Processors/Formats/Impl/Parquet/Write.cpp +++ b/src/Processors/Formats/Impl/Parquet/Write.cpp @@ -220,14 +220,12 @@ struct StatisticsStringRef parq::Statistics s; if (min.ptr == nullptr) return s; - if (static_cast(min.len) <= options.max_statistics_size) + if (static_cast(min.len) <= options.max_statistics_size + && static_cast(max.len) <= options.max_statistics_size) { s.__set_min_value(std::string(reinterpret_cast(min.ptr), static_cast(min.len))); - s.__set_is_min_value_exact(true); - } - if (static_cast(max.len) <= options.max_statistics_size) - { s.__set_max_value(std::string(reinterpret_cast(max.ptr), static_cast(max.len))); + s.__set_is_min_value_exact(true); s.__set_is_max_value_exact(true); } return s; @@ -250,7 +248,7 @@ struct StatisticsStringRef int t = memcmp(a.ptr, b.ptr, std::min(a.len, b.len)); if (t != 0) return t; - return a.len - b.len; + return int(a.len) - int(b.len); } }; @@ -829,6 +827,10 @@ void writeColumnImpl( if (options.write_page_index) { bool all_null_page = data_count == 0; + bool has_stats = page_stats.__isset.min_value && page_stats.__isset.max_value; + if (!all_null_page && !has_stats) + s.indexes.column_index_valid = false; + s.indexes.column_index.min_values.push_back(page_stats.min_value); s.indexes.column_index.max_values.push_back(page_stats.max_value); if (has_null_count) @@ -1265,6 +1267,9 @@ static void writePageIndex(FileWriteState & file, WriteBuffer & out) chassert(rg.column_indexes.size() == rg.row_group.columns.size()); for (size_t j = 0; j < rg.column_indexes.size(); ++j) { + if (!rg.column_indexes.at(j).column_index_valid) + continue; + auto & column = rg.row_group.columns.at(j); column.__set_column_index_offset(file.offset); size_t length = serializeThriftStruct(rg.column_indexes.at(j).column_index, out); diff --git a/src/Processors/Formats/Impl/Parquet/Write.h b/src/Processors/Formats/Impl/Parquet/Write.h index bf027227e6d1..36355ac2f048 100644 --- a/src/Processors/Formats/Impl/Parquet/Write.h +++ b/src/Processors/Formats/Impl/Parquet/Write.h @@ -75,6 +75,9 @@ struct ColumnChunkIndexes { parq::ColumnIndex column_index; // if write_page_index parq::OffsetIndex offset_index; // if write_page_index + /// Set to false when a non-null page has stats dropped (e.g. value exceeded max_statistics_size). + /// When false, the column index must not be written because it would contain invalid bounds. + bool column_index_valid = true; parq::BloomFilterHeader bloom_filter_header; PODArray bloom_filter_data; // if write_bloom_filter, and not flushed yet }; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index 938f25b89fa8..c0e92c2989aa 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -344,7 +344,7 @@ const ActionsDAG::Node * addMonotonicChain(ActionsDAG & dag, const ActionsDAG::N args.push_back(&dag.addColumn({child->column, child->result_type, child->result_name})); } - return &dag.addFunction(node->function_base, std::move(args), {}); + return &dag.addFunction(node->function_base, std::move(args), node->result_name); } struct SortingInputOrder diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index bb9be23acbf0..757a37046efd 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -2040,7 +2040,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead( find_exact_ranges, query_info_.isFinal()); - MergeTreeDataSelectExecutor::filterPartsByQueryConditionCache(result.parts_with_ranges, query_info_, vector_search_parameters, context_, log); + MergeTreeDataSelectExecutor::filterPartsByQueryConditionCache(result.parts_with_ranges, query_info_, vector_search_parameters, mutations_snapshot, context_, log); if (indexes->use_skip_indexes && !indexes->skip_indexes.useful_indices.empty() && query_info_.isFinal() && settings[Setting::use_skip_indexes_if_final_exact_mode]) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 48a412ef781f..4a5502e1562b 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -602,7 +602,8 @@ void ReadFromRemote::addLazyPipe( my_scalars["_shard_num"] = Block{ {DataTypeUInt32().createColumnConst(1, my_shard.shard_info.shard_num), std::make_shared(), "_shard_num"}}; auto remote_query_executor = std::make_shared( - std::move(connections), query_string, header, my_context, my_throttler, my_scalars, my_external_tables, stage_to_use, my_shard.query_plan); + std::move(connections), query_string, header, my_context, my_throttler, my_scalars, my_external_tables, stage_to_use, + my_shard.query_plan, /*extension=*/std::nullopt, my_shard.shard_info.pool); auto pipe = createRemoteSourcePipe( remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending, parallel_marshalling_threads); diff --git a/src/Processors/Transforms/IntersectOrExceptTransform.cpp b/src/Processors/Transforms/IntersectOrExceptTransform.cpp index 180b0c11a3cc..412cd6c425d6 100644 --- a/src/Processors/Transforms/IntersectOrExceptTransform.cpp +++ b/src/Processors/Transforms/IntersectOrExceptTransform.cpp @@ -9,15 +9,11 @@ IntersectOrExceptTransform::IntersectOrExceptTransform(SharedHeader header_, Ope : IProcessor(InputPorts(2, header_), {header_}) , current_operator(operator_) { - const Names & columns = header_->getNames(); - size_t num_columns = columns.empty() ? header_->columns() : columns.size(); + size_t num_columns = header_->columns(); - key_columns_pos.reserve(columns.size()); + key_columns_pos.reserve(num_columns); for (size_t i = 0; i < num_columns; ++i) - { - auto pos = columns.empty() ? i : header_->getPositionByName(columns[i]); - key_columns_pos.emplace_back(pos); - } + key_columns_pos.emplace_back(i); } diff --git a/src/Processors/tests/gtest_write_parquet_page_index.cpp b/src/Processors/tests/gtest_write_parquet_page_index.cpp index 789f3beca67a..9674703dcbfd 100644 --- a/src/Processors/tests/gtest_write_parquet_page_index.cpp +++ b/src/Processors/tests/gtest_write_parquet_page_index.cpp @@ -367,5 +367,47 @@ TEST(Parquet, WriteParquetPageIndexArrowEncoder) /// arrow doesn't write statistics to data page headers /*expect_statistics_in_page_headers*/ false); } + +/// Regression test for https://github.com/ClickHouse/ClickHouse/issues/103039 +/// When a page has a short min and a long max (exceeding max_statistics_size=4096), +/// the column index must not be written because it would contain invalid bounds +/// (e.g. min_value="a", max_value="" which violates min <= max). +TEST(Parquet, WriteParquetPageIndexOversizedStringStats) +{ + FormatSettings format_settings; + format_settings.parquet.row_group_rows = 10000; + format_settings.parquet.use_custom_encoder = true; + format_settings.parquet.parallel_encoding = false; + format_settings.parquet.write_page_index = true; + format_settings.parquet.data_page_size = 32; + + std::vector> values; + std::vector col; + col.push_back("a"); + col.push_back(String(5000, 'z')); + values.push_back(col); + + auto source = multiColumnsSource( + {std::make_shared()}, values, 1); + String path = "/tmp/test_oversized_stats.parquet"; + writeParquet(source, format_settings, path); + + auto reader = parquet::ParquetFileReader::OpenFile(path); + auto metadata = reader->metadata(); + + ASSERT_EQ(metadata->num_row_groups(), 1); + auto row_group = metadata->RowGroup(0); + ASSERT_EQ(row_group->num_columns(), 1); + + auto column_chunk = row_group->ColumnChunk(0); + auto column_index_location = column_chunk->GetColumnIndexLocation(); + auto offset_index_location = column_chunk->GetOffsetIndexLocation(); + + ASSERT_FALSE(column_index_location.has_value()); + + ASSERT_TRUE(offset_index_location.has_value()); + ASSERT_GT(offset_index_location.value().offset, 0); + ASSERT_GT(offset_index_location.value().length, 0); +} } #endif diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index 8e2e0f0d3079..d2dafcaae6b4 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -182,10 +182,15 @@ RemoteQueryExecutor::RemoteQueryExecutor( const Tables & external_tables_, QueryProcessingStage::Enum stage_, std::shared_ptr query_plan_, - std::optional extension_) + std::optional extension_, + ConnectionPoolWithFailoverPtr pool) : RemoteQueryExecutor(query_, header_, context_, scalars_, external_tables_, stage_, std::move(query_plan_), extension_) { - create_connections = [this, connections_, throttler, extension_](AsyncCallback) mutable + /// Capture `pool` in the lambda to prevent the connection pool from being destroyed + /// while entries are still in use. The Entry objects hold raw references (via PoolEntryHelper) + /// back to the pool's internal PooledObject and PoolBase structures, so the pool must + /// outlive all Entry objects. + create_connections = [this, connections_, throttler, extension_, pool](AsyncCallback) mutable { auto res = std::make_unique(std::move(connections_), context, throttler); if (extension_ && extension_->replica_info) diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h index d309027d17ff..7464990bf4f2 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.h +++ b/src/QueryPipeline/RemoteQueryExecutor.h @@ -83,6 +83,8 @@ class RemoteQueryExecutor std::optional extension_ = std::nullopt); /// Accepts several connections already taken from pool. + /// The optional `pool` parameter keeps the connection pool alive while entries are in use, + /// preventing use-after-free when the pool would otherwise be destroyed before the entries. RemoteQueryExecutor( std::vector && connections_, const String & query_, @@ -93,7 +95,8 @@ class RemoteQueryExecutor const Tables & external_tables_ = Tables(), QueryProcessingStage::Enum stage_ = QueryProcessingStage::Complete, std::shared_ptr query_plan_ = nullptr, - std::optional extension_ = std::nullopt); + std::optional extension_ = std::nullopt, + ConnectionPoolWithFailoverPtr pool = nullptr); /// Takes a pool and gets one or several connections from it. RemoteQueryExecutor( diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index 23b73f3d823f..b52c25244ddb 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -232,7 +232,8 @@ void ReadFromCluster::initializePipeline(QueryPipelineBuilder & pipeline, const Tables(), processed_stage, nullptr, - RemoteQueryExecutor::Extension{.task_iterator = extension->task_iterator, .replica_info = std::move(replica_info)}); + RemoteQueryExecutor::Extension{.task_iterator = extension->task_iterator, .replica_info = std::move(replica_info)}, + shard_info.pool); remote_query_executor->setLogger(log); Pipe pipe{std::make_shared( diff --git a/src/Storages/Kafka/StorageKafka2.cpp b/src/Storages/Kafka/StorageKafka2.cpp index c55da9c24790..6a1a1cde5e84 100644 --- a/src/Storages/Kafka/StorageKafka2.cpp +++ b/src/Storages/Kafka/StorageKafka2.cpp @@ -206,6 +206,11 @@ void StorageKafka2::partialShutdown() task->holder->deactivate(); } is_active = false; + /// Reset the active node holder while the old ZooKeeper session is still alive (even if expired). + /// EphemeralNodeHolder stores a raw ZooKeeper reference, so resetting it here prevents a + /// use-after-free: setZooKeeper() called afterwards may free the old session, and the holder's + /// destructor would then access a dangling reference when checking zookeeper.expired(). + replica_is_active_node = nullptr; } bool StorageKafka2::activate() diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp index 15882eae0f88..79925b40c21a 100644 --- a/src/Storages/MaterializedView/RefreshTask.cpp +++ b/src/Storages/MaterializedView/RefreshTask.cpp @@ -223,6 +223,11 @@ void RefreshTask::shutdown() set_handle.reset(); view = nullptr; + + /// Wake up any threads blocked in wait(), so they can see !view and throw TABLE_IS_DROPPED. + /// Without this, wait() would block forever after deactivate() prevents the background task + /// from running (and therefore from ever notifying refresh_cv). + refresh_cv.notify_all(); } void RefreshTask::drop(ContextPtr context) @@ -395,8 +400,10 @@ void RefreshTask::wait() std::unique_lock lock(mutex); refresh_cv.wait(lock, [&] { - return state != RefreshState::Running && state != RefreshState::Scheduling && - state != RefreshState::RunningOnAnotherReplica && (state == RefreshState::Disabled || !scheduling.out_of_schedule_refresh_requested); + return !view + || (state != RefreshState::Running && state != RefreshState::Scheduling + && state != RefreshState::RunningOnAnotherReplica + && (state == RefreshState::Disabled || !scheduling.out_of_schedule_refresh_requested)); }); throw_if_error(); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index bdffdf4f2d26..72925c51dc60 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -207,6 +207,9 @@ IMergeTreeDataPart::MinMaxIndex::WrittenFiles IMergeTreeDataPart::MinMaxIndex::s void IMergeTreeDataPart::MinMaxIndex::update(const Block & block, const Names & column_names) { + if (block.rows() == 0) + return; + if (!initialized) hyperrectangle.reserve(column_names.size()); diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 7e92b0ed8c9a..4817ad41390f 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -1435,7 +1435,11 @@ bool KeyCondition::tryPrepareSetIndex( for (size_t indexes_mapping_index = 0; indexes_mapping_index < indexes_mapping_size; ++indexes_mapping_index) { - const auto & key_column_type = data_types[indexes_mapping_index]; + /// Recursively strip LowCardinality from the key column type (including inside Tuples). + /// When castColumnAccurateOrNull targets a LowCardinality type and the source value + /// is out-of-range (e.g. Int64 → LowCardinality(UInt32)), accurateOrNull produces nulls + /// that get inserted into the non-nullable ColumnUnique dictionary, crashing the server. + auto key_column_type = recursiveRemoveLowCardinality(data_types[indexes_mapping_index]); size_t set_element_index = indexes_mapping[indexes_mapping_index].tuple_index; auto set_element_type = set_types[set_element_index]; ColumnPtr set_column = set_columns[set_element_index]; diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 602c968b642e..c021809f7177 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,11 @@ namespace ErrorCodes extern const int NO_SUCH_COLUMN_IN_TABLE; } +namespace FailPoints +{ + extern const char patch_parts_reverse_column_order[]; +} + namespace { @@ -328,6 +334,17 @@ void addPatchPartsColumns( required_virtuals.insert(patch_system_columns.begin(), patch_system_columns.end()); Names patch_columns_to_read_names(patch_columns_to_read_set.begin(), patch_columns_to_read_set.end()); + + fiu_do_on(FailPoints::patch_parts_reverse_column_order, + { + /// Simulate non-deterministic NameSet iteration producing different column + /// orderings for different patches. This reproduces the bug fixed in + /// getUpdatedHeader (applyPatches.cpp) where sortColumns() normalizes order + /// before the positional assertCompatibleHeader comparison. + if (i % 2 == 1) + std::reverse(patch_columns_to_read_names.begin(), patch_columns_to_read_names.end()); + }); + result.patch_columns[i] = storage_snapshot->getColumnsByNames(options, patch_columns_to_read_names); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 58b5efae215d..a9aefdbd8822 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -9797,6 +9797,9 @@ bool MergeTreeData::supportsTrivialCountOptimization(const StorageSnapshotPtr & const auto & snapshot_data = assert_cast(*storage_snapshot->data); const auto & mutations_snapshot = snapshot_data.mutations_snapshot; + if (!mutations_snapshot) + return !settings[Setting::apply_mutations_on_fly] && !settings[Setting::apply_patch_parts]; + return !mutations_snapshot->hasDataMutations() && !mutations_snapshot->hasPatchParts(); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index c1e1165e2354..074d62d1b3ce 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include namespace DB @@ -28,6 +29,12 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int INCORRECT_FILE_NAME; + extern const int FAULT_INJECTED; +} + +namespace FailPoints +{ + extern const char wide_part_writer_fail_in_add_streams[]; } namespace @@ -193,7 +200,12 @@ void MergeTreeDataPartWriterWide::addStreams( || (settings.use_adaptive_write_buffer_for_dynamic_subcolumns && ISerialization::isDynamicSubcolumn(substream_path, substream_path.size())); query_write_settings.adaptive_write_buffer_initial_size = settings.adaptive_write_buffer_initial_size; - column_streams[stream_name] = std::make_unique>( + fiu_do_on(FailPoints::wide_part_writer_fail_in_add_streams, + { + throw Exception(ErrorCodes::FAULT_INJECTED, "Injected failure in Wide part writer addStreams"); + }); + + column_streams.emplace(stream_name, std::make_unique>( stream_name, data_part_storage, stream_name, DATA_FILE_EXTENSION, @@ -202,7 +214,7 @@ void MergeTreeDataPartWriterWide::addStreams( max_compress_block_size, marks_compression_codec, settings.marks_compress_block_size, - query_write_settings); + query_write_settings)); if (columns_to_load_marks.contains(name_and_type.name)) cached_marks.emplace(stream_name, std::make_unique()); @@ -386,7 +398,7 @@ void MergeTreeDataPartWriterWide::writeSingleMark( void MergeTreeDataPartWriterWide::flushMarkToFile(const StreamNameAndMark & stream_with_mark, size_t rows_in_mark) { - auto & stream = *column_streams[stream_with_mark.stream_name]; + auto & stream = *column_streams.at(stream_with_mark.stream_name); WriteBuffer & marks_out = stream.compress_marks ? stream.marks_compressed_hashing : stream.marks_hashing; writeBinaryLittleEndian(stream_with_mark.mark.offset_in_compressed_file, marks_out); @@ -425,7 +437,7 @@ StreamsWithMarks MergeTreeDataPartWriterWide::getCurrentMarksForColumn( if (is_offsets && offset_columns.contains(stream_name)) return; - auto & stream = *column_streams[stream_name]; + auto & stream = *column_streams.at(stream_name); /// There could already be enough data to compress into the new block. if (stream.compressed_hashing.offset() >= min_compress_block_size) @@ -820,8 +832,9 @@ void MergeTreeDataPartWriterWide::finish(bool sync) void MergeTreeDataPartWriterWide::cancel() noexcept { - for (auto & stream : column_streams) - stream.second->cancel(); + for (auto & stream : column_streams) + if (stream.second) + stream.second->cancel(); column_streams.clear(); serialization_states.clear(); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 9f6f19239945..c6e8b7bac16e 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1060,6 +1060,7 @@ void MergeTreeDataSelectExecutor::filterPartsByQueryConditionCache( RangesInDataParts & parts_with_ranges, const SelectQueryInfo & select_query_info, const std::optional & vector_search_parameters, + const MergeTreeData::MutationsSnapshotPtr & mutations_snapshot, const ContextPtr & context, LoggerPtr log) { @@ -1067,7 +1068,9 @@ void MergeTreeDataSelectExecutor::filterPartsByQueryConditionCache( if (!settings[Setting::use_query_condition_cache] || !settings[Setting::allow_experimental_analyzer] || (!select_query_info.prewhere_info && !select_query_info.filter_actions_dag) - || (vector_search_parameters.has_value())) /// vector search has filter in the ORDER BY + || (vector_search_parameters.has_value()) /// vector search has filter in the ORDER BY + || select_query_info.isFinal() + || (mutations_snapshot->hasDataMutations() || mutations_snapshot->hasPatchParts())) return; QueryConditionCachePtr query_condition_cache = context->getQueryConditionCache(); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 7be2415ed133..7d2b5ab231e3 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -215,6 +215,7 @@ class MergeTreeDataSelectExecutor RangesInDataParts & parts_with_ranges, const SelectQueryInfo & select_query_info, const std::optional & vector_search_parameters, + const MergeTreeData::MutationsSnapshotPtr & mutations_snapshot, const ContextPtr & context, LoggerPtr log); diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp index af1ba6953f81..59d6dd6e1192 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp @@ -493,7 +493,7 @@ bool MergeTreeConditionBloomFilterText::traverseTreeEquals( { if (function_name == "has" || function_name == "mapContainsKey" || function_name == "mapContains") { - out.key_column = *key_index; + out.key_column = *map_key_index; out.function = RPNElement::FUNCTION_HAS; out.bloom_filter = std::make_unique(params); auto & value = const_value.safeGet(); @@ -502,7 +502,7 @@ bool MergeTreeConditionBloomFilterText::traverseTreeEquals( } if (function_name == "mapContainsKeyLike") { - out.key_column = *key_index; + out.key_column = *map_key_index; out.function = RPNElement::FUNCTION_HAS; out.bloom_filter = std::make_unique(params); auto & value = const_value.safeGet(); diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp index 5fe191e906d7..8ec0be20960f 100644 --- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp @@ -700,9 +700,7 @@ bool MergeTreeIndexConditionSet::checkDAGUseless(const ActionsDAG::Node & node, } if (node.column && isColumnConst(*node.column)) { - Field literal; - node.column->get(0, literal); - return !atomic && literal.safeGet(); + return !atomic && node.column->getBool(0); } if (node.type == ActionsDAG::ActionType::FUNCTION) { diff --git a/src/Storages/MergeTree/PatchParts/PatchPartsUtils.cpp b/src/Storages/MergeTree/PatchParts/PatchPartsUtils.cpp index 15bda514a3d0..e00bd7be6e51 100644 --- a/src/Storages/MergeTree/PatchParts/PatchPartsUtils.cpp +++ b/src/Storages/MergeTree/PatchParts/PatchPartsUtils.cpp @@ -68,6 +68,14 @@ StorageMetadataPtr getPatchPartMetadata(ColumnsDescription patch_part_desc, Cont { StorageInMemoryMetadata part_metadata; + /// Ensure patch part system columns are present. + /// They may be missing when creating empty coverage parts + /// (e.g. DROP PART for a patch part), because createEmptyPart + /// only includes data columns from table metadata. + for (const auto & col : getPatchPartSystemColumns()) + if (!patch_part_desc.has(col.name)) + patch_part_desc.add(ColumnDescription(col.name, col.type)); + /// Use hash of column names to put patch parts with different structure to different partitions. auto part_identifier = std::make_shared("_part"); auto columns_hash = getColumnsHash(patch_part_desc.getNamesOfPhysical()); diff --git a/src/Storages/MergeTree/PatchParts/applyPatches.cpp b/src/Storages/MergeTree/PatchParts/applyPatches.cpp index 6b5149b92844..0cf987543c02 100644 --- a/src/Storages/MergeTree/PatchParts/applyPatches.cpp +++ b/src/Storages/MergeTree/PatchParts/applyPatches.cpp @@ -230,9 +230,13 @@ IColumn::Patch CombinedPatchBuilder::createPatchForColumn(const String & column_ for (const auto & patch_block : all_patch_blocks) { + const auto & patch_column = patch_block.getByName(column_name).column; + if (!patch_column) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Column {} has null data in patch block", column_name); + IColumn::Patch::Source source = { - .column = *patch_block.getByName(column_name).column, + .column = *patch_column, .versions = getColumnUInt64Data(patch_block, PartDataVersionColumn::name), }; @@ -266,12 +270,18 @@ Block getUpdatedHeader(const PatchesToApply & patches, const NameSet & updated_c for (const auto & column : patch->patch_blocks[0]) { - /// Ignore columns that are not updated. - if (!updated_columns.contains(column.name)) + /// Ignore columns that are not updated or have no data. + if (!updated_columns.contains(column.name) || !column.column) header.erase(column.name); } - headers.push_back(std::move(header)); + /// Sort columns by name so that assertCompatibleHeader below compares + /// matching columns at the same positions. Patch blocks may arrive with + /// different column orderings because addPatchPartsColumns collects names + /// from a NameSet (unordered_set) whose iteration order is non-deterministic. + /// Downstream consumers use name-based lookups, so order does not matter + /// for correctness — only for this positional compatibility check. + headers.push_back(header.sortColumns()); } for (size_t i = 1; i < headers.size(); ++i) @@ -293,7 +303,7 @@ bool canApplyPatchesRaw(const PatchesToApply & patches) { for (const auto & column : patch->patch_blocks.front()) { - if (!isPatchPartSystemColumn(column.name) && !canApplyPatchInplace(*column.column)) + if (!isPatchPartSystemColumn(column.name) && column.column && !canApplyPatchInplace(*column.column)) return false; } } @@ -325,9 +335,16 @@ void applyPatchesToBlockRaw( chassert(patch_to_apply->patch_blocks.size() == 1); const auto & patch_block = patch_to_apply->patch_blocks.front(); + if (!patch_block.has(result_column.name)) + continue; + + const auto & patch_column = patch_block.getByName(result_column.name).column; + if (!patch_column) + continue; + IColumn::Patch::Source source = { - .column = *patch_block.getByName(result_column.name).column, + .column = *patch_column, .versions = getColumnUInt64Data(patch_block, PartDataVersionColumn::name), }; diff --git a/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp b/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp index 9e8240b61955..74ddfa19ef9a 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp +++ b/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp @@ -1,5 +1,7 @@ #include +#include +#include #include #include #include @@ -30,6 +32,8 @@ void StorageFromMergeTreeProjection::read( size_t max_block_size, size_t num_streams) { + context->checkAccess(AccessType::SELECT, parent_storage->getStorageID()); + const auto & snapshot_data = assert_cast(*storage_snapshot->data); const auto & parts = snapshot_data.parts; diff --git a/src/Storages/NATS/NATSHandler.cpp b/src/Storages/NATS/NATSHandler.cpp index 5b49651b5a9a..ea89044fc482 100644 --- a/src/Storages/NATS/NATSHandler.cpp +++ b/src/Storages/NATS/NATSHandler.cpp @@ -36,6 +36,14 @@ NATSHandler::NATSHandler(LoggerPtr log_) execute_tasks_scheduler.data = this; } +NATSHandler::~NATSHandler() +{ + /// Close the async handle before UVLoop destructor runs, + /// otherwise uv_loop_close reads from already-destroyed memory. + uv_close(reinterpret_cast(&execute_tasks_scheduler), nullptr); + uv_run(loop.getLoop(), UV_RUN_NOWAIT); +} + void NATSHandler::runLoop() { { diff --git a/src/Storages/NATS/NATSHandler.h b/src/Storages/NATS/NATSHandler.h index 9268eefd39c2..8c7336d56119 100644 --- a/src/Storages/NATS/NATSHandler.h +++ b/src/Storages/NATS/NATSHandler.h @@ -22,6 +22,7 @@ class NATSHandler public: explicit NATSHandler(LoggerPtr log_); + ~NATSHandler(); /// Loop for background thread worker. void runLoop(); diff --git a/src/Storages/NATS/NATS_fwd.h b/src/Storages/NATS/NATS_fwd.h index d3af6dd12afc..f324df8a8c2a 100644 --- a/src/Storages/NATS/NATS_fwd.h +++ b/src/Storages/NATS/NATS_fwd.h @@ -13,6 +13,7 @@ using ValueMaskingFunc = std::function; static inline std::unordered_map SETTINGS_TO_HIDE = { {"nats_password", DEFAULT_MASKING_RULE}, + {"nats_token", DEFAULT_MASKING_RULE}, {"nats_credential_file", DEFAULT_MASKING_RULE}, {"nats_url", [](const DB::Field & value) { diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h index 1f01c5c12207..50552ba9c5c4 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h @@ -163,12 +163,22 @@ class DataLakeConfiguration : public BaseStorageConfiguration, public std::enabl return std::nullopt; } + bool supportsTotalRows() const override + { + return DataLakeMetadata::supportsTotalRows(); + } + std::optional totalRows(ContextPtr local_context) override { assertInitialized(); return current_metadata->totalRows(local_context); } + bool supportsTotalBytes() const override + { + return DataLakeMetadata::supportsTotalBytes(); + } + std::optional totalBytes(ContextPtr local_context) override { assertInitialized(); diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h index 962e965574b4..2e5e9a5e5955 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h @@ -76,7 +76,9 @@ class IDataLakeMetadata : boost::noncopyable virtual void modifyFormatSettings(FormatSettings &) const {} + static constexpr bool supportsTotalRows() { return false; } virtual std::optional totalRows(ContextPtr) const { return {}; } + static constexpr bool supportsTotalBytes() { return false; } virtual std::optional totalBytes(ContextPtr) const { return {}; } /// Some data lakes specify information for reading files from disks. diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h index e755c72946b8..a4c2e6d1b94f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h @@ -89,7 +89,9 @@ class IcebergMetadata : public IDataLakeMetadata IcebergHistory getHistory(ContextPtr local_context) const; + static constexpr bool supportsTotalRows() { return true; } std::optional totalRows(ContextPtr Local_context) const override; + static constexpr bool supportsTotalBytes() { return true; } std::optional totalBytes(ContextPtr Local_context) const override; ColumnMapperPtr getColumnMapperForObject(ObjectInfoPtr object_info) const override; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 7dfbee6f1672..b56a849f8866 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -458,6 +458,8 @@ void alter( metadata_json_generator.generateAddColumnMetadata(params[0].column_name, params[0].data_type); break; case AlterCommand::Type::DROP_COLUMN: + if (params[0].clear) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Clear column is not supported for iceberg. Please use UPDATE instead"); metadata_json_generator.generateDropColumnMetadata(params[0].column_name); break; case AlterCommand::Type::MODIFY_COLUMN: diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 11765fc01ad2..f8f4135c5142 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -314,6 +314,9 @@ bool StorageObjectStorage::updateExternalDynamicMetadataIfExists(ContextPtr quer std::optional StorageObjectStorage::totalRows(ContextPtr query_context) const { + if (!configuration->supportsTotalRows()) + return std::nullopt; + configuration->update( object_storage, query_context, @@ -325,6 +328,9 @@ std::optional StorageObjectStorage::totalRows(ContextPtr query_context) std::optional StorageObjectStorage::totalBytes(ContextPtr query_context) const { + if (!configuration->supportsTotalBytes()) + return std::nullopt; + configuration->update( object_storage, query_context, diff --git a/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h b/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h index 223ced7ff203..9db351a790db 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h @@ -131,7 +131,9 @@ class StorageObjectStorageConfiguration virtual bool isDataLakeConfiguration() const { return false; } + virtual bool supportsTotalRows() const { return false; } virtual std::optional totalRows(ContextPtr) { return {}; } + virtual bool supportsTotalBytes() const { return false; } virtual std::optional totalBytes(ContextPtr) { return {}; } virtual bool hasExternalDynamicMetadata() { return false; } diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 7a66fc212818..10b6c38fa7a2 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -378,6 +378,23 @@ Chunk StorageObjectStorageSource::generate() } #endif + /// Convert any Const columns to full columns before returning. + /// This is necessary because when chunks with different Const values (e.g., partition columns + /// from different files in DeltaLake) are squashed together during INSERT, the squashing code + /// doesn't properly handle merging Const columns with different constant values. + /// By converting to full columns here, we ensure the values are preserved correctly. + if (chunk.hasColumns()) + { + size_t chunk_num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + for (auto & column : columns) + { + if (column->isConst()) + column = column->cloneResized(chunk_num_rows)->convertToFullColumnIfConst(); + } + chunk.setColumns(std::move(columns), chunk_num_rows); + } + return chunk; } diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 4b8cbc4bbfba..9a7ed4ef046d 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -561,7 +562,6 @@ void StorageBuffer::read( static void appendBlock(LoggerPtr log, const Block & from, Block & to) { size_t rows = from.rows(); - size_t old_rows = to.rows(); size_t old_bytes = to.bytes(); if (to.empty()) @@ -572,7 +572,15 @@ static void appendBlock(LoggerPtr log, const Block & from, Block & to) from.checkNumberOfRows(); to.checkNumberOfRows(); + /// Take checkpoints of all destination columns before any modifications + /// to be able to rollback in case of an exception in the middle of insertion. + ColumnCheckpoints checkpoints; + checkpoints.reserve(to.columns()); + for (size_t column_no = 0; column_no < to.columns(); ++column_no) + checkpoints.push_back(to.getByPosition(column_no).column->getCheckpoint()); + MutableColumnPtr last_col; + size_t mutated_columns = 0; try { MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; @@ -598,6 +606,7 @@ static void appendBlock(LoggerPtr log, const Block & from, Block & to) LockMemoryExceptionInThread temporarily_ignore_any_memory_limits(VariableContext::Global); last_col = IColumn::mutate(std::move(to.getByPosition(column_no).column)); } + ++mutated_columns; /// In case of ColumnAggregateFunction aggregate states will /// be allocated from the query context but can be destroyed from the @@ -630,10 +639,11 @@ static void appendBlock(LoggerPtr log, const Block & from, Block & to) try { - for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) + for (size_t column_no = 0; column_no < mutated_columns; ++column_no) { ColumnPtr & col_to = to.getByPosition(column_no).column; - /// If there is no column, then the exception was thrown in the middle of append, in the insertRangeFrom() + /// If there is no column, the exception was thrown in the middle of append, + /// during insertRangeFrom() — move last_col back so we can roll it back. if (!col_to) { col_to = std::move(last_col); @@ -643,8 +653,11 @@ static void appendBlock(LoggerPtr log, const Block & from, Block & to) /// But if there is still nothing, abort if (!col_to) throw Exception(ErrorCodes::LOGICAL_ERROR, "No column to rollback"); - if (col_to->size() != old_rows) - col_to = col_to->cut(0, old_rows); + + /// Rollback to the state before the exception. + auto mutable_col = IColumn::mutate(std::move(col_to)); + mutable_col->rollback(*checkpoints[column_no]); + col_to = std::move(mutable_col); } } catch (...) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 761c152cb6da..11722898739a 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1241,7 +1241,10 @@ std::optional StorageDistributed::distributedWriteBetweenDistribu /// INSERT SELECT query returns empty block auto remote_query_executor - = std::make_shared(std::move(connections), new_query_str, std::make_shared(Block{}), query_context); + = std::make_shared( + std::move(connections), new_query_str, std::make_shared(Block{}), query_context, + /*throttler=*/nullptr, Scalars{}, Tables{}, QueryProcessingStage::Complete, + /*query_plan=*/nullptr, /*extension=*/std::nullopt, shard_info.pool); QueryPipeline remote_pipeline(std::make_shared( remote_query_executor, false, settings[Setting::async_socket_for_remote], settings[Setting::async_query_sending_for_remote])); remote_pipeline.complete(std::make_shared(remote_query_executor->getSharedHeader())); @@ -1367,7 +1370,8 @@ std::optional StorageDistributed::distributedWriteFromClusterStor Tables{}, QueryProcessingStage::Complete, nullptr, - RemoteQueryExecutor::Extension{.task_iterator = extension.task_iterator, .replica_info = std::move(replica_info)}); + RemoteQueryExecutor::Extension{.task_iterator = extension.task_iterator, .replica_info = std::move(replica_info)}, + replicas.pool); Pipe pipe{std::make_shared( remote_query_executor, diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index f77244086486..0c10675356cf 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -1085,10 +1085,11 @@ void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collec auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0); auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef()[Setting::max_compress_block_size]; + auto self = std::static_pointer_cast(shared_from_this()); auto with_retries = std::make_shared ( getLogger(fmt::format("StorageKeeperMapBackup ({})", getStorageID().getNameForLogs())), - [&] { return getClient(); }, + [self] { return self->getClient(); }, BackupKeeperSettings(backup_entries_collector.getContext()), backup_entries_collector.getContext()->getProcessListElement() ); diff --git a/src/Storages/System/StorageSystemAsynchronousInserts.cpp b/src/Storages/System/StorageSystemAsynchronousInserts.cpp index 64775988ac05..2385a67ffa30 100644 --- a/src/Storages/System/StorageSystemAsynchronousInserts.cpp +++ b/src/Storages/System/StorageSystemAsynchronousInserts.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include namespace DB { @@ -37,6 +39,9 @@ void StorageSystemAsynchronousInserts::fillData(MutableColumns & res_columns, Co if (!insert_queue) return; + const auto current_user_id = context->getUserID(); + const bool show_all = context->getAccess()->isGranted(AccessType::SHOW_USERS); + for (size_t shard_num = 0; shard_num < insert_queue->getPoolSize(); ++shard_num) { auto [queue, queue_lock] = insert_queue->getQueueLocked(shard_num); @@ -45,6 +50,9 @@ void StorageSystemAsynchronousInserts::fillData(MutableColumns & res_columns, Co { const auto & [key, data] = elem; + if (!show_all && key.user_id != current_user_id) + continue; + auto time_in_microseconds = [](const time_point & timestamp) { auto time_diff = duration_cast(steady_clock::now() - timestamp); diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index 064736620152..d85cb2413c66 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -1089,6 +1089,7 @@ const char * auto_contributors[] { "Rafael Acevedo", "Rafael David Tinoco", "Rafael Roquetto", + "Rahul", "Rajakavitha Kodhandapani", "Rajkumar", "Rajkumar Varada", diff --git a/src/Storages/System/StorageSystemJemalloc.cpp b/src/Storages/System/StorageSystemJemalloc.cpp index f486973be6d1..5706a1b7ec76 100644 --- a/src/Storages/System/StorageSystemJemalloc.cpp +++ b/src/Storages/System/StorageSystemJemalloc.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -47,12 +48,20 @@ void fillJemallocBins(MutableColumns & res_columns) auto ndalloc = getJeMallocValue(fmt::format("stats.arenas.{}.bins.{}.ndalloc", MALLCTL_ARENAS_ALL, bin).c_str()); auto nmalloc = getJeMallocValue(fmt::format("stats.arenas.{}.bins.{}.nmalloc", MALLCTL_ARENAS_ALL, bin).c_str()); + auto nregs = getJeMallocValue(fmt::format("arenas.bin.{}.nregs", bin).c_str()); + auto curslabs = getJeMallocValue(fmt::format("stats.arenas.{}.bins.{}.curslabs", MALLCTL_ARENAS_ALL, bin).c_str()); + auto curregs = getJeMallocValue(fmt::format("stats.arenas.{}.bins.{}.curregs", MALLCTL_ARENAS_ALL, bin).c_str()); + size_t col_num = 0; res_columns.at(col_num++)->insert(bin_index); res_columns.at(col_num++)->insert(0); res_columns.at(col_num++)->insert(size); res_columns.at(col_num++)->insert(nmalloc); res_columns.at(col_num++)->insert(ndalloc); + + res_columns.at(col_num++)->insert(nregs); + res_columns.at(col_num++)->insert(curslabs); + res_columns.at(col_num++)->insert(curregs); } /// Bins for large allocations @@ -69,6 +78,10 @@ void fillJemallocBins(MutableColumns & res_columns) res_columns.at(col_num++)->insert(size); res_columns.at(col_num++)->insert(nmalloc); res_columns.at(col_num++)->insert(ndalloc); + + res_columns.at(col_num++)->insertDefault(); + res_columns.at(col_num++)->insertDefault(); + res_columns.at(col_num++)->insertDefault(); } } @@ -93,14 +106,24 @@ StorageSystemJemallocBins::StorageSystemJemallocBins(const StorageID & table_id_ ColumnsDescription StorageSystemJemallocBins::getColumnsDescription() { - return ColumnsDescription + auto description = ColumnsDescription { { "index", std::make_shared(), "Index of the bin ordered by size."}, { "large", std::make_shared(), "True for large allocations and False for small."}, { "size", std::make_shared(), "Size of allocations in this bin."}, { "allocations", std::make_shared(), "Number of allocations."}, { "deallocations", std::make_shared(), "Number of deallocations."}, + { "nregs", std::make_shared(), "Number of regions per slab."}, + { "curslabs", std::make_shared(), "Current number of slabs."}, + { "curregs", std::make_shared(), "Current number of regions for this size class."}, }; + + description.setAliases({ + {"availregs", std::make_shared(), "nregs * curslabs"}, + {"util", std::make_shared(), "curregs / availregs"}, + }); + + return description; } Pipe StorageSystemJemallocBins::read( diff --git a/src/Storages/System/StorageSystemJemalloc.h b/src/Storages/System/StorageSystemJemalloc.h index 0cd29d991310..d457998b5c6f 100644 --- a/src/Storages/System/StorageSystemJemalloc.h +++ b/src/Storages/System/StorageSystemJemalloc.h @@ -6,8 +6,6 @@ namespace DB { -class Context; - class StorageSystemJemallocBins final : public IStorage { public: diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 0d6a6ab176a2..728167de8807 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -107,6 +107,9 @@ ColumnsDescription TableFunctionFile::getActualTableStructure(ContextPtr context throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema inference is not supported for table function '{}' with file descriptor", getName()); size_t total_bytes_to_read = 0; + if (context->getApplicationType() != Context::ApplicationType::LOCAL) + context->checkAccess(AccessType::READ, toStringSource(AccessTypeObjects::Source::FILE)); + Strings paths; std::optional archive_info; if (path_to_archive.empty()) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 2abb41a5d60e..b9d59854f842 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -100,7 +100,7 @@ def parse_args() -> argparse.Namespace: help="don't push reports to S3 and github", ) parser.add_argument("--push", action="store_true", help=argparse.SUPPRESS) - parser.add_argument("--os", default=["ubuntu", "alpine"], help=argparse.SUPPRESS) + parser.add_argument("--os", default=["ubuntu", "alpine", "distroless"], help=argparse.SUPPRESS) parser.add_argument( "--no-ubuntu", action=DelOS, @@ -115,6 +115,13 @@ def parse_args() -> argparse.Namespace: default=argparse.SUPPRESS, help="don't build alpine image", ) + parser.add_argument( + "--no-distroless", + action=DelOS, + nargs=0, + default=argparse.SUPPRESS, + help="don't build distroless image", + ) parser.add_argument( "--allow-build-reuse", action="store_true", @@ -229,10 +236,25 @@ def build_and_push_image( cmd_args = list(init_args) urls = [] if direct_urls: - if os == "ubuntu" and "clickhouse-server" in image.repo: - urls = [url for url in direct_urls[arch] if ".deb" in url] + # distroless and ubuntu-server use an Ubuntu builder with dpkg, so they + # need .deb packages. alpine and ubuntu-keeper use .tgz packages. + uses_deb = os == "distroless" or ( + os == "ubuntu" and "clickhouse-server" in image.repo + ) + if uses_deb: + urls = [ + url + for url in direct_urls[arch] + if ".deb" in url and "-dbg" not in url + ] else: - urls = [url for url in direct_urls[arch] if ".tgz" in url] + # For keeper/alpine tgz builds, only pass the keeper tgz. + # Excluding clickhouse-common-static.tgz avoids a large unnecessary download. + tgz_urls = [url for url in direct_urls[arch] if ".tgz" in url] + if "keeper" in image.repo: + urls = [url for url in tgz_urls if "clickhouse-keeper" in url] + else: + urls = tgz_urls cmd_args.extend( buildx_args(repo_urls, arch, direct_urls=urls, version=version.describe) ) @@ -397,7 +419,16 @@ def main(): "clickhouse-common-static", ] elif "clickhouse-keeper" in image_repo: - PACKAGES = ["clickhouse-keeper"] + # Both packages are needed to cover all three keeper image variants: + # distroless: installs from .deb via dpkg; clickhouse-common-static + # provides the clickhouse multi-tool binary (clickhouse-keeper + # is a symlink to it). clickhouse-keeper .deb is not published + # separately, so the common-static .deb is the only source. + # alpine/ubuntu: installs from .tgz; clickhouse-keeper provides the + # standalone keeper binary and its symlinks. The common-static + # .tgz is implicitly excluded because the url filter below + # keeps only urls containing "clickhouse-keeper" in the name. + PACKAGES = ["clickhouse-common-static", "clickhouse-keeper"] else: assert False, "BUG" urls = read_build_urls(build_name, Path(REPORT_PATH)) diff --git a/tests/integration/test_database_glue/test.py b/tests/integration/test_database_glue/test.py index 55ebc65e738f..de611316899f 100644 --- a/tests/integration/test_database_glue/test.py +++ b/tests/integration/test_database_glue/test.py @@ -225,6 +225,9 @@ def create_clickhouse_glue_table( """ ) + show_result = node.query(f"SHOW DATABASE {CATALOG_NAME}") + assert minio_secret_key not in show_result + def drop_clickhouse_glue_table( node, database_name, table_name ): diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 29b83000108d..a772b390dd31 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -145,7 +145,6 @@ def create_clickhouse_iceberg_database( assert minio_secret_key not in show_result assert "HIDDEN" in show_result - def create_clickhouse_iceberg_table( started_cluster, node, database_name, table_name, schema, additional_settings={} ): diff --git a/tests/integration/test_keeper_read_during_close/__init__.py b/tests/integration/test_keeper_read_during_close/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_keeper_read_during_close/configs/enable_keeper.xml b/tests/integration/test_keeper_read_during_close/configs/enable_keeper.xml new file mode 100644 index 000000000000..d171039aedc9 --- /dev/null +++ b/tests/integration/test_keeper_read_during_close/configs/enable_keeper.xml @@ -0,0 +1,30 @@ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + * + + + 10000 + 3000 + 2000 + 500 + trace + + + 0 + 0 + 0 + + + + + 1 + localhost + 9234 + + + + diff --git a/tests/integration/test_keeper_read_during_close/test.py b/tests/integration/test_keeper_read_during_close/test.py new file mode 100644 index 000000000000..5fc672b9aa2c --- /dev/null +++ b/tests/integration/test_keeper_read_during_close/test.py @@ -0,0 +1,235 @@ +"""Test that read requests from session B are not silently dropped +when the write request they were batched with belongs to session A +and session A expires before the batch commits. + +The bug: read_request_queue keys reads by (sessionA.id, sessionA.xid). +When sessionA expires, finishSession erases read_request_queue[sessionA.id], +silently dropping sessionB's read. SessionB's get() then hangs until timeout. +""" + +import logging +import queue +import socket +import struct +import threading +import time +import traceback +import os + +import pytest +from kazoo.client import KazooClient + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/enable_keeper.xml"], + stay_alive=True, +) + + +class MicroClient: + @staticmethod + def make_connect_request(timeout_ms=10000): + """ConnectRequest: protocol_version(4) + last_zxid(8) + timeout(4) + session_id(8) + passwd_len(4) + passwd(16) + readonly(1)""" + return struct.pack('>iqiqi 16s', + 0, # protocol version + 0, # last zxid seen + timeout_ms, # requested timeout + 0, # session id (0 = new session) + 16, # passwd length + b'\x00' * 16, # passwd + ) + + @staticmethod + def make_set_request(xid, path: bytes, data: bytes, version=-1): + body = struct.pack('>ii', xid, 5) # xid + opcode 5 = SetData + body += struct.pack('>i', len(path)) + path.encode() + body += struct.pack('>i', len(data)) + data + body += struct.pack('>i', version) + return body + + @staticmethod + def make_frame(payload: bytes) -> bytes: + return struct.pack('>i', len(payload)) + payload + + @staticmethod + def recv_exactly(sock, n): + """Receive exactly n bytes or raise.""" + buf = b'' + while len(buf) < n: + chunk = sock.recv(n - len(buf)) + if not chunk: + raise ConnectionError(f"Connection closed after {len(buf)}/{n} bytes") + buf += chunk + return buf + + @staticmethod + def recv_frame(sock): + """Read a length-prefixed ZK frame.""" + length = struct.unpack('>i', MicroClient.recv_exactly(sock, 4))[0] + return MicroClient.recv_exactly(sock, length) + + @staticmethod + def parse_connect_response(data): + # ConnectResponse: protocol_version(4) + timeout(4) + session_id(8) + passwd_len(4) + passwd(variable) + proto_ver, timeout, session_id = struct.unpack_from('>iiq', data, 0) + passwd_len = struct.unpack_from('>i', data, 16)[0] + passwd = data[20:20 + passwd_len] + return { + 'protocol_version': proto_ver, + 'timeout': timeout, + 'session_id': hex(session_id), + 'passwd': passwd.hex(), + } + + @staticmethod + def parse_reply_header(data): + # ReplyHeader: xid(4) + zxid(8) + err(4) + xid, zxid, err = struct.unpack_from('>iqi', data, 0) + return { + 'xid': xid, + 'zxid': zxid, + 'err': err, + } + + @staticmethod + def parse_stat(data, offset=0): + """Parse a ZK Stat structure (10 fields, 8+8+8+8+4+4+4+8+4+4+8 = 68 bytes).""" + fields = struct.unpack_from('>qqqqiiiqiiq', data, offset) + names = ['czxid', 'mzxid', 'ctime', 'mtime', 'version', 'cversion', + 'aversion', 'ephemeralOwner', 'dataLength', 'numChildren', 'pzxid'] + return dict(zip(names, fields)) + + def __init__(self, hostname, port): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(5.0) + self.sock.connect((hostname, port)) + self.next_xid = 1; + + self.sock.sendall(MicroClient.make_frame(MicroClient.make_connect_request())) + + connect_resp = MicroClient.recv_frame(self.sock) + parsed = MicroClient.parse_connect_response(connect_resp) + + if int(parsed['session_id'], 16) == 0: + raise Exception("Server rejected session (session_id=0)") + + def send_set_request(self, path, data, version=-1): + xid = self.next_xid + self.next_xid += 1 + self.sock.sendall(MicroClient.make_frame(MicroClient.make_set_request(xid, path, data, version))) + return xid + + def recv_set_response(self): + set_resp = MicroClient.recv_frame(self.sock) + header = MicroClient.parse_reply_header(set_resp) + return header + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def get_zk(timeout=30.0): + return keeper_utils.get_fake_zk(cluster, "node", timeout=timeout) + + +def kill_session_socket(zk): + """Forcefully kill the underlying TCP connection without sending Close. + This causes the session to expire after session_timeout_ms.""" + zk._connection._socket.close() + +def test_read_not_dropped_on_session_close(started_cluster): + """Verify that reads from one session are not silently lost + when a concurrent writer session dies.""" + + reader_zk = get_zk() + reader_zk.create("/test_read_close", b"initial") + + # Sanity-check our custom keeper client implementation. + hostname = cluster.get_instance_ip("node") + port = 9181 + temp_zk = MicroClient(hostname, port) + xid = temp_zk.send_set_request("/test_read_close", b"micro") + resp = temp_zk.recv_set_response() + assert resp["xid"] == xid + assert resp["err"] == 0 + + assert reader_zk.get("/test_read_close")[0] == b"micro" + + logging.getLogger('kazoo').setLevel(logging.WARNING) + + fail_event = threading.Event() + stop_event = threading.Event() + errors = queue.Queue() + + def reader_loop(): + """Continuously read /test_read_close. Each get() should complete + within a reasonable time. If it hangs, that's the bug.""" + counter = 0 + try: + zk = get_zk(timeout=10.0) + while not stop_event.is_set(): + zk.get("/test_read_close") + counter += 1 + except: + errors.put(traceback.format_exc()) + fail_event.set() + # This prints around 2200 (on my machine, as of the time of writing, with 10 second test duration). + print(f"sent {counter} read requests") + + def writer_loop(): + """Rapidly create sessions that write and then die (raw socket close). + The intent is that the write and a concurrent read from reader_loop + end up in the same batch, keyed under this writer session's identity.""" + counter = 0 + try: + while not stop_event.is_set(): + counter += 1 + writer_zk = MicroClient(hostname, port) + writer_zk.send_set_request("/test_read_close", f"data_{counter}".encode()) + # Kill the TCP socket without sending Close. + # The session will expire after session_timeout_ms (3s). + writer_zk.sock.close() + + # assert writer_zk.recv_set_response()["err"] == 0 + except: + errors.put(traceback.format_exc()) + fail_event.set() + # This prints around 1800 (on my machine, as of the time of writing, with 10 second test duration). + print(f"sent {counter} write requests") + + + # Run the reader and writer concurrently + reader_thread = threading.Thread(target=reader_loop, daemon=True) + writer_thread = threading.Thread(target=writer_loop, daemon=True) + + reader_thread.start() + writer_thread.start() + + # Run for a few seconds — enough for many session create/expire cycles + # given session_timeout_ms=3000, dead_session_check_period_ms=500. + fail_event.wait(10) + + stop_event.set() + reader_thread.join(timeout=10) + writer_thread.join(timeout=10) + + if fail_event.is_set(): + raise Exception(errors.get(block=False)) + + assert not reader_thread.is_alive(), "Reader thread is stuck" + assert not writer_thread.is_alive(), "Writer thread is stuck" + + # Cleanup + reader_zk.delete("/test_read_close") + reader_zk.stop() + reader_zk.close() diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index cf7437be8ea5..1ca836c0e406 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -292,6 +292,17 @@ def test_create_table(): f"Kafka() SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '{password}', format_avro_schema_registry_url = 'http://schema_user:{password}@domain.com'", f"S3('http://minio1:9001/root/data/test5.csv.gz', 'CSV', access_key_id = 'minio', secret_access_key = '{password}', compression_method = 'gzip')", f"Redis('localhost', 0, '{password}') PRIMARY KEY x;", + f"JDBC('DSN=mydb;Uid=user;Pwd={password}', 'mydb', 'mytable')", + f"ODBC('DSN=mydb;Uid=user;Pwd={password}', 'mydb', 'mytable')", + f"JDBC('jdbc://user:{password}@localhost:5432/mydb', 'mydb', 'mytable')", + f"ODBC('odbc://user:{password}@localhost:5432/mydb', 'mydb', 'mytable')", + f"JDBC(named_collection_1, datasource = 'DSN=mydb;Uid=user;Pwd={password}', external_database = 'mydb', external_table = 'mytable')", + f"ODBC(named_collection_1, connection_settings = 'DSN=mydb;Uid=user;Pwd={password}', external_database = 'mydb', external_table = 'mytable')", + f"JDBC(named_collection_1, datasource = 'jdbc://user:{password}@localhost:5432/mydb', external_database = 'mydb', external_table = 'mytable')", + f"ODBC(named_collection_1, connection_settings = 'odbc://user:{password}@localhost:5432/mydb', external_database = 'mydb', external_table = 'mytable')", + (f"JDBC(named_collection_1, datasource = 'DSN=mydb;Uid=user;Pwd={password}', connection_settings = 'DSN=mydb2;Uid=user2;Pwd={password}', external_database = 'mydb', external_table = 'mytable')", "ARGUMENTS"), + (f"JDBC(named_collection_1, connection_settings = 'jdbc://user2:{password}@localhost:5432/mydb2', external_database = 'mydb', datasource = 'jdbc://user:{password}@localhost:5432/mydb', external_table = 'mytable')", "ARGUMENTS"), + (f"NATS() SETTINGS nats_url = 'localhost:4222', nats_subjects = 'subject', nats_format = 'JSONEachRow', nats_token = '{password}'", "CANNOT_CONNECT_NATS"), ] def make_test_case(i): @@ -372,6 +383,17 @@ def make_test_case(i): "CREATE TABLE table34 (`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://schema_user:[HIDDEN]@domain.com'", "CREATE TABLE table35 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test5.csv.gz', 'CSV', access_key_id = 'minio', secret_access_key = '[HIDDEN]', compression_method = 'gzip')", "CREATE TABLE table36 (`x` int) ENGINE = Redis('localhost', 0, '[HIDDEN]') PRIMARY KEY x", + "CREATE TABLE table37 (`x` int) ENGINE = JDBC('[HIDDEN]', 'mydb', 'mytable')", + "CREATE TABLE table38 (`x` int) ENGINE = ODBC('[HIDDEN]', 'mydb', 'mytable')", + "CREATE TABLE table39 (`x` int) ENGINE = JDBC('jdbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')", + "CREATE TABLE table40 (`x` int) ENGINE = ODBC('odbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')", + "CREATE TABLE table41 (`x` int) ENGINE = JDBC(named_collection_1, datasource = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')", + "CREATE TABLE table42 (`x` int) ENGINE = ODBC(named_collection_1, connection_settings = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')", + "CREATE TABLE table43 (`x` int) ENGINE = JDBC(named_collection_1, datasource = 'jdbc://user:[HIDDEN]@localhost:5432/mydb', external_database = 'mydb', external_table = 'mytable')", + "CREATE TABLE table44 (`x` int) ENGINE = ODBC(named_collection_1, connection_settings = 'odbc://user:[HIDDEN]@localhost:5432/mydb', external_database = 'mydb', external_table = 'mytable')", + "CREATE TABLE table45 (`x` int) ENGINE = JDBC(named_collection_1, datasource = '[HIDDEN]', connection_settings = '[HIDDEN]', external_database = '[HIDDEN]', external_table = '[HIDDEN]')", + "CREATE TABLE table46 (`x` int) ENGINE = JDBC(named_collection_1, connection_settings = '[HIDDEN]', external_database = '[HIDDEN]', datasource = '[HIDDEN]', external_table = '[HIDDEN]')", + "CREATE TABLE table47 (`x` int) ENGINE = NATS SETTINGS nats_url = 'localhost:4222', nats_subjects = 'subject', nats_format = 'JSONEachRow', nats_token = '[HIDDEN]'", ], must_not_contain=[password], ) @@ -491,7 +513,15 @@ def test_table_functions(): f"icebergAzure('{azure_storage_account_url}', 'cont', 'test_simple_6.csv', '{azure_account_name}', '{azure_account_key}', 'CSV', 'none', 'auto')", f"deltaLakeAzure('{azure_storage_account_url}', 'cont', 'test_simple_6.csv', '{azure_account_name}', '{azure_account_key}', 'CSV', 'none', 'auto')", f"hudi('http://minio1:9001/root/data/test7.csv', 'minio', '{password}')", - f"redis('localhost', 'key', 'key Int64', 0, '{password}')" + f"redis('localhost', 'key', 'key Int64', 0, '{password}')", + f"jdbc('DSN=mydb;Uid=user;Pwd={password}', 'mydb', 'mytable')", + f"odbc('DSN=mydb;Uid=user;Pwd={password}', 'mydb', 'mytable')", + f"jdbc('jdbc://user:{password}@localhost:5432/mydb', 'mydb', 'mytable')", + f"odbc('odbc://user:{password}@localhost:5432/mydb', 'mydb', 'mytable')", + f"jdbc(named_collection_1, datasource = 'DSN=mydb;Uid=user;Pwd={password}')", + f"odbc(named_collection_1, connection_settings = 'DSN=mydb;Uid=user;Pwd={password}')", + f"jdbc(named_collection_1, datasource = 'jdbc://user:{password}@localhost:5432/mydb')", + f"odbc(named_collection_1, connection_settings = 'odbc://user:{password}@localhost:5432/mydb')", ] def make_test_case(i): @@ -578,6 +608,14 @@ def make_test_case(i): f"CREATE TABLE tablefunc43 (`x` int) AS deltaLakeAzure('{azure_storage_account_url}', 'cont', 'test_simple_6.csv', '{azure_account_name}', '[HIDDEN]', 'CSV', 'none', 'auto')", "CREATE TABLE tablefunc44 (`x` int) AS hudi('http://minio1:9001/root/data/test7.csv', 'minio', '[HIDDEN]')", "CREATE TABLE tablefunc45 (`x` int) AS redis('localhost', 'key', 'key Int64', 0, '[HIDDEN]')", + "CREATE TABLE tablefunc46 (`x` int) AS jdbc('[HIDDEN]', 'mydb', 'mytable')", + "CREATE TABLE tablefunc47 (`x` int) AS odbc('[HIDDEN]', 'mydb', 'mytable')", + "CREATE TABLE tablefunc48 (`x` int) AS jdbc('jdbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')", + "CREATE TABLE tablefunc49 (`x` int) AS odbc('odbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')", + "CREATE TABLE tablefunc50 (`x` int) AS jdbc(named_collection_1, datasource = '[HIDDEN]')", + "CREATE TABLE tablefunc51 (`x` int) AS odbc(named_collection_1, connection_settings = '[HIDDEN]')", + "CREATE TABLE tablefunc52 (`x` int) AS jdbc(named_collection_1, datasource = 'jdbc://user:[HIDDEN]@localhost:5432/mydb')", + "CREATE TABLE tablefunc53 (`x` int) AS odbc(named_collection_1, connection_settings = 'odbc://user:[HIDDEN]@localhost:5432/mydb')", ], must_not_contain=[password], ) diff --git a/tests/integration/test_scheduler_cpu_preemptive/test.py b/tests/integration/test_scheduler_cpu_preemptive/test.py index 334af94517a2..93322653a0f6 100644 --- a/tests/integration/test_scheduler_cpu_preemptive/test.py +++ b/tests/integration/test_scheduler_cpu_preemptive/test.py @@ -36,6 +36,9 @@ def start_cluster(): def clear_workloads_and_resources(): node.query( f""" + drop workload if exists production2; + drop workload if exists development2; + drop workload if exists staging; drop workload if exists production; drop workload if exists development; drop workload if exists admin; @@ -182,6 +185,16 @@ def assert_query(node, query_id, slots): "ConcurrencyControlDownscales", lambda x: x == 0, ) + # Verify ConcurrencyControlWaitMicroseconds is populated at query level. + # With independent pools all running concurrently, each query will experience + # scheduler wait time > 0. This serves as a smoke test for the wait timer fix + # (the metric is tracked at ThreadGroup level, not per-thread). + assert_profile_event( + node, + query_id, + "ConcurrencyControlWaitMicroseconds", + lambda x: x > 0, + ) # NOTE: checking thread_ids length is pointless, because query could downscale and then upscale again, gaining more threads than slots assert_query(node, 'test_production', 15) @@ -191,10 +204,12 @@ def assert_query(node, query_id, slots): # For debugging purposes LOG = [] +LOG_LOCK = threading.Lock() def mylog(message: str, *args) -> None: # Format a human-readable timestamp and append a tuple to LOG timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) - LOG.append((timestamp, message, args)) + with LOG_LOCK: + LOG.append((timestamp, message, args)) class QueryPool: @@ -204,9 +219,12 @@ def __init__(self, num_queries: int, workload: str) -> None: self.stop_event: threading.Event = threading.Event() self.threads: list[threading.Thread] = [] self.stopped: bool = True + self.errors: int = 0 + self._errors_lock: threading.Lock = threading.Lock() def start_random(self, max_billions: int, max_threads: int) -> None: assert self.stopped, "Pool is already running" + self.stopped = False def query_thread() -> None: while not self.stop_event.is_set(): @@ -223,6 +241,7 @@ def query_thread() -> None: def start_fixed(self, billions: int, max_threads: int) -> None: assert self.stopped, "Pool is already running" + self.stopped = False def query_thread() -> None: while not self.stop_event.is_set(): @@ -237,6 +256,41 @@ def query_thread() -> None: for thread in self.threads: thread.start() + def start_fixed_stop_on_error(self, billions: int, max_threads: int) -> None: + assert self.stopped, "Pool is already running" + self.stopped = False + + def query_thread() -> None: + while not self.stop_event.is_set(): + mylog(f"Running query in workload {self.workload}") + try: + node.query( + f"with (select {billions * 1000000000})::UInt64 as n select count(*) from numbers_mt(n) settings " + f"workload='{self.workload}', max_threads={max_threads}" + ) + except QueryRuntimeException as e: + mylog(f"Query in workload {self.workload} failed with exception: {e}") + with self._errors_lock: + self.errors += 1 + + for _ in range(self.num_queries): + self.threads.append(threading.Thread(target=query_thread, args=())) + for thread in self.threads: + thread.start() + + def get_errors(self) -> int: + with self._errors_lock: + return self.errors + + def wait_for_all_errors(self) -> None: + """Wait until all threads have failed with errors, then stop the pool.""" + while True: + with self._errors_lock: + if self.errors >= self.num_queries: + break + time.sleep(0.01) + self.stop() + def stop(self) -> None: mylog(f"Stopping workload {self.workload}") self.stop_event.set() @@ -416,3 +470,63 @@ def test_downscaling(with_custom_config): finally: for tid in active_ids: development.stop(tid) + + +def test_create_workload_under_load(): + """Test that creating a WORKLOAD while queries are running does not cause crashes or deadlocks.""" + node.query( + f""" + create resource cpu (master thread, worker thread); + create workload all settings max_concurrent_threads=3; + create workload production in all settings weight=1; + create workload development in all settings weight=1, max_cpus=1; + """ + ) + + production = QueryPool(2, "production") + production.start_fixed_stop_on_error(1, 2) + development = QueryPool(2, "development") + development.start_fixed_stop_on_error(1, 2) + time.sleep(1) + + assert production.get_errors() == 0, "Errors occurred in production workload" + assert development.get_errors() == 0, "Errors occurred in development workload" + + # Try to create a new workload while the queries are running + # This is sibling workload, so it should not affect existing queries + node.query( + f"create workload staging in all settings weight=2, max_cpus=1;" + ) + time.sleep(1) + assert production.get_errors() == 0, "Errors occurred in production workload" + assert development.get_errors() == 0, "Errors occurred in development workload" + + # This make production non-usable, as it will be not a leaf workload anymore + node.query( + f"create workload production2 in production;" + ) + time.sleep(1) + production.wait_for_all_errors() + assert development.get_errors() == 0, "Errors occurred in development workload" + + # Restart load inside new workload + production2 = QueryPool(2, "production2") + production2.start_fixed_stop_on_error(1, 2) + + # This make development non-usable, as it will be not a leaf workload anymore + node.query( + f"create workload development2 in development;" + ) + time.sleep(1) + development.wait_for_all_errors() + assert production2.get_errors() == 0, "Errors occurred in production2 workload" + + # Restart load inside new workload + development2 = QueryPool(2, "development2") + development2.start_fixed_stop_on_error(1, 2) + + # Cleanup + production2.stop() + development2.stop() + assert development2.get_errors() == 0, "Errors occurred in development2 workload" + assert production2.get_errors() == 0, "Errors occurred in production2 workload" diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py index 6241ed6ecd8a..26319b946c57 100644 --- a/tests/integration/test_storage_delta/test.py +++ b/tests/integration/test_storage_delta/test.py @@ -3569,3 +3569,129 @@ def test_write_column_order(started_cluster): ) assert num_rows * 2 == int(instance.query(f"SELECT count() FROM {table_name}")) + + +def test_network_activity_with_system_tables(started_cluster): + instance = started_cluster.instances["node1"] + bucket = started_cluster.minio_bucket + table_name = randomize_table_name("test_network_activity_with_system_tables") + result_file = f"{table_name}_data" + + schema = pa.schema([("id", pa.int32(), False), ("name", pa.string(), False)]) + empty_arrays = [pa.array([], type=pa.int32()), pa.array([], type=pa.string())] + write_deltalake( + f"s3://root/{result_file}", + pa.Table.from_arrays(empty_arrays, schema=schema), + storage_options=get_storage_options(started_cluster), + mode="overwrite", + ) + + instance.query( + f""" + CREATE TABLE {table_name} (id Int32, name String) ENGINE = DeltaLake('http://{started_cluster.minio_ip}:{started_cluster.minio_port}/{bucket}/{result_file}/', 'minio', '{minio_secret_key}') + """ + ) + + instance.query( + f"INSERT INTO {table_name} SELECT number as name, toString(number) as id from numbers(10)" + ) + + query_id = f"{table_name}_query" + instance.query( + f"SELECT * FROM system.tables WHERE name = '{table_name}'", query_id=query_id + ) + + instance.query("SYSTEM FLUSH LOGS text_log") + + assert 0 == int( + instance.query( + f"SELECT count() FROM system.text_log WHERE query_id = '{query_id}' AND message LIKE '%Initialized scan state%'" + ) + ) + + +@pytest.mark.parametrize("cluster", [False, True]) +def test_partition_columns_3(started_cluster, cluster): + """Test for bug https://github.com/ClickHouse/ClickHouse/issues/95526 + + Reproduces issue where partition column values become incorrect when inserting + from DeltaLake into ClickHouse with many columns and type conversions. + """ + node = started_cluster.instances["node1"] + table_name = randomize_table_name("test_partition_columns_jumbled") + + schema = pa.schema( + [ + ("id", pa.int32()), + ("region", pa.string()), + ("state", pa.string()), + ] + ) + + data = [ + pa.array([1, 2], type=pa.int32()), + pa.array(["west", "east"], type=pa.string()), + pa.array(["CA", "NY"], type=pa.string()), + ] + + storage_options = { + "AWS_ENDPOINT_URL": f"http://{started_cluster.minio_ip}:{started_cluster.minio_port}", + "AWS_ACCESS_KEY_ID": minio_access_key, + "AWS_SECRET_ACCESS_KEY": minio_secret_key, + "AWS_ALLOW_HTTP": "true", + "AWS_S3_ALLOW_UNSAFE_RENAME": "true", + } + path = f"s3://root/{table_name}" + table = pa.Table.from_arrays(data, schema=schema) + + write_deltalake( + path, table, storage_options=storage_options, partition_by=["region", "state"] + ) + + if cluster: + delta_function = f""" + deltaLakeCluster( + cluster, + 'http://{started_cluster.minio_ip}:{started_cluster.minio_port}/root/{table_name}' , + '{minio_access_key}', + '{minio_secret_key}') + """ + else: + delta_function = f""" + deltaLake( + 'http://{started_cluster.minio_ip}:{started_cluster.minio_port}/root/{table_name}' , + '{minio_access_key}', + '{minio_secret_key}', + SETTINGS allow_experimental_delta_kernel_rs=0) + """ + + dst_table = f"{table_name}_dst" + node.query(f""" + CREATE TABLE {dst_table} ( + id Int32, + region String, + state String + ) ENGINE = MergeTree() + ORDER BY id + """) + + node.query(f""" + INSERT INTO {dst_table} + SELECT * FROM {delta_function} + """) + + result_from_delta = node.query( + f"SELECT * FROM {delta_function} ORDER BY id", + settings={"allow_experimental_delta_kernel_rs": 1, "use_hive_partitioning": 0}, + ).strip() + + result_from_table = node.query( + f"SELECT * FROM {dst_table} ORDER BY id" + ).strip() + + assert result_from_delta == result_from_table, \ + f"Partition columns jumbled!\nFrom DeltaLake:\n{result_from_delta}\n\nFrom table:\n{result_from_table}" + + expected = "1\twest\tCA\n2\teast\tNY" + assert result_from_table == expected, \ + f"Data doesn't match!\nExpected:\n{expected}\n\nGot:\n{result_from_table}" diff --git a/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference b/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference index 548952c3a6a6..8a079f11d9ca 100644 --- a/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference +++ b/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference @@ -1 +1 @@ -CREATE TABLE default.BannerDict\n(\n `BannerID` UInt64,\n `CompaignID` UInt64\n)\nENGINE = ODBC(\'DSN=pgconn;Database=postgres\', \'somedb\', \'bannerdict\') +CREATE TABLE default.BannerDict\n(\n `BannerID` UInt64,\n `CompaignID` UInt64\n)\nENGINE = ODBC(\'[HIDDEN]\', \'somedb\', \'bannerdict\') diff --git a/tests/queries/0_stateless/02253_empty_part_checksums.reference b/tests/queries/0_stateless/02253_empty_part_checksums.reference index 65a8c9ee65e7..d5a418ff4619 100644 --- a/tests/queries/0_stateless/02253_empty_part_checksums.reference +++ b/tests/queries/0_stateless/02253_empty_part_checksums.reference @@ -5,4 +5,4 @@ 0 1 0 -0_0_0_0 Wide 370db59d5dcaef5d762b11d319c368c7 514a8be2dac94fd039dbd230065e58a4 b324ada5cd6bb14402c1e59200bd003a +0_0_0_0 Wide 85adbaf60cad8c08f040d4cb27830cf4 e73297470a3016870e8f281b48b2dd68 b324ada5cd6bb14402c1e59200bd003a diff --git a/tests/queries/0_stateless/02374_analyzer_join_using.reference b/tests/queries/0_stateless/02374_analyzer_join_using.reference index e83f8f37aba6..63e648c14d68 100644 --- a/tests/queries/0_stateless/02374_analyzer_join_using.reference +++ b/tests/queries/0_stateless/02374_analyzer_join_using.reference @@ -102,12 +102,12 @@ SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeN FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (test_value); -- { serverError UNKNOWN_IDENTIFIER } SELECT 'First JOIN INNER second JOIN INNER'; First JOIN INNER second JOIN INNER -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING (id) INNER JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -123,12 +123,12 @@ SELECT 1 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING ( SELECT id FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id INNER JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN INNER second JOIN LEFT'; First JOIN INNER second JOIN LEFT -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING (id) LEFT JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -144,13 +144,13 @@ SELECT 1 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING ( SELECT id FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id LEFT JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN INNER second JOIN RIGHT'; First JOIN INNER second JOIN RIGHT -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING (id) RIGHT JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String -4 UInt64 0 UInt64 String 0 UInt64 String 4 UInt64 Join_3_Value_4 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String +4 UInt64 1 UInt64 String 1 UInt32 String 5 UInt64 Join_3_Value_4 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -168,13 +168,13 @@ SELECT 1 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING ( SELECT id FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id RIGHT JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN INNER second JOIN FULL'; First JOIN INNER second JOIN FULL -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING (id) FULL JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 String 0 UInt64 String 4 UInt64 Join_3_Value_4 String -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String +0 UInt64 1 UInt64 String 1 UInt32 String 5 UInt64 Join_3_Value_4 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -192,12 +192,12 @@ SELECT 1 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING ( SELECT id FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id FULL JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN LEFT second JOIN INNER'; First JOIN LEFT second JOIN INNER -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (id) INNER JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -213,13 +213,13 @@ SELECT 1 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (i SELECT id FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id INNER JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN LEFT second JOIN LEFT'; First JOIN LEFT second JOIN LEFT -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (id) LEFT JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String -2 UInt64 2 UInt64 Join_1_Value_2 String 0 UInt64 String 0 UInt64 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String +2 UInt64 3 UInt64 Join_1_Value_2 String 1 UInt32 String 1 UInt64 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -237,13 +237,13 @@ SELECT 1 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (i SELECT id FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id LEFT JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN LEFT second JOIN RIGHT'; First JOIN LEFT second JOIN RIGHT -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (id) RIGHT JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String -4 UInt64 0 UInt64 String 0 UInt64 String 4 UInt64 Join_3_Value_4 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String +4 UInt64 1 UInt64 String 1 UInt32 String 5 UInt64 Join_3_Value_4 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -261,14 +261,14 @@ SELECT 1 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (i SELECT id FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id RIGHT JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN LEFT second JOIN FULL'; First JOIN LEFT second JOIN FULL -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (id) FULL JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 String 0 UInt64 String 4 UInt64 Join_3_Value_4 String -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String -2 UInt64 2 UInt64 Join_1_Value_2 String 0 UInt64 String 0 UInt64 String +0 UInt64 1 UInt64 String 1 UInt32 String 5 UInt64 Join_3_Value_4 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String +2 UInt64 3 UInt64 Join_1_Value_2 String 1 UInt32 String 1 UInt64 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -288,12 +288,12 @@ SELECT 1 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (i SELECT id FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id FULL JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN RIGHT second JOIN INNER'; First JOIN RIGHT second JOIN INNER -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING (id) INNER JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String +0 UInt64 1 UInt32 Join_1_Value_0 String 1 UInt64 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt32 Join_1_Value_1 String 2 UInt64 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -309,13 +309,13 @@ SELECT 1 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING ( SELECT id FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id INNER JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN RIGHT second JOIN LEFT'; First JOIN RIGHT second JOIN LEFT -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING (id) LEFT JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String -3 UInt64 0 UInt64 String 3 UInt64 Join_2_Value_3 String 0 UInt64 String +0 UInt64 1 UInt32 Join_1_Value_0 String 1 UInt64 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt32 Join_1_Value_1 String 2 UInt64 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String +3 UInt64 1 UInt32 String 4 UInt64 Join_2_Value_3 String 1 UInt64 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -333,13 +333,13 @@ SELECT 1 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING ( SELECT id FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id LEFT JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN RIGHT second JOIN RIGHT'; First JOIN RIGHT second JOIN RIGHT -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING (id) RIGHT JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String -4 UInt64 0 UInt64 String 0 UInt64 String 4 UInt64 Join_3_Value_4 String +0 UInt64 1 UInt32 Join_1_Value_0 String 1 UInt64 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt32 Join_1_Value_1 String 2 UInt64 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String +4 UInt64 1 UInt32 String 1 UInt64 String 5 UInt64 Join_3_Value_4 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -357,14 +357,14 @@ SELECT 1 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING ( SELECT id FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id RIGHT JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN RIGHT second JOIN FULL'; First JOIN RIGHT second JOIN FULL -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING (id) FULL JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 String 0 UInt64 String 4 UInt64 Join_3_Value_4 String -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String -3 UInt64 0 UInt64 String 3 UInt64 Join_2_Value_3 String 0 UInt64 String +0 UInt64 1 UInt32 String 1 UInt64 String 5 UInt64 Join_3_Value_4 String +0 UInt64 1 UInt32 Join_1_Value_0 String 1 UInt64 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt32 Join_1_Value_1 String 2 UInt64 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String +3 UInt64 1 UInt32 String 4 UInt64 Join_2_Value_3 String 1 UInt64 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -384,13 +384,13 @@ SELECT 1 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING ( SELECT id FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id FULL JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN FULL second JOIN INNER'; First JOIN FULL second JOIN INNER -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) INNER JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 String 3 UInt64 Join_2_Value_3 String 0 UInt64 Join_3_Value_0 String -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String +0 UInt64 1 UInt64 String 4 UInt32 Join_2_Value_3 String 1 UInt64 Join_3_Value_0 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -406,14 +406,14 @@ SELECT 1 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (i SELECT id FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id INNER JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN FULL second JOIN LEFT'; First JOIN FULL second JOIN LEFT -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) LEFT JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 String 3 UInt64 Join_2_Value_3 String 0 UInt64 Join_3_Value_0 String -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String -2 UInt64 2 UInt64 Join_1_Value_2 String 0 UInt64 String 0 UInt64 String +0 UInt64 1 UInt64 String 4 UInt32 Join_2_Value_3 String 1 UInt64 Join_3_Value_0 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String +2 UInt64 3 UInt64 Join_1_Value_2 String 1 UInt32 String 1 UInt64 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -433,14 +433,14 @@ SELECT 1 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (i SELECT id FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id LEFT JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN FULL second JOIN RIGHT'; First JOIN FULL second JOIN RIGHT -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) RIGHT JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 String 3 UInt64 Join_2_Value_3 String 0 UInt64 Join_3_Value_0 String -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String -4 UInt64 0 UInt64 String 0 UInt64 String 4 UInt64 Join_3_Value_4 String +0 UInt64 1 UInt64 String 4 UInt32 Join_2_Value_3 String 1 UInt64 Join_3_Value_0 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String +4 UInt64 1 UInt64 String 1 UInt32 String 5 UInt64 Join_3_Value_4 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) @@ -458,15 +458,15 @@ SELECT 1 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (i SELECT id FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id RIGHT JOIN test_table_join_3 AS t3 USING (id); -- { serverError AMBIGUOUS_IDENTIFIER } SELECT 'First JOIN FULL second JOIN FULL'; First JOIN FULL second JOIN FULL -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) FULL JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; -0 UInt64 0 UInt64 String 0 UInt64 String 4 UInt64 Join_3_Value_4 String -0 UInt64 0 UInt64 String 3 UInt64 Join_2_Value_3 String 0 UInt64 Join_3_Value_0 String -0 UInt64 0 UInt64 Join_1_Value_0 String 0 UInt64 Join_2_Value_0 String 0 UInt64 Join_3_Value_0 String -1 UInt64 1 UInt64 Join_1_Value_1 String 1 UInt64 Join_2_Value_1 String 1 UInt64 Join_3_Value_1 String -2 UInt64 2 UInt64 Join_1_Value_2 String 0 UInt64 String 0 UInt64 String +0 UInt64 1 UInt64 String 1 UInt32 String 5 UInt64 Join_3_Value_4 String +0 UInt64 1 UInt64 String 4 UInt32 Join_2_Value_3 String 1 UInt64 Join_3_Value_0 String +0 UInt64 1 UInt64 Join_1_Value_0 String 1 UInt32 Join_2_Value_0 String 1 UInt64 Join_3_Value_0 String +1 UInt64 2 UInt64 Join_1_Value_1 String 2 UInt32 Join_2_Value_1 String 2 UInt64 Join_3_Value_1 String +2 UInt64 3 UInt64 Join_1_Value_2 String 1 UInt32 String 1 UInt64 String SELECT '--'; -- SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeName(t2_value), t3.value AS t3_value, toTypeName(t3_value) diff --git a/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2 b/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2 index 20e452d3e0d2..f1be214407b6 100644 --- a/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2 +++ b/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2 @@ -64,8 +64,8 @@ FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 USING SELECT 'First JOIN {{ first_join_type }} second JOIN {{ second_join_type }}'; -SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), -t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) +SELECT id AS using_id, toTypeName(using_id), t1.id + 1 AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), +t2.id + 1 AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value), t3.id + 1 AS t3_id, toTypeName(t3_id), t3.value AS t3_value, toTypeName(t3_value) FROM test_table_join_1 AS t1 {{ first_join_type }} JOIN test_table_join_2 AS t2 USING (id) {{ second_join_type }} JOIN test_table_join_3 AS t3 USING(id) ORDER BY ALL; diff --git a/tests/queries/0_stateless/02735_parquet_encoder.reference b/tests/queries/0_stateless/02735_parquet_encoder.reference index e44ce99ed46f..e0b2f5655db0 100644 --- a/tests/queries/0_stateless/02735_parquet_encoder.reference +++ b/tests/queries/0_stateless/02735_parquet_encoder.reference @@ -43,7 +43,7 @@ datetime Nullable(DateTime64(3, \'UTC\')) (1000000,NULL,NULL,'0','-1294970296') (1000000,NULL,NULL,'-2147483296','2147481000') (100000,900000,NULL,'100009','999999') -[(2,NULL,NULL,'','[]')] +[(2,NULL,NULL,NULL,NULL)] 1 1 0 1 5090915589685802007 diff --git a/tests/queries/0_stateless/03100_lwu_45_query_condition_cache.reference b/tests/queries/0_stateless/03100_lwu_45_query_condition_cache.reference new file mode 100644 index 000000000000..ddc60e3ba0f5 --- /dev/null +++ b/tests/queries/0_stateless/03100_lwu_45_query_condition_cache.reference @@ -0,0 +1,2 @@ +0 +100000 diff --git a/tests/queries/0_stateless/03100_lwu_45_query_condition_cache.sql b/tests/queries/0_stateless/03100_lwu_45_query_condition_cache.sql new file mode 100644 index 000000000000..f1db5ce9adda --- /dev/null +++ b/tests/queries/0_stateless/03100_lwu_45_query_condition_cache.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS t_lwu_condition_cache; + +SET use_query_condition_cache = 1; +SET enable_lightweight_update = 1; +SET apply_patch_parts = 1; + +CREATE TABLE t_lwu_condition_cache +( + id UInt64 DEFAULT generateSnowflakeID(), + exists UInt8 +) +ENGINE = MergeTree ORDER BY id +SETTINGS index_granularity = 8192, enable_block_number_column = 1, enable_block_offset_column = 1; + +INSERT INTO t_lwu_condition_cache (exists) SELECT 0 FROM numbers(100000); + +SELECT count() FROM t_lwu_condition_cache WHERE exists; + +UPDATE t_lwu_condition_cache SET exists = 1 WHERE 1; + +SELECT count() FROM t_lwu_condition_cache WHERE exists; + +DROP TABLE IF EXISTS t_lwu_condition_cache; diff --git a/tests/queries/0_stateless/03285_analyzer_array_join_nested.reference b/tests/queries/0_stateless/03285_analyzer_array_join_nested.reference index bd66e2daf4d5..787a1d8c2fa2 100644 --- a/tests/queries/0_stateless/03285_analyzer_array_join_nested.reference +++ b/tests/queries/0_stateless/03285_analyzer_array_join_nested.reference @@ -12,7 +12,7 @@ QUERY id: 0 FUNCTION id: 2, function_name: tupleElement, function_type: ordinary, result_type: String ARGUMENTS LIST id: 3, nodes: 2 - COLUMN id: 4, column_name: __array_join_exp_1, result_type: Tuple(names String, values Int64), source_id: 5 + COLUMN id: 4, column_name: __array_join_exp_1, result_type: Tuple(names String), source_id: 5 CONSTANT id: 6, constant_value: \'names\', constant_value_type: String JOIN TREE ARRAY_JOIN id: 5, is_left: 0 @@ -20,18 +20,17 @@ QUERY id: 0 TABLE id: 7, alias: __table2, table_name: default.hourly JOIN EXPRESSIONS LIST id: 8, nodes: 1 - COLUMN id: 9, alias: __array_join_exp_1, column_name: __array_join_exp_1, result_type: Tuple(names String, values Int64), source_id: 5 + COLUMN id: 9, alias: __array_join_exp_1, column_name: __array_join_exp_1, result_type: Tuple(names String), source_id: 5 EXPRESSION - FUNCTION id: 10, function_name: nested, function_type: ordinary, result_type: Array(Tuple(names String, values Int64)) + FUNCTION id: 10, function_name: nested, function_type: ordinary, result_type: Array(Tuple(names String)) ARGUMENTS - LIST id: 11, nodes: 3 - CONSTANT id: 12, constant_value: Array_[\'names\', \'values\'], constant_value_type: Array(String) + LIST id: 11, nodes: 2 + CONSTANT id: 12, constant_value: Array_[\'names\'], constant_value_type: Array(String) COLUMN id: 13, column_name: metric.names, result_type: Array(String), source_id: 7 - COLUMN id: 14, column_name: metric.values, result_type: Array(Int64), source_id: 7 SELECT tupleElement(__array_join_exp_1, \'names\') AS `metric.names` FROM default.hourly AS __table2 -ARRAY JOIN nested(_CAST([\'names\', \'values\'], \'Array(String)\'), __table2.`metric.names`, __table2.`metric.values`) AS __array_join_exp_1 +ARRAY JOIN nested(_CAST([\'names\'], \'Array(String)\'), __table2.`metric.names`) AS __array_join_exp_1 explain query tree dump_ast = 1 SELECT metric.names @@ -44,7 +43,7 @@ QUERY id: 0 FUNCTION id: 2, function_name: tupleElement, function_type: ordinary, result_type: String ARGUMENTS LIST id: 3, nodes: 2 - COLUMN id: 4, column_name: __array_join_exp_1, result_type: Tuple(names String, values Int64), source_id: 5 + COLUMN id: 4, column_name: __array_join_exp_1, result_type: Tuple(names String), source_id: 5 CONSTANT id: 6, constant_value: \'names\', constant_value_type: String JOIN TREE ARRAY_JOIN id: 5, is_left: 0 @@ -52,18 +51,17 @@ QUERY id: 0 TABLE id: 7, alias: __table2, table_name: default.hourly JOIN EXPRESSIONS LIST id: 8, nodes: 1 - COLUMN id: 9, alias: __array_join_exp_1, column_name: __array_join_exp_1, result_type: Tuple(names String, values Int64), source_id: 5 + COLUMN id: 9, alias: __array_join_exp_1, column_name: __array_join_exp_1, result_type: Tuple(names String), source_id: 5 EXPRESSION - FUNCTION id: 10, function_name: nested, function_type: ordinary, result_type: Array(Tuple(names String, values Int64)) + FUNCTION id: 10, function_name: nested, function_type: ordinary, result_type: Array(Tuple(names String)) ARGUMENTS - LIST id: 11, nodes: 3 - CONSTANT id: 12, constant_value: Array_[\'names\', \'values\'], constant_value_type: Array(String) + LIST id: 11, nodes: 2 + CONSTANT id: 12, constant_value: Array_[\'names\'], constant_value_type: Array(String) COLUMN id: 13, column_name: metric.names, result_type: Array(String), source_id: 7 - COLUMN id: 14, column_name: metric.values, result_type: Array(Int64), source_id: 7 SELECT tupleElement(__array_join_exp_1, \'names\') AS `metric.names` FROM default.hourly AS __table2 -ARRAY JOIN nested(_CAST([\'names\', \'values\'], \'Array(String)\'), __table2.`metric.names`, __table2.`metric.values`) AS __array_join_exp_1 +ARRAY JOIN nested(_CAST([\'names\'], \'Array(String)\'), __table2.`metric.names`) AS __array_join_exp_1 -- { echoOn } SELECT nested(['click', 'house'], x.b.first, x.b.second) AS n, toTypeName(n) FROM tab; diff --git a/tests/queries/0_stateless/03389_regexp_rewrite_nullable_group_by.reference b/tests/queries/0_stateless/03389_regexp_rewrite_nullable_group_by.reference new file mode 100644 index 000000000000..d099bc72639f --- /dev/null +++ b/tests/queries/0_stateless/03389_regexp_rewrite_nullable_group_by.reference @@ -0,0 +1,4 @@ +abc123 +abc123 +\N +\N diff --git a/tests/queries/0_stateless/03389_regexp_rewrite_nullable_group_by.sql b/tests/queries/0_stateless/03389_regexp_rewrite_nullable_group_by.sql new file mode 100644 index 000000000000..47b44703fbff --- /dev/null +++ b/tests/queries/0_stateless/03389_regexp_rewrite_nullable_group_by.sql @@ -0,0 +1,4 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/88218 +-- RegexpFunctionRewritePass must handle Nullable result types from group_by_use_nulls +SET enable_analyzer = 1; +SELECT replaceRegexpOne(identity('abc123'), '^(abc)$', '\\1') GROUP BY 1, toLowCardinality(9), 1 WITH CUBE SETTINGS group_by_use_nulls=1; diff --git a/tests/queries/0_stateless/03538_optimize_rewrite_regexp_functions.reference b/tests/queries/0_stateless/03538_optimize_rewrite_regexp_functions.reference index 54c9b5d8fda7..12102a657fa5 100644 --- a/tests/queries/0_stateless/03538_optimize_rewrite_regexp_functions.reference +++ b/tests/queries/0_stateless/03538_optimize_rewrite_regexp_functions.reference @@ -38,62 +38,9 @@ FROM system.one AS __table1 EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpAll(identity('abc123'), '^123|456$', ''); SELECT replaceRegexpAll(identity(\'abc123\'), \'^123|456$\', \'\') AS `replaceRegexpAll(identity(\'abc123\'), \'^123|456$\', \'\')` FROM system.one AS __table1 --- Rule 2: If a replaceRegexpOne function has a replacement of nothing other than \1 and some subpatterns in the regexp, or \0 and no subpatterns in the regexp, rewrite it with extract. +-- Rule 2 (replaceRegexpOne -> extract) was removed because extract returns empty string on non-match, +-- while replaceRegexpOne returns the original string, making them semantically different. --- NOTE: \0 is specially treated as NUL instead of capture group reference. Need to use \\0 instead. - --- Only \0, no capture group (should rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^abc123$', '\\0'); -SELECT extract(identity(\'abc123\'), \'^abc123$\') AS `replaceRegexpOne(identity(\'abc123\'), \'^abc123$\', \'\\\\\\\\0\')` -FROM system.one AS __table1 --- Only \1, with one capture group (should rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^(abc)$', '\1'); -SELECT extract(identity(\'abc123\'), \'^(abc)$\') AS `replaceRegexpOne(identity(\'abc123\'), \'^(abc)$\', \'\\\\\\\\1\')` -FROM system.one AS __table1 --- Only \1, no capture group (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^abc$', '\1'); -SELECT replaceRegexpOne(identity(\'abc123\'), \'^abc$\', \'\\\\1\') AS `replaceRegexpOne(identity(\'abc123\'), \'^abc$\', \'\\\\\\\\1\')` -FROM system.one AS __table1 --- Pattern not full (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^abc', '\\0'); -SELECT replaceRegexpOne(identity(\'abc123\'), \'^abc\', \'\\\\0\') AS `replaceRegexpOne(identity(\'abc123\'), \'^abc\', \'\\\\\\\\0\')` -FROM system.one AS __table1 --- Pattern not full (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), 'abc$', '\\0'); -SELECT replaceRegexpOne(identity(\'abc123\'), \'abc$\', \'\\\\0\') AS `replaceRegexpOne(identity(\'abc123\'), \'abc$\', \'\\\\\\\\0\')` -FROM system.one AS __table1 --- Pattern not full (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), 'abc', '\\0'); -SELECT replaceRegexpOne(identity(\'abc123\'), \'abc\', \'\\\\0\') AS `replaceRegexpOne(identity(\'abc123\'), \'abc\', \'\\\\\\\\0\')` -FROM system.one AS __table1 --- Pattern not full (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^abc\\$', '\\0'); -SELECT replaceRegexpOne(identity(\'abc123\'), \'^abc\\\\$\', \'\\\\0\') AS `replaceRegexpOne(identity(\'abc123\'), \'^abc\\\\\\\\$\', \'\\\\\\\\0\')` -FROM system.one AS __table1 --- Pattern not full (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^ab|c$', '\\0'); -SELECT replaceRegexpOne(identity(\'abc123\'), \'^ab|c$\', \'\\\\0\') AS `replaceRegexpOne(identity(\'abc123\'), \'^ab|c$\', \'\\\\\\\\0\')` -FROM system.one AS __table1 --- \0 with extra characters (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^abc123$', 'pre\\0post'); -SELECT replaceRegexpOne(identity(\'abc123\'), \'^abc123$\', \'pre\\\\0post\') AS `replaceRegexpOne(identity(\'abc123\'), \'^abc123$\', \'pre\\\\\\\\0post\')` -FROM system.one AS __table1 --- \1 with two capture groups (should rewrite — only \1 used) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^(a)(b)$', '\1'); -SELECT extract(identity(\'abc123\'), \'^(a)(b)$\') AS `replaceRegexpOne(identity(\'abc123\'), \'^(a)(b)$\', \'\\\\\\\\1\')` -FROM system.one AS __table1 --- \2 used (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^(a)(b)$', '\2'); -SELECT replaceRegexpOne(identity(\'abc123\'), \'^(a)(b)$\', \'\\\\2\') AS `replaceRegexpOne(identity(\'abc123\'), \'^(a)(b)$\', \'\\\\\\\\2\')` -FROM system.one AS __table1 --- Mixed content in replacement (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^(abc)$', 'X\1Y'); -SELECT replaceRegexpOne(identity(\'abc123\'), \'^(abc)$\', \'X\\\\1Y\') AS `replaceRegexpOne(identity(\'abc123\'), \'^(abc)$\', \'X\\\\\\\\1Y\')` -FROM system.one AS __table1 --- Escaped backslash in replacement (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^(abc)$', '\\\\1'); -SELECT replaceRegexpOne(identity(\'abc123\'), \'^(abc)$\', \'\\\\\\\\1\') AS `replaceRegexpOne(identity(\'abc123\'), \'^(abc)$\', \'\\\\\\\\\\\\\\\\1\')` -FROM system.one AS __table1 -- Rule 3: If an extract function has a regexp with some subpatterns and the regexp starts with ^.* or ending with an unescaped .*$, remove this prefix and/or suffix. -- Starts with ^.* (should strip prefix) @@ -134,19 +81,11 @@ SELECT extract(identity(\'abc123\'), \'(abc).*\') AS `extract(identity(\'abc123\ FROM system.one AS __table1 -- Cascade tests --- Rule 1 + Rule 2: replaceRegexpAll to replaceRegexpOne to extract +-- Rule 1 only: replaceRegexpAll to replaceRegexpOne (Rule 2 removed) EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpAll(identity('abc'), '^(abc)', '\1'); SELECT replaceRegexpOne(identity(\'abc\'), \'^(abc)\', \'\\\\1\') AS `replaceRegexpAll(identity(\'abc\'), \'^(abc)\', \'\\\\\\\\1\')` FROM system.one AS __table1 --- Rule 2 + 3: replaceRegexpOne -> extract -> simplified extract -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc'), '^.*(abc).*$','\1'); -SELECT extract(identity(\'abc\'), \'(abc)\') AS `replaceRegexpOne(identity(\'abc\'), \'^.*(abc).*$\', \'\\\\\\\\1\')` -FROM system.one AS __table1 --- Rule 1 + 2 + 3: replaceRegexpAll -> replaceRegexpOne -> extract -> simplified extract -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpAll(identity('abc'), '^.*(abc).*$','\1'); -SELECT extract(identity(\'abc\'), \'(abc)\') AS `replaceRegexpAll(identity(\'abc\'), \'^.*(abc).*$\', \'\\\\\\\\1\')` -FROM system.one AS __table1 --- ClickBench Q28 +-- ClickBench Q28: Rule 1 only: regexp_replace to replaceRegexpOne EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT REGEXP_REPLACE(identity('some referer'), '^https?://(?:www\.)?([^/]+)/.*$', '\1'); -SELECT extract(identity(\'some referer\'), \'^https?://(?:www\\\\.)?([^/]+)/\') AS `REGEXP_REPLACE(identity(\'some referer\'), \'^https?://(?:www\\\\\\\\.)?([^/]+)/.*$\', \'\\\\\\\\1\')` +SELECT replaceRegexpOne(identity(\'some referer\'), \'^https?://(?:www\\\\.)?([^/]+)/.*$\', \'\\\\1\') AS `REGEXP_REPLACE(identity(\'some referer\'), \'^https?://(?:www\\\\\\\\.)?([^/]+)/.*$\', \'\\\\\\\\1\')` FROM system.one AS __table1 diff --git a/tests/queries/0_stateless/03538_optimize_rewrite_regexp_functions.sql b/tests/queries/0_stateless/03538_optimize_rewrite_regexp_functions.sql index e5f37eb54c9e..3e0e3194442d 100644 --- a/tests/queries/0_stateless/03538_optimize_rewrite_regexp_functions.sql +++ b/tests/queries/0_stateless/03538_optimize_rewrite_regexp_functions.sql @@ -28,49 +28,8 @@ EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpAll(identity( -- Pattern with alternatives (should NOT rewrite) EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpAll(identity('abc123'), '^123|456$', ''); --- Rule 2: If a replaceRegexpOne function has a replacement of nothing other than \1 and some subpatterns in the regexp, or \0 and no subpatterns in the regexp, rewrite it with extract. - --- NOTE: \0 is specially treated as NUL instead of capture group reference. Need to use \\0 instead. - --- Only \0, no capture group (should rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^abc123$', '\\0'); - --- Only \1, with one capture group (should rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^(abc)$', '\1'); - --- Only \1, no capture group (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^abc$', '\1'); - --- Pattern not full (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^abc', '\\0'); - --- Pattern not full (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), 'abc$', '\\0'); - --- Pattern not full (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), 'abc', '\\0'); - --- Pattern not full (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^abc\\$', '\\0'); - --- Pattern not full (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^ab|c$', '\\0'); - --- \0 with extra characters (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^abc123$', 'pre\\0post'); - --- \1 with two capture groups (should rewrite — only \1 used) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^(a)(b)$', '\1'); - --- \2 used (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^(a)(b)$', '\2'); - --- Mixed content in replacement (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^(abc)$', 'X\1Y'); - --- Escaped backslash in replacement (should NOT rewrite) -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc123'), '^(abc)$', '\\\\1'); - +-- Rule 2 (replaceRegexpOne -> extract) was removed because extract returns empty string on non-match, +-- while replaceRegexpOne returns the original string, making them semantically different. -- Rule 3: If an extract function has a regexp with some subpatterns and the regexp starts with ^.* or ending with an unescaped .*$, remove this prefix and/or suffix. @@ -104,14 +63,8 @@ EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT extract(identity('abc123') -- Cascade tests --- Rule 1 + Rule 2: replaceRegexpAll to replaceRegexpOne to extract +-- Rule 1 only: replaceRegexpAll to replaceRegexpOne (Rule 2 removed) EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpAll(identity('abc'), '^(abc)', '\1'); --- Rule 2 + 3: replaceRegexpOne -> extract -> simplified extract -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpOne(identity('abc'), '^.*(abc).*$','\1'); - --- Rule 1 + 2 + 3: replaceRegexpAll -> replaceRegexpOne -> extract -> simplified extract -EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT replaceRegexpAll(identity('abc'), '^.*(abc).*$','\1'); - --- ClickBench Q28 +-- ClickBench Q28: Rule 1 only: regexp_replace to replaceRegexpOne EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT REGEXP_REPLACE(identity('some referer'), '^https?://(?:www\.)?([^/]+)/.*$', '\1'); diff --git a/tests/queries/0_stateless/03707_set_index_bad_get_null_bug.reference b/tests/queries/0_stateless/03707_set_index_bad_get_null_bug.reference new file mode 100644 index 000000000000..6c3792712ba4 --- /dev/null +++ b/tests/queries/0_stateless/03707_set_index_bad_get_null_bug.reference @@ -0,0 +1,14 @@ +Expression + Filter + ReadFromMergeTree + Indexes: + PrimaryKey + Condition: true + Parts: 1/1 + Granules: 1/1 + Skip + Name: v + Description: set GRANULARITY 1 + Parts: 0/1 + Granules: 0/1 + Ranges: 0 diff --git a/tests/queries/0_stateless/03707_set_index_bad_get_null_bug.sql b/tests/queries/0_stateless/03707_set_index_bad_get_null_bug.sql new file mode 100644 index 000000000000..814c08cd312d --- /dev/null +++ b/tests/queries/0_stateless/03707_set_index_bad_get_null_bug.sql @@ -0,0 +1,13 @@ +drop table if exists test; +CREATE table test +( + `ts` Int64, + `v` LowCardinality(String), + INDEX v v TYPE set(0) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY (ts); + +INSERT INTO test (v) FORMAT Values ('VALUE1'); + +EXPLAIN indexes = 1, description=0 SELECT CAST(NULL, 'Nullable(String)') AS source, v AS v FROM test WHERE (source = 'VALUE1') OR (v ILIKE 'VALUE1'); diff --git a/tests/queries/0_stateless/03716_multiple_joins_using_top_level_identifier.reference b/tests/queries/0_stateless/03716_multiple_joins_using_top_level_identifier.reference new file mode 100644 index 000000000000..5b5f70835c39 --- /dev/null +++ b/tests/queries/0_stateless/03716_multiple_joins_using_top_level_identifier.reference @@ -0,0 +1,4 @@ +a_1 v +b_1 w +_1 v +b_1 w diff --git a/tests/queries/0_stateless/03716_multiple_joins_using_top_level_identifier.sql b/tests/queries/0_stateless/03716_multiple_joins_using_top_level_identifier.sql new file mode 100644 index 000000000000..820fc46048bb --- /dev/null +++ b/tests/queries/0_stateless/03716_multiple_joins_using_top_level_identifier.sql @@ -0,0 +1,36 @@ +SET analyzer_compatibility_join_using_top_level_identifier = 1; + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; + +CREATE TABLE t1 (id String, val String) ENGINE = MergeTree() ORDER BY id; +CREATE TABLE t2 (id String, code String) ENGINE = MergeTree() ORDER BY id; +CREATE TABLE t3 (id String, code String) ENGINE = MergeTree() ORDER BY id; + +INSERT INTO t1 VALUES ('a', 'v'), ('b', 'w'); +INSERT INTO t2 VALUES ('b', 'c'); +INSERT INTO t3 VALUES ('a_1', 'c'), ('b_1', 'd'); + +SET enable_analyzer = 1; + +SELECT t1.id || '_1' AS id, t1.val +FROM t1 +LEFT JOIN t2 ON t1.id = t2.id +LEFT JOIN t3 USING (id) +ORDER BY t1.val +; + +SELECT t2.id || '_1' AS id, t1.val +FROM t1 +LEFT JOIN t2 ON t1.id = t2.id +LEFT JOIN t3 USING (id) +ORDER BY t1.val +; + +SELECT t1.id || t2.id || '_1' AS id, t1.val +FROM t1 +INNER JOIN t2 ON t1.id = t2.id +LEFT JOIN t3 USING (id) +ORDER BY t1.val +; -- { serverError AMBIGUOUS_IDENTIFIER } diff --git a/tests/queries/0_stateless/03732_toweek_partition_pruning.reference b/tests/queries/0_stateless/03732_toweek_partition_pruning.reference new file mode 100644 index 000000000000..02af9a39522c --- /dev/null +++ b/tests/queries/0_stateless/03732_toweek_partition_pruning.reference @@ -0,0 +1,2 @@ +49 2 +52 2 diff --git a/tests/queries/0_stateless/03732_toweek_partition_pruning.sql b/tests/queries/0_stateless/03732_toweek_partition_pruning.sql new file mode 100644 index 000000000000..bd1ce72d7713 --- /dev/null +++ b/tests/queries/0_stateless/03732_toweek_partition_pruning.sql @@ -0,0 +1,21 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/90240 +-- toWeek() incorrectly claimed monotonicity, causing partition pruning +-- to skip December partitions for weeks 49-52. + +DROP TABLE IF EXISTS test_toweek_pruning; + +CREATE TABLE test_toweek_pruning (date Date, value String) +ENGINE = MergeTree +PARTITION BY toYYYYMM(date) +ORDER BY date; + +INSERT INTO test_toweek_pruning VALUES + ('2025-11-30', 'x'), ('2025-12-01', 'x'), ('2025-12-07', 'x'), + ('2025-12-08', 'x'), ('2025-12-14', 'x'), ('2025-12-15', 'x'), + ('2025-12-21', 'x'), ('2025-12-22', 'x'), ('2025-12-28', 'x'), + ('2025-12-29', 'x'), ('2025-12-31', 'x'); + +SELECT toWeek(date, 3), count() FROM test_toweek_pruning WHERE toWeek(date, 3) = 49 GROUP BY 1; +SELECT toWeek(date, 3), count() FROM test_toweek_pruning WHERE toWeek(date, 3) = 52 GROUP BY 1; + +DROP TABLE test_toweek_pruning; diff --git a/tests/queries/0_stateless/03789_to_year_week_monotonicity_key_condition.reference b/tests/queries/0_stateless/03789_to_year_week_monotonicity_key_condition.reference index 7e87f95616da..a8960301ac05 100644 --- a/tests/queries/0_stateless/03789_to_year_week_monotonicity_key_condition.reference +++ b/tests/queries/0_stateless/03789_to_year_week_monotonicity_key_condition.reference @@ -33,8 +33,11 @@ CREATE TABLE t (dt DateTime) ENGINE=MergeTree ORDER BY dt SETTINGS index_granula INSERT INTO t SELECT toDateTime('2020-01-01 00:00:00') + number * 3600 FROM numbers(24 * 40); SELECT count() FROM t -WHERE toWeek(dt) = toWeek(toDateTime('2020-01-15 00:00:00')) SETTINGS force_primary_key = 1, max_rows_to_read = 169; +WHERE toWeek(dt) = toWeek(toDateTime('2020-01-15 00:00:00')); 168 +SELECT count() +FROM t +WHERE toWeek(dt) = toWeek(toDateTime('2020-01-15 00:00:00')) SETTINGS force_primary_key = 1; -- { serverError INDEX_NOT_USED } DROP TABLE IF EXISTS t; CREATE TABLE t (s LowCardinality(String)) ENGINE = MergeTree ORDER BY s; INSERT INTO t VALUES ('2020-01-10 00:00:00'), ('2020-01-2 00:00:00'); diff --git a/tests/queries/0_stateless/03789_to_year_week_monotonicity_key_condition.sql b/tests/queries/0_stateless/03789_to_year_week_monotonicity_key_condition.sql index fb550e72051f..5e81bfe84ce5 100644 --- a/tests/queries/0_stateless/03789_to_year_week_monotonicity_key_condition.sql +++ b/tests/queries/0_stateless/03789_to_year_week_monotonicity_key_condition.sql @@ -40,7 +40,11 @@ INSERT INTO t SELECT toDateTime('2020-01-01 00:00:00') + number * 3600 FROM numb SELECT count() FROM t -WHERE toWeek(dt) = toWeek(toDateTime('2020-01-15 00:00:00')) SETTINGS force_primary_key = 1, max_rows_to_read = 169; +WHERE toWeek(dt) = toWeek(toDateTime('2020-01-15 00:00:00')); + +SELECT count() +FROM t +WHERE toWeek(dt) = toWeek(toDateTime('2020-01-15 00:00:00')) SETTINGS force_primary_key = 1; -- { serverError INDEX_NOT_USED } DROP TABLE IF EXISTS t; CREATE TABLE t (s LowCardinality(String)) ENGINE = MergeTree ORDER BY s; diff --git a/tests/queries/0_stateless/03801_attach_view_with_sql_security.reference b/tests/queries/0_stateless/03801_attach_view_with_sql_security.reference new file mode 100644 index 000000000000..4a703b3be841 --- /dev/null +++ b/tests/queries/0_stateless/03801_attach_view_with_sql_security.reference @@ -0,0 +1,2 @@ +ACCESS_DENIED +ACCESS_DENIED diff --git a/tests/queries/0_stateless/03801_attach_view_with_sql_security.sh b/tests/queries/0_stateless/03801_attach_view_with_sql_security.sh new file mode 100755 index 000000000000..98b791ffcffe --- /dev/null +++ b/tests/queries/0_stateless/03801_attach_view_with_sql_security.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Tags: no-parallel + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user03801_${CLICKHOUSE_DATABASE}_$RANDOM" +db=${CLICKHOUSE_DATABASE} + +${CLICKHOUSE_CLIENT} <&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "NO ERROR" + +${CLICKHOUSE_CLIENT} --user $user --query " + ATTACH VIEW $db.test_mv UUID '8025ef9c-d735-4c16-ab4c-7f1f5110d049' + (s String) SQL SECURITY NONE + AS SELECT * FROM $db.test_table; +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "NO ERROR" + +${CLICKHOUSE_CLIENT} --query "GRANT ALLOW SQL SECURITY NONE ON *.* TO $user;" + +${CLICKHOUSE_CLIENT} --user $user --query " + ATTACH VIEW $db.test_mv UUID '7025ef9c-d735-4c16-ab4c-7f1f5110d049' + (s String) SQL SECURITY NONE + AS SELECT * FROM $db.test_table + SETTINGS send_logs_level = 'error'; +" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE $db.test_mv;" +${CLICKHOUSE_CLIENT} --query "DROP USER $user;" diff --git a/tests/queries/0_stateless/03813_mergetree_projection_grants.reference b/tests/queries/0_stateless/03813_mergetree_projection_grants.reference new file mode 100644 index 000000000000..a6219591f168 --- /dev/null +++ b/tests/queries/0_stateless/03813_mergetree_projection_grants.reference @@ -0,0 +1,4 @@ +=== mergeTreeProjection without grant === +ACCESS_DENIED +=== mergeTreeProjection with grant === +1 diff --git a/tests/queries/0_stateless/03813_mergetree_projection_grants.sh b/tests/queries/0_stateless/03813_mergetree_projection_grants.sh new file mode 100755 index 000000000000..f49a45e513ea --- /dev/null +++ b/tests/queries/0_stateless/03813_mergetree_projection_grants.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel + +# Test that mergeTreeProjection checks table grants correctly. +# This function should require SELECT permission on the source table. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_proj_grants_mt" +$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS test_user_03813" + +$CLICKHOUSE_CLIENT -q " +CREATE TABLE test_proj_grants_mt (key Int, value Int, PROJECTION proj_sum (SELECT key, sum(value) GROUP BY key)) +ENGINE = MergeTree() ORDER BY key +" + +$CLICKHOUSE_CLIENT -q "INSERT INTO test_proj_grants_mt SELECT number % 10, number FROM numbers(1000)" +$CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE test_proj_grants_mt FINAL" + +# Create user without any grants +$CLICKHOUSE_CLIENT -q "CREATE USER test_user_03813" + +# Test mergeTreeProjection - should fail without SELECT grant +echo "=== mergeTreeProjection without grant ===" +$CLICKHOUSE_CLIENT --user test_user_03813 -q " +SELECT count() FROM mergeTreeProjection(currentDatabase(), test_proj_grants_mt, proj_sum) +" 2>&1 | grep -o 'ACCESS_DENIED' | head -1 + +# Grant SELECT permission +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON ${CLICKHOUSE_DATABASE}.test_proj_grants_mt TO test_user_03813" + +# Test mergeTreeProjection - should work with SELECT grant +echo "=== mergeTreeProjection with grant ===" +$CLICKHOUSE_CLIENT --user test_user_03813 -q " +SELECT count() > 0 FROM mergeTreeProjection(currentDatabase(), test_proj_grants_mt, proj_sum) +" + +# Cleanup +$CLICKHOUSE_CLIENT -q "DROP TABLE test_proj_grants_mt" +$CLICKHOUSE_CLIENT -q "DROP USER test_user_03813" diff --git a/tests/queries/0_stateless/03821_json_skip_path_fix.reference b/tests/queries/0_stateless/03821_json_skip_path_fix.reference new file mode 100644 index 000000000000..be9e7e81710a --- /dev/null +++ b/tests/queries/0_stateless/03821_json_skip_path_fix.reference @@ -0,0 +1 @@ +{"path_1":42,"path_2":42} diff --git a/tests/queries/0_stateless/03821_json_skip_path_fix.sql b/tests/queries/0_stateless/03821_json_skip_path_fix.sql new file mode 100644 index 000000000000..8c770fd14e29 --- /dev/null +++ b/tests/queries/0_stateless/03821_json_skip_path_fix.sql @@ -0,0 +1,2 @@ +select '{"path_1" : 42, "path_2" : 42, "path" : {"a" : 42}}'::JSON(SKIP path); + diff --git a/tests/queries/0_stateless/03822_file_function_read_on_file_grant.reference b/tests/queries/0_stateless/03822_file_function_read_on_file_grant.reference new file mode 100644 index 000000000000..865d4ff5230b --- /dev/null +++ b/tests/queries/0_stateless/03822_file_function_read_on_file_grant.reference @@ -0,0 +1,4 @@ +ACCESS_DENIED +ACCESS_DENIED +FILE_DOESNT_EXIST +CANNOT_STAT diff --git a/tests/queries/0_stateless/03822_file_function_read_on_file_grant.sh b/tests/queries/0_stateless/03822_file_function_read_on_file_grant.sh new file mode 100755 index 000000000000..6db574849a79 --- /dev/null +++ b/tests/queries/0_stateless/03822_file_function_read_on_file_grant.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_03822_${CLICKHOUSE_DATABASE}_$RANDOM" +missing_txt="missing_03822_${CLICKHOUSE_DATABASE}_$RANDOM.txt" +missing_csv="missing_03822_${CLICKHOUSE_DATABASE}_$RANDOM.csv" + +${CLICKHOUSE_CLIENT} <&1 | grep -c "ACCESS_DENIED") >= 1 )) && echo "ACCESS_DENIED" || echo "UNEXPECTED"; +(( $(${CLICKHOUSE_CLIENT} --user $user --query "DESCRIBE TABLE file('$missing_csv', 'CSV')" 2>&1 | grep -c "ACCESS_DENIED") >= 1 )) && echo "ACCESS_DENIED" || echo "UNEXPECTED"; + +${CLICKHOUSE_CLIENT} --query "GRANT READ ON FILE TO $user"; + +(( $(${CLICKHOUSE_CLIENT} --user $user --query "SELECT file('$missing_txt')" 2>&1 | grep -c "FILE_DOESNT_EXIST") >= 1 )) && echo "FILE_DOESNT_EXIST" || echo "UNEXPECTED"; +(( $(${CLICKHOUSE_CLIENT} --user $user --query "DESCRIBE TABLE file('$missing_csv', 'CSV')" 2>&1 | grep -c "CANNOT_STAT") >= 1 )) && echo "CANNOT_STAT" || echo "UNEXPECTED"; + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS $user"; diff --git a/tests/queries/0_stateless/03822_revoke_default_role.reference b/tests/queries/0_stateless/03822_revoke_default_role.reference new file mode 100644 index 000000000000..1d674b07c13c --- /dev/null +++ b/tests/queries/0_stateless/03822_revoke_default_role.reference @@ -0,0 +1,4 @@ +CREATE USER user_03822_default IDENTIFIED WITH no_password DEFAULT ROLE role_03822_default +role_03822_default 0 1 +After revoke: +CREATE USER user_03822_default IDENTIFIED WITH no_password DEFAULT ROLE NONE diff --git a/tests/queries/0_stateless/03822_revoke_default_role.sh b/tests/queries/0_stateless/03822_revoke_default_role.sh new file mode 100755 index 000000000000..aa51bd26e643 --- /dev/null +++ b/tests/queries/0_stateless/03822_revoke_default_role.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +user_name="user_03822_${CLICKHOUSE_DATABASE}" +role_name="role_03822_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS ${user_name}" +$CLICKHOUSE_CLIENT -q "DROP ROLE IF EXISTS ${role_name}" + +$CLICKHOUSE_CLIENT -q "CREATE USER ${user_name}" +$CLICKHOUSE_CLIENT -q "CREATE ROLE ${role_name}" + +$CLICKHOUSE_CLIENT -q "GRANT ${role_name} TO ${user_name}" +$CLICKHOUSE_CLIENT -q "SET DEFAULT ROLE ${role_name} TO ${user_name}" +$CLICKHOUSE_CLIENT -q "SHOW CREATE USER ${user_name}" +$CLICKHOUSE_CLIENT --user ${user_name} -q "SHOW CURRENT ROLES" + +echo "After revoke:" +$CLICKHOUSE_CLIENT -q "REVOKE ${role_name} FROM ${user_name}" +$CLICKHOUSE_CLIENT -q "SHOW CREATE USER ${user_name}" +$CLICKHOUSE_CLIENT --user ${user_name} -q "SHOW CURRENT ROLES" + +$CLICKHOUSE_CLIENT -q "DROP USER ${user_name}" +$CLICKHOUSE_CLIENT -q "DROP ROLE ${role_name}" diff --git a/tests/queries/0_stateless/03831_index_of_assume_sorted_const_exception.reference b/tests/queries/0_stateless/03831_index_of_assume_sorted_const_exception.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/03831_index_of_assume_sorted_const_exception.sql b/tests/queries/0_stateless/03831_index_of_assume_sorted_const_exception.sql new file mode 100644 index 000000000000..6bbc1f0ccf33 --- /dev/null +++ b/tests/queries/0_stateless/03831_index_of_assume_sorted_const_exception.sql @@ -0,0 +1,4 @@ +-- indexOfAssumeSorted with incompatible types in constant array should throw an exception, not crash (std::terminate from noexcept). +-- https://github.com/ClickHouse/ClickHouse/issues/92975 +SELECT indexOfAssumeSorted(['1.1.1.1'::IPv4], 0); -- { serverError BAD_TYPE_OF_FIELD } +SELECT indexOfAssumeSorted(['172.181.59.225'::IPv4], 3350671033650519441::Int8); -- { serverError BAD_TYPE_OF_FIELD } diff --git a/tests/queries/0_stateless/03835_todate_monotonicity_boundary.reference b/tests/queries/0_stateless/03835_todate_monotonicity_boundary.reference new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/tests/queries/0_stateless/03835_todate_monotonicity_boundary.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/03835_todate_monotonicity_boundary.sql b/tests/queries/0_stateless/03835_todate_monotonicity_boundary.sql new file mode 100644 index 000000000000..4460c5da7b68 --- /dev/null +++ b/tests/queries/0_stateless/03835_todate_monotonicity_boundary.sql @@ -0,0 +1,18 @@ +-- Regression test for off-by-one in ToDateMonotonicity boundary check. +-- The toDate function treats values <= DATE_LUT_MAX_DAY_NUM (65535) as day numbers +-- and values > 65535 as unix timestamps. The monotonicity check must correctly +-- identify ranges crossing this boundary as non-monotonic. +-- Previously caused LOGICAL_ERROR "Invalid binary search result in MergeTreeSetIndex" in debug builds. +-- https://github.com/ClickHouse/ClickHouse/issues/90461 + +DROP TABLE IF EXISTS t_todate_mono; + +CREATE TABLE t_todate_mono (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 1; +INSERT INTO t_todate_mono SELECT number FROM numbers(100000); + +-- With index_granularity=1, mark 65535 covers the range [65535, 65536], +-- which crosses the DATE_LUT_MAX_DAY_NUM boundary. +-- The toDate conversion in the key condition chain must report this range as non-monotonic. +SELECT count() > 0 FROM t_todate_mono WHERE toDate(x) IN (toDate(12345), toDate(67890)); + +DROP TABLE t_todate_mono; diff --git a/tests/queries/0_stateless/03902_format_datetime_fractional_msan.reference b/tests/queries/0_stateless/03902_format_datetime_fractional_msan.reference new file mode 100644 index 000000000000..9ab82215084b --- /dev/null +++ b/tests/queries/0_stateless/03902_format_datetime_fractional_msan.reference @@ -0,0 +1,9 @@ +January 12345678 +January 1234 +January 000000 +January 1234 +January 0 +005 +050 +005000 +00 diff --git a/tests/queries/0_stateless/03902_format_datetime_fractional_msan.sql b/tests/queries/0_stateless/03902_format_datetime_fractional_msan.sql new file mode 100644 index 000000000000..d3fa89558188 --- /dev/null +++ b/tests/queries/0_stateless/03902_format_datetime_fractional_msan.sql @@ -0,0 +1,18 @@ +-- Test for use-of-uninitialized-value in formatDateTime fractional second formatters. +-- %M is a variable-width formatter, so the output buffer is not pre-filled with the template. +-- The fractional second formatters must fully initialize their output bytes. + +-- mysqlFractionalSecondScaleNumDigits (formatdatetime_f_prints_scale_number_of_digits = 1) +SELECT formatDateTime(toDateTime64('2024-01-01 12:00:00.12345678', 8, 'UTC'), '%M %f') SETTINGS formatdatetime_f_prints_scale_number_of_digits = 1; +SELECT formatDateTime(toDateTime64('2024-01-01 12:00:00.1234', 4, 'UTC'), '%M %f') SETTINGS formatdatetime_f_prints_scale_number_of_digits = 1; +SELECT formatDateTime(toDateTime64('2024-01-01 12:00:00', 0, 'UTC'), '%M %f') SETTINGS formatdatetime_f_prints_scale_number_of_digits = 1; + +-- mysqlFractionalSecondSingleZero (formatdatetime_f_prints_single_zero = 1) +SELECT formatDateTime(toDateTime64('2024-01-01 12:00:00.1234', 4, 'UTC'), '%M %f') SETTINGS formatdatetime_f_prints_single_zero = 1; +SELECT formatDateTime(toDateTime64('2024-01-01 12:00:00', 0, 'UTC'), '%M %f') SETTINGS formatdatetime_f_prints_single_zero = 1; + +-- jodaFractionOfSecond: leading zeros must be preserved (e.g. fractional_second=5, scale=3 -> "005") +SELECT formatDateTimeInJodaSyntax(toDateTime64('2024-01-01 12:00:00.005', 3, 'UTC'), 'SSS'); +SELECT formatDateTimeInJodaSyntax(toDateTime64('2024-01-01 12:00:00.050', 3, 'UTC'), 'SSS'); +SELECT formatDateTimeInJodaSyntax(toDateTime64('2024-01-01 12:00:00.005', 3, 'UTC'), 'SSSSSS'); +SELECT formatDateTimeInJodaSyntax(toDateTime64('2024-01-01 12:00:00.005', 3, 'UTC'), 'SS'); diff --git a/tests/queries/0_stateless/03903_cancellation_checker_large_timeout.reference b/tests/queries/0_stateless/03903_cancellation_checker_large_timeout.reference new file mode 100644 index 000000000000..e8183f05f5db --- /dev/null +++ b/tests/queries/0_stateless/03903_cancellation_checker_large_timeout.reference @@ -0,0 +1,3 @@ +1 +1 +1 diff --git a/tests/queries/0_stateless/03903_cancellation_checker_large_timeout.sql b/tests/queries/0_stateless/03903_cancellation_checker_large_timeout.sql new file mode 100644 index 000000000000..edadd3e08f05 --- /dev/null +++ b/tests/queries/0_stateless/03903_cancellation_checker_large_timeout.sql @@ -0,0 +1,9 @@ +-- Tags: no-fasttest +-- Test that extremely large max_execution_time values don't cause livelock in CancellationChecker. +-- The timeout is internally capped to 1 year to prevent overflow in std::condition_variable::wait_for. +-- This query should complete quickly without hanging, regardless of the huge timeout value. + +SET max_execution_time = 9223372041; -- Close to INT64_MAX / 1000000000, would overflow when converted to nanoseconds +SELECT 1; +SELECT 1; +SELECT 1; diff --git a/tests/queries/0_stateless/03903_join_alias_dups.reference b/tests/queries/0_stateless/03903_join_alias_dups.reference new file mode 100644 index 000000000000..30ea790176ca --- /dev/null +++ b/tests/queries/0_stateless/03903_join_alias_dups.reference @@ -0,0 +1,4 @@ +42 +1 g +42 +1 g diff --git a/tests/queries/0_stateless/03903_join_alias_dups.sql.j2 b/tests/queries/0_stateless/03903_join_alias_dups.sql.j2 new file mode 100644 index 000000000000..71aad85c3d14 --- /dev/null +++ b/tests/queries/0_stateless/03903_join_alias_dups.sql.j2 @@ -0,0 +1,32 @@ +{% for enable_analyzer in [0, 1] -%} + +SET enable_analyzer = {{ enable_analyzer }}; +SET join_algorithm = 'hash'; + +SELECT A.g +FROM ( SELECT 1::Int8 AS d ) AS B +JOIN ( SELECT 1::Int8 as d, g, 42::Int32 AS g FROM ( SELECT '128' AS g ) ) AS A +USING (d); + +WITH B AS ( +SELECT + 1 AS d + ), + A AS ( +SELECT + g, d, + MAX(IF(m = 'A', g, NULL)) AS g +FROM + ( + SELECT + 'g' AS g, 1 d, + 'A' m + ) +GROUP BY ALL ) +SELECT + B.*, + A.g +FROM + B +LEFT JOIN A USING d; +{% endfor -%} diff --git a/tests/queries/0_stateless/03903_query_condition_cache_cte_constant_folding.reference b/tests/queries/0_stateless/03903_query_condition_cache_cte_constant_folding.reference new file mode 100644 index 000000000000..48f6d1611d27 --- /dev/null +++ b/tests/queries/0_stateless/03903_query_condition_cache_cte_constant_folding.reference @@ -0,0 +1,3 @@ +20000 2021 2022 +20000 2020 2021 +20000 2018 2019 diff --git a/tests/queries/0_stateless/03903_query_condition_cache_cte_constant_folding.sql b/tests/queries/0_stateless/03903_query_condition_cache_cte_constant_folding.sql new file mode 100644 index 000000000000..23e016f1c775 --- /dev/null +++ b/tests/queries/0_stateless/03903_query_condition_cache_cte_constant_folding.sql @@ -0,0 +1,43 @@ +-- Tags: no-random-settings + +-- Test for query condition cache correctness with CTE constant folding. +-- When constants are folded from CTE expressions, different constant values must produce +-- different hashes. Otherwise the query condition cache returns wrong results. +-- https://github.com/ClickHouse/ClickHouse/issues/96060 + +SET use_query_condition_cache = 1; + +DROP TABLE IF EXISTS test_qcc_cte; + +CREATE TABLE test_qcc_cte (activity_year Int16) ENGINE = MergeTree ORDER BY activity_year; +-- Need enough rows to have multiple granules so the cache can incorrectly exclude some. +INSERT INTO test_qcc_cte SELECT number % 10 + 2018 FROM numbers(100000); + +SYSTEM DROP QUERY CONDITION CACHE; + +-- First query: addMonths('2022-12-01', 0) -> year = 2022, filter: year IN (2021, 2022) +WITH block_0 AS ( + SELECT *, addMonths('2022-12-01'::date, 0) AS report_month + FROM test_qcc_cte +) +SELECT count(), min(activity_year), max(activity_year) FROM block_0 +WHERE (activity_year = toYear(report_month)) OR (activity_year = toYear(report_month) - 1); + +-- Second query: addMonths('2022-12-01', -12) -> year = 2021, filter: year IN (2020, 2021) +-- Without the fix, this would return wrong results due to cache hash collision. +WITH block_0 AS ( + SELECT *, addMonths('2022-12-01'::date, -12) AS report_month + FROM test_qcc_cte +) +SELECT count(), min(activity_year), max(activity_year) FROM block_0 +WHERE (activity_year = toYear(report_month)) OR (activity_year = toYear(report_month) - 1); + +-- Third query: addMonths('2022-12-01', -36) -> year = 2019, filter: year IN (2018, 2019) +WITH block_0 AS ( + SELECT *, addMonths('2022-12-01'::date, -36) AS report_month + FROM test_qcc_cte +) +SELECT count(), min(activity_year), max(activity_year) FROM block_0 +WHERE (activity_year = toYear(report_month)) OR (activity_year = toYear(report_month) - 1); + +DROP TABLE test_qcc_cte; diff --git a/tests/queries/0_stateless/03913_data_type_function_null_arg_hash.reference b/tests/queries/0_stateless/03913_data_type_function_null_arg_hash.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/03913_data_type_function_null_arg_hash.sql b/tests/queries/0_stateless/03913_data_type_function_null_arg_hash.sql new file mode 100644 index 000000000000..956fce12ae8d --- /dev/null +++ b/tests/queries/0_stateless/03913_data_type_function_null_arg_hash.sql @@ -0,0 +1,3 @@ +-- Regression test: DataTypeFunction::updateHashImpl must handle null argument types +-- https://s3.amazonaws.com/clickhouse-test-reports/json.html?REF=master&sha=b9e68f4b9b0b33c7db43b00afb3eff4ff2050694&name_0=MasterCI&name_1=AST%20fuzzer%20%28amd_ubsan%29 +SELECT arrayFold((acc, x) -> plus(acc, toString(NULL, toLowCardinality(toUInt128(4)), materialize(4), 'aaaa', materialize(4), 4, 4, 1), x), range(number), ((acc, x) -> if(x % 2, arrayPushFront(acc, x), arrayPushBack(acc, x)))) FROM system.numbers LIMIT 0; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } diff --git a/tests/queries/0_stateless/03915_bech32_invalid_witness_version.reference b/tests/queries/0_stateless/03915_bech32_invalid_witness_version.reference new file mode 100644 index 000000000000..55b1c2ff759b --- /dev/null +++ b/tests/queries/0_stateless/03915_bech32_invalid_witness_version.reference @@ -0,0 +1,3 @@ +bc1qw3jhxaq2azfhz +bc1pw3jhxaqz7j562 +bc1sw3jhxaq2aj0ly diff --git a/tests/queries/0_stateless/03915_bech32_invalid_witness_version.sql b/tests/queries/0_stateless/03915_bech32_invalid_witness_version.sql new file mode 100644 index 000000000000..2182f04eba2e --- /dev/null +++ b/tests/queries/0_stateless/03915_bech32_invalid_witness_version.sql @@ -0,0 +1,19 @@ +-- Tags: no-fasttest +-- Test valid witness versions (0-16) +SELECT bech32Encode('bc', 'test', 0); +SELECT bech32Encode('bc', 'test', 1); +SELECT bech32Encode('bc', 'test', 16); + +-- Test invalid witness versions (should throw BAD_ARGUMENTS exception) +SELECT bech32Encode('bc', 'test', 17); -- { serverError BAD_ARGUMENTS } +SELECT bech32Encode('bc', 'test', 32); -- { serverError BAD_ARGUMENTS } +SELECT bech32Encode('bc', 'test', 40); -- { serverError BAD_ARGUMENTS } +SELECT bech32Encode('bc', 'test', 255); -- { serverError BAD_ARGUMENTS } + +-- Test the original fuzzer repro case (witness version 40 causing buffer overflow) +DROP TABLE IF EXISTS hex_data_test; +SET allow_suspicious_low_cardinality_types=1; +CREATE TABLE hex_data_test (hrp String, data String, witver LowCardinality(UInt8)) ENGINE = Memory; +INSERT INTO hex_data_test VALUES ('bc', 'test_data', 0); +SELECT bech32Encode(hrp, toFixedString('751e76e8199196d454941c45d1b3a323f1433bd6', 40), 40) FROM hex_data_test; -- { serverError BAD_ARGUMENTS } +DROP TABLE hex_data_test; diff --git a/tests/queries/0_stateless/03916_ttl_minmax_projection_epoch_bug.reference b/tests/queries/0_stateless/03916_ttl_minmax_projection_epoch_bug.reference new file mode 100644 index 000000000000..b261da18d51a --- /dev/null +++ b/tests/queries/0_stateless/03916_ttl_minmax_projection_epoch_bug.reference @@ -0,0 +1,2 @@ +1 +0 diff --git a/tests/queries/0_stateless/03916_ttl_minmax_projection_epoch_bug.sql b/tests/queries/0_stateless/03916_ttl_minmax_projection_epoch_bug.sql new file mode 100644 index 000000000000..b56e198112e2 --- /dev/null +++ b/tests/queries/0_stateless/03916_ttl_minmax_projection_epoch_bug.sql @@ -0,0 +1,29 @@ + + +DROP TABLE IF EXISTS test_ttl_minmax_epoch; + +CREATE TABLE test_ttl_minmax_epoch +( + timestamp DateTime64(9, 'UTC') +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(timestamp) +ORDER BY (timestamp) +TTL timestamp + INTERVAL 1 MINUTE SETTINGS index_granularity = 1; + +-- rows from ~1-60 seconds ago, some will expire during merge +INSERT INTO test_ttl_minmax_epoch +SELECT + now64(9, 'UTC') - toIntervalSecond(1 + rand() % 60) AS timestamp +FROM numbers(1000); + +OPTIMIZE TABLE test_ttl_minmax_epoch FINAL; + +SELECT (SELECT min(timestamp) FROM test_ttl_minmax_epoch) = + (SELECT min(timestamp) FROM test_ttl_minmax_epoch SETTINGS optimize_use_implicit_projections = 0); + +SELECT countIf(min_time < '1971-01-01') AS parts_with_epoch_mintime +FROM system.parts +WHERE table = 'test_ttl_minmax_epoch' AND database = currentDatabase() AND active; + +DROP TABLE test_ttl_minmax_epoch; diff --git a/tests/queries/0_stateless/03916_window_functions_group_by_use_nulls.reference b/tests/queries/0_stateless/03916_window_functions_group_by_use_nulls.reference new file mode 100644 index 000000000000..cf4a6235a6b3 --- /dev/null +++ b/tests/queries/0_stateless/03916_window_functions_group_by_use_nulls.reference @@ -0,0 +1,31 @@ + +a +--- +a +a +a +a +a +a +a +a + +a +--- +a +\N +--- +hello world +hello world +hello world +--- +test +test +test +--- +\N x +x \N +--- +1 42 3 +1 42 3 +1 42 3 diff --git a/tests/queries/0_stateless/03916_window_functions_group_by_use_nulls.sql b/tests/queries/0_stateless/03916_window_functions_group_by_use_nulls.sql new file mode 100644 index 000000000000..2cd62789abe3 --- /dev/null +++ b/tests/queries/0_stateless/03916_window_functions_group_by_use_nulls.sql @@ -0,0 +1,35 @@ +SET enable_analyzer = 1; + +-- https://github.com/ClickHouse/ClickHouse/issues/82499 +-- Window functions with `group_by_use_nulls = 1` and CUBE/ROLLUP/GROUPING SETS +-- could crash because the aggregate function was created with non-nullable argument +-- types, but the actual data columns became nullable after GROUP BY. + +-- Original reproducer from the issue: +CREATE DICTIONARY d0 (c0 Int) PRIMARY KEY (c0) SOURCE(NULL()) LAYOUT(HASHED()) LIFETIME(1); +SELECT min('a') OVER () FROM d0 GROUP BY 'a', c0 WITH CUBE WITH TOTALS SETTINGS group_by_use_nulls = 1; +DROP DICTIONARY d0; + +SELECT '---'; + +SELECT min('a') OVER () FROM numbers(3) GROUP BY 'a', number WITH CUBE WITH TOTALS SETTINGS group_by_use_nulls = 1; + +SELECT '---'; + +WITH 'a' AS x SELECT leadInFrame(x) OVER (ORDER BY x NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) GROUP BY ROLLUP(x) ORDER BY 1 NULLS LAST SETTINGS group_by_use_nulls = 1; + +SELECT '---'; + +SELECT max('hello') OVER (), min('world') OVER () FROM numbers(2) GROUP BY number WITH ROLLUP SETTINGS group_by_use_nulls = 1; + +SELECT '---'; + +SELECT any('test') OVER () FROM numbers(2) GROUP BY GROUPING SETS(('test', number), ('test')) SETTINGS group_by_use_nulls = 1; + +SELECT '---'; + +WITH 'x' AS v SELECT lag(v) OVER (ORDER BY v), lead(v) OVER (ORDER BY v) GROUP BY ROLLUP(v) ORDER BY 1 NULLS FIRST SETTINGS group_by_use_nulls = 1, enable_analyzer = 1; -- lag/lead require the analyzer + +SELECT '---'; + +SELECT min(1) OVER (), max(42) OVER (), sum(1) OVER () FROM numbers(2) GROUP BY number WITH ROLLUP SETTINGS group_by_use_nulls = 1; diff --git a/tests/queries/0_stateless/03925_json_shared_data_buckets_missing_stream_bug.reference b/tests/queries/0_stateless/03925_json_shared_data_buckets_missing_stream_bug.reference new file mode 100644 index 000000000000..e9c6b09c7b3c --- /dev/null +++ b/tests/queries/0_stateless/03925_json_shared_data_buckets_missing_stream_bug.reference @@ -0,0 +1,20 @@ +0 {"a":42} +0 {"a":42} +1 {"a":42} +1 {"a":42} +2 {"a":42} +2 {"a":42} +3 {"a":42} +3 {"a":42} +4 {"a":42} +4 {"a":42} +5 {"a":42} +5 {"a":42} +6 {"a":42} +6 {"a":42} +7 {"a":42} +7 {"a":42} +8 {"a":42} +8 {"a":42} +9 {"a":42} +9 {"a":42} diff --git a/tests/queries/0_stateless/03925_json_shared_data_buckets_missing_stream_bug.sql b/tests/queries/0_stateless/03925_json_shared_data_buckets_missing_stream_bug.sql new file mode 100644 index 000000000000..dafa2b2065fb --- /dev/null +++ b/tests/queries/0_stateless/03925_json_shared_data_buckets_missing_stream_bug.sql @@ -0,0 +1,11 @@ +drop table if exists src; +drop table if exists dst; +create table src (id UInt64, json JSON) engine=MergeTree order by id settings min_bytes_for_wide_part=1, object_serialization_version='v3', object_shared_data_serialization_version_for_zero_level_parts='map_with_buckets'; +create table dst (id UInt64, json JSON) engine=MergeTree order by id settings min_bytes_for_wide_part=1, object_serialization_version='v3', object_shared_data_serialization_version_for_zero_level_parts='map_with_buckets'; +insert into src select number, '{"a" : 42}' from numbers(10); +insert into src select number, '{"a" : 42}' from numbers(10); +insert into dst select * from src order by id desc; +select * from dst order by id; +drop table src; +drop table dst; + diff --git a/tests/queries/0_stateless/03928_json_advanced_shared_data_bug.reference b/tests/queries/0_stateless/03928_json_advanced_shared_data_bug.reference new file mode 100644 index 000000000000..c0d2ee3f3b2a --- /dev/null +++ b/tests/queries/0_stateless/03928_json_advanced_shared_data_bug.reference @@ -0,0 +1,30 @@ +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{} +{} +{} +{} +{} +{} +{} +{} +{} +{} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} +{"a":[{"b":42}]} diff --git a/tests/queries/0_stateless/03928_json_advanced_shared_data_bug.sql b/tests/queries/0_stateless/03928_json_advanced_shared_data_bug.sql new file mode 100644 index 000000000000..c54c8378ae3c --- /dev/null +++ b/tests/queries/0_stateless/03928_json_advanced_shared_data_bug.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS test; + +CREATE TABLE test +( + `json` JSON(max_dynamic_paths = 1) +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 1, min_rows_for_wide_part = 1, write_marks_for_substreams_in_compact_parts = 1, object_serialization_version = 'v3', object_shared_data_serialization_version = 'advanced', object_shared_data_serialization_version_for_zero_level_parts = 'advanced', object_shared_data_buckets_for_wide_part = 1, index_granularity = 100; + +INSERT INTO test SELECT multiIf(number < 10, '{"a" : [{"b" : 42}]}', number < 20, '{}', '{"a" : [{"b" : 42}]}') from numbers(30); + +SELECT * FROM test SETTINGS max_block_size=10; + +DROP TABLE test; + diff --git a/tests/queries/0_stateless/03988_grace_hash_join_leftover_blocks.reference b/tests/queries/0_stateless/03988_grace_hash_join_leftover_blocks.reference new file mode 100644 index 000000000000..3894b6e08587 --- /dev/null +++ b/tests/queries/0_stateless/03988_grace_hash_join_leftover_blocks.reference @@ -0,0 +1,6 @@ +9 +9 +9 +9 +9 +9 diff --git a/tests/queries/0_stateless/03988_grace_hash_join_leftover_blocks.sql b/tests/queries/0_stateless/03988_grace_hash_join_leftover_blocks.sql new file mode 100644 index 000000000000..521ca6af3cf3 --- /dev/null +++ b/tests/queries/0_stateless/03988_grace_hash_join_leftover_blocks.sql @@ -0,0 +1,79 @@ +DROP TABLE IF EXISTS test; +DROP TABLE IF EXISTS test2; + +SET max_joined_block_size_rows = 5; +SET enable_analyzer = 1; + +CREATE TABLE test +( + c0 Int, + c1 Date +) +ENGINE = MergeTree() +ORDER BY (c1); + +INSERT INTO test (c0, c1) VALUES +(1,'1995-01-28'), +(1,'1995-01-29'), +(1,'1995-01-30'); + +CREATE TABLE test2 +( + c0 Int, + c1 Date +) +ENGINE = MergeTree() +ORDER BY (c0); + +INSERT INTO test2 (c1, c0) VALUES +('1992-12-14',1), +('1992-12-14',1), +('1989-05-06',1); + +SELECT + count() +FROM test +LEFT JOIN test2 + ON test.c0 = test2.c0 + AND test.c1 >= test2.c1 +SETTINGS join_algorithm='parallel_hash'; + +SELECT + count() +FROM test2 +LEFT JOIN test + ON test.c0 = test2.c0 + AND test.c1 >= test2.c1 +SETTINGS join_algorithm='grace_hash', grace_hash_join_initial_buckets = 2; + +SELECT + count() +FROM test +RIGHT JOIN test2 + ON test.c0 = test2.c0 + AND test.c1 >= test2.c1 +SETTINGS join_algorithm='parallel_hash'; + +SELECT + count() +FROM test2 +RIGHT JOIN test + ON test.c0 = test2.c0 + AND test.c1 >= test2.c1 +SETTINGS join_algorithm='grace_hash', grace_hash_join_initial_buckets = 2; + +SELECT + count() +FROM test +FULL JOIN test2 + ON test.c0 = test2.c0 + AND test.c1 >= test2.c1 +SETTINGS join_algorithm='parallel_hash'; + +SELECT + count() +FROM test2 +FULL JOIN test + ON test.c0 = test2.c0 + AND test.c1 >= test2.c1 +SETTINGS join_algorithm='grace_hash', grace_hash_join_initial_buckets = 2; \ No newline at end of file diff --git a/tests/queries/0_stateless/03988_map_contains_key_like_tokenbf.reference b/tests/queries/0_stateless/03988_map_contains_key_like_tokenbf.reference new file mode 100644 index 000000000000..599d91218700 --- /dev/null +++ b/tests/queries/0_stateless/03988_map_contains_key_like_tokenbf.reference @@ -0,0 +1,18 @@ +1 +0 +1 +1 +1 +1 +1 +1 +0 +hostname +Verify skip index is used +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/03988_map_contains_key_like_tokenbf.sql b/tests/queries/0_stateless/03988_map_contains_key_like_tokenbf.sql new file mode 100644 index 000000000000..d14c3fc91371 --- /dev/null +++ b/tests/queries/0_stateless/03988_map_contains_key_like_tokenbf.sql @@ -0,0 +1,65 @@ +-- Test for issue https://github.com/ClickHouse/ClickHouse/issues/97792 + +SET parallel_replicas_local_plan = 1; + +DROP TABLE IF EXISTS t_map_tokenbf; + +CREATE TABLE t_map_tokenbf +( + metadata Map(String, String), + created_at DateTime64(3), + INDEX index_metadata_keys mapKeys(metadata) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1, + INDEX index_metadata_vals mapValues(metadata) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY created_at; + +INSERT INTO t_map_tokenbf VALUES ({'hostname': 'myhost', 'env': 'prod'}, now()); + +SELECT count() FROM t_map_tokenbf WHERE mapContainsKeyLike(metadata, '%host%'); -- 1 +SELECT count() FROM t_map_tokenbf WHERE mapContainsKeyLike(metadata, '%bad%'); -- 0 +SELECT count() FROM t_map_tokenbf WHERE mapContains(metadata, 'hostname'); -- 1 +SELECT count() FROM t_map_tokenbf WHERE mapContainsKey(metadata, 'env'); -- 1 +SELECT count() FROM t_map_tokenbf WHERE has(mapKeys(metadata), 'env'); -- 1 +SELECT count() FROM t_map_tokenbf WHERE has(metadata, 'hostname'); -- 1 +SELECT count() FROM t_map_tokenbf WHERE mapContainsValue(metadata, 'prod'); -- 1 +SELECT count() FROM t_map_tokenbf WHERE mapContainsValueLike(metadata, '%host%'); -- 1 +SELECT count() FROM t_map_tokenbf WHERE mapContainsValueLike(metadata, '%random%'); -- 0 + +SELECT arrayJoin(mapKeys(mapExtractKeyLike(metadata, '%host%'))) as extracted_metadata +FROM t_map_tokenbf +WHERE mapContainsKeyLike(metadata, '%host%') +GROUP BY extracted_metadata; + +-- Verify that skip index was used - all should return 1 +SELECT 'Verify skip index is used'; + +SELECT COUNT(*) FROM ( + EXPLAIN indexes=1 SELECT count() FROM t_map_tokenbf WHERE mapContainsKeyLike(metadata, '%host%') + ) WHERE explain LIKE '%index_metadata%'; + +SELECT COUNT(*) FROM ( + EXPLAIN indexes=1 SELECT count() FROM t_map_tokenbf WHERE mapContains(metadata, 'hostname') + ) WHERE explain LIKE '%index_metadata%'; + +SELECT COUNT(*) FROM ( + EXPLAIN indexes=1 SELECT count() FROM t_map_tokenbf WHERE mapContainsKey(metadata, 'env') + ) WHERE explain LIKE '%index_metadata%'; + +SELECT COUNT(*) FROM ( + EXPLAIN indexes=1 SELECT count() FROM t_map_tokenbf WHERE has(mapKeys(metadata), 'env') + ) WHERE explain LIKE '%index_metadata%'; + +SELECT COUNT(*) FROM ( + EXPLAIN indexes=1 SELECT count() FROM t_map_tokenbf WHERE has(metadata, 'hostname') + ) WHERE explain LIKE '%index_metadata%'; + +SELECT COUNT(*) FROM ( + EXPLAIN indexes=1 SELECT count() FROM t_map_tokenbf WHERE mapContainsValue(metadata, 'prod') + ) WHERE explain LIKE '%index_metadata%'; + +SELECT COUNT(*) FROM ( + EXPLAIN indexes=1 SELECT count() FROM t_map_tokenbf WHERE mapContainsValueLike(metadata, '%random%') + ) WHERE explain LIKE '%index_metadata%'; + +DROP TABLE t_map_tokenbf; diff --git a/tests/queries/0_stateless/03988_zookeeper_send_receive_race.reference b/tests/queries/0_stateless/03988_zookeeper_send_receive_race.reference new file mode 100644 index 000000000000..d86bac9de59a --- /dev/null +++ b/tests/queries/0_stateless/03988_zookeeper_send_receive_race.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/03988_zookeeper_send_receive_race.sh b/tests/queries/0_stateless/03988_zookeeper_send_receive_race.sh new file mode 100755 index 000000000000..8b4a8a3d92f4 --- /dev/null +++ b/tests/queries/0_stateless/03988_zookeeper_send_receive_race.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-fasttest + +# Regression test for a data race in ZooKeeper client between sendThread and receiveThread. +# +# sendThread used to mutate the request (addRootPath, has_watch) AFTER copying it +# into the operations map, while receiveThread could concurrently read from the +# same shared request object via the operations map. This caused a data race on +# the request's path string (std::string reallocation during addRootPath vs +# concurrent getPath() read), leading to SIGBUS/use-after-free crashes. +# +# Under TSAN this test reliably detects the race before the fix. +# The key is to generate many concurrent ZooKeeper requests through the server's +# shared ZK session so sendThread and receiveThread are both actively working on +# the operations map at the same time. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS t_zk_race; + CREATE TABLE t_zk_race (key UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_zk_race', 'r1') + ORDER BY key; +" + +ZK_PATH="/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_zk_race" + +# Flood the server's shared ZK connection with concurrent reads from +# system.zookeeper. Each SELECT issues ZK list/get requests that go through +# sendThread (addRootPath + operations map insert) and receiveThread +# (operations map read for timeout + response handling) on the same session. +# +# Use clickhouse-benchmark for maximum ZK operations/sec on a single session. +# --timelimit ensures the test runs long enough for TSAN to catch the race. +echo "SELECT count() FROM system.zookeeper WHERE path = '$ZK_PATH' FORMAT Null" | \ + ${CLICKHOUSE_BENCHMARK} --concurrency 30 --iterations 100000 --timelimit 10 2>&1 | grep -q "Executed" || true + +echo "OK" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_zk_race" diff --git a/tests/queries/0_stateless/04001_virtual_row_conversions_join_column_names.reference b/tests/queries/0_stateless/04001_virtual_row_conversions_join_column_names.reference new file mode 100644 index 000000000000..3d454cd4efb8 --- /dev/null +++ b/tests/queries/0_stateless/04001_virtual_row_conversions_join_column_names.reference @@ -0,0 +1,16 @@ +-10 +-1 +0 +0 +1 +10 +- +0 1 +0 1 +0 2 +0 2 +- +0 1 +0 1 +0 2 +0 2 diff --git a/tests/queries/0_stateless/04001_virtual_row_conversions_join_column_names.sql b/tests/queries/0_stateless/04001_virtual_row_conversions_join_column_names.sql new file mode 100644 index 000000000000..30e1af99e362 --- /dev/null +++ b/tests/queries/0_stateless/04001_virtual_row_conversions_join_column_names.sql @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS t0; +DROP TABLE IF EXISTS t1; + +SET allow_suspicious_low_cardinality_types = 1; +SET enable_analyzer = 1; +CREATE TABLE t0 (c0 LowCardinality(Int)) ENGINE = MergeTree() ORDER BY (c0); +CREATE TABLE t1 (c0 Nullable(Int)) ENGINE = MergeTree() ORDER BY tuple(); + +INSERT INTO TABLE t0 (c0) VALUES (0), (1); +INSERT INTO TABLE t0 (c0) VALUES (-10), (10); +INSERT INTO TABLE t0 (c0) VALUES (0), (-1); +INSERT INTO TABLE t1 (c0) VALUES (1), (2); + +SET read_in_order_use_virtual_row = 1; + + +SELECT CAST(c0, 'Int32') a FROM t0 ORDER BY a; + +SELECT '-'; +SELECT * FROM t0 JOIN t1 ON t1.c0.null = t0.c0 +ORDER BY t0.c0, t1.c0; + +SELECT '-'; + +SELECT * FROM t0 JOIN t1 ON t1.c0.null = t0.c0 +ORDER BY t0.c0, t1.c0 +SETTINGS join_algorithm = 'full_sorting_merge'; diff --git a/tests/queries/0_stateless/04003_array_join_in_filter_outer_to_inner_join.reference b/tests/queries/0_stateless/04003_array_join_in_filter_outer_to_inner_join.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/04003_array_join_in_filter_outer_to_inner_join.sql b/tests/queries/0_stateless/04003_array_join_in_filter_outer_to_inner_join.sql new file mode 100644 index 000000000000..3d63ae1a423d --- /dev/null +++ b/tests/queries/0_stateless/04003_array_join_in_filter_outer_to_inner_join.sql @@ -0,0 +1,19 @@ +-- Regression: segfault in executeActionForPartialResult when filter expression contains arrayJoin +-- and the convertOuterJoinToInnerJoin optimization tries to evaluate the filter with partial (null) arguments. + +SET enable_analyzer = 1; +SELECT DISTINCT + 2, + 1048575 +FROM numbers(1) AS l, + numbers(2, isZeroOrNull(assumeNotNull(1))) AS r +ANY INNER JOIN r AS alias37 ON equals(alias37.number, r.number) +RIGHT JOIN l AS alias44 ON equals(alias44.number, alias37.number) +ANY INNER JOIN alias44 AS alias48 ON equals(alias48.number, r.number) +ANY RIGHT JOIN r AS alias52 ON equals(alias52.number, alias37.number) +WHERE equals(isNull(toLowCardinality(toUInt128(2))), arrayJoin([*, 13, 13, 13, toNullable(13), 13])) +GROUP BY + materialize(1), + isNull(toUInt128(2)), + and(and(1048575, isZeroOrNull(1), isNullable(isNull(1))), materialize(13), isNull(toUInt256(materialize(2))), *, and(*, and(1, nan, isNull(isNull(1)), isZeroOrNull(1), 1048575), 13)) +WITH CUBE; diff --git a/tests/queries/0_stateless/04004_view_comment_before_as_select.reference b/tests/queries/0_stateless/04004_view_comment_before_as_select.reference new file mode 100644 index 000000000000..6353405096a4 --- /dev/null +++ b/tests/queries/0_stateless/04004_view_comment_before_as_select.reference @@ -0,0 +1,2 @@ +CREATE VIEW v\nAS (SELECT 1)\nCOMMENT \'test\' +CREATE MATERIALIZED VIEW v\nENGINE = MergeTree\nORDER BY c\nAS (SELECT 1 AS c)\nCOMMENT \'test\' diff --git a/tests/queries/0_stateless/04004_view_comment_before_as_select.sql b/tests/queries/0_stateless/04004_view_comment_before_as_select.sql new file mode 100644 index 000000000000..91b761945067 --- /dev/null +++ b/tests/queries/0_stateless/04004_view_comment_before_as_select.sql @@ -0,0 +1,3 @@ +-- Forward compatibility: accept COMMENT before AS SELECT (syntax produced by 26.2+). +SELECT formatQuery('CREATE VIEW v COMMENT \'test\' AS SELECT 1'); +SELECT formatQuery('CREATE MATERIALIZED VIEW v ENGINE = MergeTree ORDER BY c COMMENT \'test\' AS SELECT 1 AS c'); diff --git a/tests/queries/0_stateless/04023_issue_98484_drop_patch_part.reference b/tests/queries/0_stateless/04023_issue_98484_drop_patch_part.reference new file mode 100644 index 000000000000..d86bac9de59a --- /dev/null +++ b/tests/queries/0_stateless/04023_issue_98484_drop_patch_part.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/04023_issue_98484_drop_patch_part.sh b/tests/queries/0_stateless/04023_issue_98484_drop_patch_part.sh new file mode 100755 index 000000000000..c416c1405b33 --- /dev/null +++ b/tests/queries/0_stateless/04023_issue_98484_drop_patch_part.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: requires lightweight_delete_mode setting + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Test for issue #98484: DROP PART on patch part should not crash server. +# The bug was that getPatchPartMetadata() built a partition key expression +# referencing _part column, but the ColumnsDescription passed from +# createEmptyPart() only contained data columns, causing UNKNOWN_IDENTIFIER +# inside a NOEXCEPT_SCOPE which triggered std::terminate(). + +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE t_98484 (c0 Int32, c1 String, c2 Int8) + ENGINE = MergeTree() ORDER BY tuple() + SETTINGS enable_block_offset_column = 1, enable_block_number_column = 1 +" + +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_98484 VALUES (1, 'hello', 10)" +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_98484 VALUES (2, 'world', 20)" +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_98484 VALUES (3, 'test', 30)" + +# Create patch parts via lightweight delete +${CLICKHOUSE_CLIENT} --query "SET lightweight_delete_mode = 'lightweight_update_force'; DELETE FROM t_98484 WHERE c0 = 1" + +# Wait for mutations to complete +for _ in $(seq 1 30); do + result=$(${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_98484' AND name LIKE 'patch-%' AND active = 1") + if [ "$result" -ge 1 ]; then + break + fi + sleep 0.5 +done + +# Add column to change columns description (original trigger condition) +${CLICKHOUSE_CLIENT} --query "ALTER TABLE t_98484 ADD COLUMN c9 Nullable(Bool)" + +# Get the first active patch part name +PATCH_PART=$(${CLICKHOUSE_CLIENT} --query " + SELECT name FROM system.parts + WHERE database = currentDatabase() AND table = 't_98484' + AND name LIKE 'patch-%' AND active = 1 + ORDER BY name LIMIT 1 +") + +if [ -z "$PATCH_PART" ]; then + echo "FAIL: No patch parts found" + exit 1 +fi + +# DROP PART on the patch part - this should not crash the server +${CLICKHOUSE_CLIENT} --query "ALTER TABLE t_98484 DROP PART '$PATCH_PART'" 2>&1 + +# Verify server is still alive +${CLICKHOUSE_CLIENT} --query "SELECT 1" > /dev/null 2>&1 +if [ $? -ne 0 ]; then + echo "FAIL: Server crashed" + exit 1 +fi + +echo "OK" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE t_98484" diff --git a/tests/queries/0_stateless/04027_reverseUTF8_invalid_utf8.reference b/tests/queries/0_stateless/04027_reverseUTF8_invalid_utf8.reference new file mode 100644 index 000000000000..1f0eb51aa4f8 --- /dev/null +++ b/tests/queries/0_stateless/04027_reverseUTF8_invalid_utf8.reference @@ -0,0 +1,3 @@ +esuoHkcilC +тевирП +はちにんこ diff --git a/tests/queries/0_stateless/04027_reverseUTF8_invalid_utf8.sql b/tests/queries/0_stateless/04027_reverseUTF8_invalid_utf8.sql new file mode 100644 index 000000000000..0267c46640d4 --- /dev/null +++ b/tests/queries/0_stateless/04027_reverseUTF8_invalid_utf8.sql @@ -0,0 +1,15 @@ +-- Test that reverseUTF8 does not crash on invalid UTF-8 (truncated multi-byte sequences) +SELECT reverseUTF8(unhex('C0')) FORMAT Null; +SELECT reverseUTF8(unhex('E0')) FORMAT Null; +SELECT reverseUTF8(unhex('F0')) FORMAT Null; +SELECT reverseUTF8(unhex('E0A0')) FORMAT Null; +SELECT reverseUTF8(unhex('F09F')) FORMAT Null; +SELECT reverseUTF8(unhex('F09F98')) FORMAT Null; + +-- The original crash query from the AST fuzzer +SELECT DISTINCT reverseUTF8(maxMergeDistinct(x) IGNORE NULLS), toNullable(1) FROM (SELECT DISTINCT dictHas(tuple(toUInt16(NULL)), 13, toUInt32(6), NULL), CAST(concat(unhex('00001000'), randomString(intDiv(1048576, toNullable(1))), toLowCardinality(toFixedString('\0', 1))), 'AggregateFunction(max, String)') AS x) WITH TOTALS FORMAT Null; + +-- Verify correct behavior on valid UTF-8 +SELECT reverseUTF8('ClickHouse'); +SELECT reverseUTF8('Привет'); +SELECT reverseUTF8('こんにちは'); diff --git a/tests/queries/0_stateless/04028_recursive_cte_remote_view_segfault.reference b/tests/queries/0_stateless/04028_recursive_cte_remote_view_segfault.reference new file mode 100644 index 000000000000..6ed281c757a9 --- /dev/null +++ b/tests/queries/0_stateless/04028_recursive_cte_remote_view_segfault.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/04028_recursive_cte_remote_view_segfault.sql b/tests/queries/0_stateless/04028_recursive_cte_remote_view_segfault.sql new file mode 100644 index 000000000000..f397bff986a7 --- /dev/null +++ b/tests/queries/0_stateless/04028_recursive_cte_remote_view_segfault.sql @@ -0,0 +1,13 @@ +-- Tags: no-fasttest +-- Regression test: recursive CTE with remote() + view() used to segfault +-- because isStorageUsedInTree tried to call getStorageID() on an unresolved +-- view() TableFunctionNode whose storage was null. + +SET enable_analyzer=1; + +WITH RECURSIVE x AS ( + (SELECT 1 FROM remote('127.0.0.1', view(SELECT 1))) + UNION ALL + (SELECT 1) +) +SELECT 1 FROM x; diff --git a/tests/queries/0_stateless/04033_except_transformer_with_alias.reference b/tests/queries/0_stateless/04033_except_transformer_with_alias.reference new file mode 100644 index 000000000000..acc4c297f619 --- /dev/null +++ b/tests/queries/0_stateless/04033_except_transformer_with_alias.reference @@ -0,0 +1,16 @@ +1 x 3 +2 y 5 +1 1.5 +2 2.5 +1 1.5 +2 2.5 +1 X 15 +2 Y 25 +1 x 3 +2 y 5 +1 1.5 +2 2.5 +1 1.5 +2 2.5 +1 X 15 +2 Y 25 diff --git a/tests/queries/0_stateless/04033_except_transformer_with_alias.sql b/tests/queries/0_stateless/04033_except_transformer_with_alias.sql new file mode 100644 index 000000000000..739c252f9ff6 --- /dev/null +++ b/tests/queries/0_stateless/04033_except_transformer_with_alias.sql @@ -0,0 +1,64 @@ +-- Test that SELECT * EXCEPT (col) works when 'col' is also used as an alias in the same SELECT. +-- This used to cause LOGICAL_ERROR "Bad cast from type DB::ASTFunction to DB::ASTIdentifier" +-- because QueryNormalizer could replace the ASTIdentifier inside the EXCEPT transformer +-- with an ASTFunction from the alias before the transformer was expanded. +-- https://github.com/ClickHouse/clickhouse-core-incidents/issues/1433 + +SET allow_experimental_analyzer = 0; + +DROP TABLE IF EXISTS t_except_alias; +CREATE TABLE t_except_alias (a UInt64, b String, c Float64) ENGINE = Memory; +INSERT INTO t_except_alias VALUES (1, 'x', 1.5), (2, 'y', 2.5); + +-- Basic case: EXCEPT column name matches an alias in the same SELECT +SELECT * EXCEPT (c), toFloat64(c) * 2 AS c FROM t_except_alias ORDER BY a; + +-- Same pattern but via subquery in JOIN (triggers interpretSubquery with removeDuplicates) +SELECT t.a, sub.c +FROM t_except_alias AS t +LEFT JOIN ( + SELECT * EXCEPT (c), toString(c) AS c FROM t_except_alias +) AS sub ON t.a = sub.a +ORDER BY t.a; + +-- CTE pattern matching the original incident +WITH data AS ( + SELECT * EXCEPT (c), toFloat64(c) AS c FROM t_except_alias +) +SELECT t.a, d.c +FROM t_except_alias AS t +LEFT JOIN data AS d ON t.a = d.a +ORDER BY t.a; + +-- Multiple columns in EXCEPT with aliases +SELECT * EXCEPT (b, c), upper(b) AS b, toFloat64(c) * 10 AS c FROM t_except_alias ORDER BY a; + +DROP TABLE t_except_alias; + +-- Now test the same with the new analyzer +SET allow_experimental_analyzer = 1; + +DROP TABLE IF EXISTS t_except_alias; +CREATE TABLE t_except_alias (a UInt64, b String, c Float64) ENGINE = Memory; +INSERT INTO t_except_alias VALUES (1, 'x', 1.5), (2, 'y', 2.5); + +SELECT * EXCEPT (c), toFloat64(c) * 2 AS c FROM t_except_alias ORDER BY a; + +SELECT t.a, sub.c +FROM t_except_alias AS t +LEFT JOIN ( + SELECT * EXCEPT (c), toString(c) AS c FROM t_except_alias +) AS sub ON t.a = sub.a +ORDER BY t.a; + +WITH data AS ( + SELECT * EXCEPT (c), toFloat64(c) AS c FROM t_except_alias +) +SELECT t.a, d.c +FROM t_except_alias AS t +LEFT JOIN data AS d ON t.a = d.a +ORDER BY t.a; + +SELECT * EXCEPT (b, c), upper(b) AS b, toFloat64(c) * 10 AS c FROM t_except_alias ORDER BY a; + +DROP TABLE t_except_alias; diff --git a/tests/queries/0_stateless/04034_patch_parts_column_order_mismatch.reference b/tests/queries/0_stateless/04034_patch_parts_column_order_mismatch.reference new file mode 100644 index 000000000000..4287567495fe --- /dev/null +++ b/tests/queries/0_stateless/04034_patch_parts_column_order_mismatch.reference @@ -0,0 +1,6 @@ +1 updated1 99 9.9 999 upd1 +2 updated1 99 9.9 999 upd1 +1 updated2 88 8.8 888 upd2 +2 updated2 88 8.8 888 upd2 +1 updated2 88 8.8 888 upd2 +2 updated2 88 8.8 888 upd2 diff --git a/tests/queries/0_stateless/04034_patch_parts_column_order_mismatch.sql b/tests/queries/0_stateless/04034_patch_parts_column_order_mismatch.sql new file mode 100644 index 000000000000..3c1d29ede8d4 --- /dev/null +++ b/tests/queries/0_stateless/04034_patch_parts_column_order_mismatch.sql @@ -0,0 +1,50 @@ +-- Regression test for https://github.com/ClickHouse/clickhouse-core-incidents/issues/1021 +-- When multiple patch parts (Merge + Join mode) update the same columns, +-- the column ordering in patch blocks must be deterministic to avoid +-- LOGICAL_ERROR "Block structure mismatch in patch parts stream". +-- +-- The failpoint reverses column order for odd-indexed patches to expose any +-- code relying on positional column matching. Without the sort in +-- getUpdatedHeader, this triggers the bug. + +SET enable_lightweight_update = 1; + +SYSTEM ENABLE FAILPOINT patch_parts_reverse_column_order; + +DROP TABLE IF EXISTS t_patch_order; + +CREATE TABLE t_patch_order (id UInt64, a_col String, b_col UInt64, c_col Float64, d_col UInt32, e_col String) +ENGINE = MergeTree ORDER BY id +SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + apply_patches_on_merge = 0; + +-- Insert two separate blocks to create two base parts. +INSERT INTO t_patch_order VALUES (1, 'hello', 10, 1.5, 100, 'world'); +INSERT INTO t_patch_order VALUES (2, 'foo', 20, 2.5, 200, 'bar'); + +-- First UPDATE: creates Merge-mode patch parts for both base parts. +UPDATE t_patch_order SET a_col = 'updated1', b_col = 99, c_col = 9.9, d_col = 999, e_col = 'upd1' WHERE 1; + +-- Verify patch application works in Merge mode. +SELECT * FROM t_patch_order ORDER BY id; + +-- Merge base parts; patches become Join-mode (apply_patches_on_merge = 0). +OPTIMIZE TABLE t_patch_order FINAL; + +-- Second UPDATE: creates new Merge-mode patch parts for the merged base part. +UPDATE t_patch_order SET a_col = 'updated2', b_col = 88, c_col = 8.8, d_col = 888, e_col = 'upd2' WHERE 1; + +-- This SELECT must apply both Join-mode and Merge-mode patches simultaneously. +-- The failpoint reverses column order for odd-indexed patches. Without the fix, +-- getUpdatedHeader throws LOGICAL_ERROR because it compares patch headers positionally. +SELECT * FROM t_patch_order ORDER BY id; + +-- Materialize patches and verify final state. +ALTER TABLE t_patch_order APPLY PATCHES SETTINGS mutations_sync = 2; +SELECT * FROM t_patch_order ORDER BY id SETTINGS apply_patch_parts = 0; + +SYSTEM DISABLE FAILPOINT patch_parts_reverse_column_order; + +DROP TABLE t_patch_order; diff --git a/tests/queries/0_stateless/04038_check_table_sparse_tuple_dynamic.reference b/tests/queries/0_stateless/04038_check_table_sparse_tuple_dynamic.reference new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/tests/queries/0_stateless/04038_check_table_sparse_tuple_dynamic.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/04038_check_table_sparse_tuple_dynamic.sql b/tests/queries/0_stateless/04038_check_table_sparse_tuple_dynamic.sql new file mode 100644 index 000000000000..c406eda00b2e --- /dev/null +++ b/tests/queries/0_stateless/04038_check_table_sparse_tuple_dynamic.sql @@ -0,0 +1,12 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/96588 +-- CHECK TABLE on a Tuple with a Dynamic element and a sparse-serialized element +-- used to fail with "Unexpected size of tuple element" because deserializeOffsets +-- in SerializationSparse treated limit=0 as "read everything" instead of "read nothing". + +DROP TABLE IF EXISTS t0; + +CREATE TABLE t0 (c0 Tuple(c1 Dynamic, c2 Tuple(c3 Int))) ENGINE = MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 1, ratio_of_defaults_for_sparse_serialization = 0.9; +INSERT INTO TABLE t0 (c0) SELECT (1, (number, ), ) FROM numbers(1); +CHECK TABLE t0; + +DROP TABLE t0; diff --git a/tests/queries/0_stateless/04039_intersect_except_duplicate_column_names.reference b/tests/queries/0_stateless/04039_intersect_except_duplicate_column_names.reference new file mode 100644 index 000000000000..9ac08789a5c9 --- /dev/null +++ b/tests/queries/0_stateless/04039_intersect_except_duplicate_column_names.reference @@ -0,0 +1,4 @@ +1 1 hello world world +2 2 foo bar bar +1 1 hello world world +2 2 foo bar bar diff --git a/tests/queries/0_stateless/04039_intersect_except_duplicate_column_names.sql b/tests/queries/0_stateless/04039_intersect_except_duplicate_column_names.sql new file mode 100644 index 000000000000..66d1f76d3dcc --- /dev/null +++ b/tests/queries/0_stateless/04039_intersect_except_duplicate_column_names.sql @@ -0,0 +1,20 @@ +-- Reproducer for heap-use-after-free in IntersectOrExceptTransform +-- when the header has duplicate column names (e.g., from SELECT col, *, col). +-- The bug was that getPositionByName returned the same position for duplicate names, +-- creating duplicate entries in key_columns_pos. Then convertToFullColumnIfConst +-- on the same position freed the column a raw pointer still referenced. + +DROP TABLE IF EXISTS t_intersect_except; +CREATE TABLE t_intersect_except (id UInt32, a String, b String) ENGINE = Memory; +INSERT INTO t_intersect_except VALUES (1, 'hello', 'world'), (2, 'foo', 'bar'); + +-- SELECT id, *, b produces duplicate column names: id appears twice, b appears twice. +(SELECT id, *, b FROM t_intersect_except ORDER BY id LIMIT 10) EXCEPT DISTINCT (SELECT id, *, b FROM t_intersect_except ORDER BY id LIMIT 10); + +(SELECT id, *, b FROM t_intersect_except ORDER BY id LIMIT 10) INTERSECT DISTINCT (SELECT id, *, b FROM t_intersect_except ORDER BY id LIMIT 10); + +(SELECT id, *, b FROM t_intersect_except ORDER BY id LIMIT 10) EXCEPT ALL (SELECT id, *, b FROM t_intersect_except ORDER BY id LIMIT 10); + +(SELECT id, *, b FROM t_intersect_except ORDER BY id LIMIT 10) INTERSECT ALL (SELECT id, *, b FROM t_intersect_except ORDER BY id LIMIT 10); + +DROP TABLE t_intersect_except; diff --git a/tests/queries/0_stateless/04039_prune_array_join_columns.reference b/tests/queries/0_stateless/04039_prune_array_join_columns.reference new file mode 100644 index 000000000000..692d5c24e54a --- /dev/null +++ b/tests/queries/0_stateless/04039_prune_array_join_columns.reference @@ -0,0 +1,242 @@ +1 +2 +QUERY id: 0 + PROJECTION COLUMNS + n.a Int64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: tupleElement, function_type: ordinary, result_type: Int64 + ARGUMENTS + LIST id: 3, nodes: 2 + COLUMN id: 4, column_name: __array_join_exp_1, result_type: Tuple(a Int64), source_id: 5 + CONSTANT id: 6, constant_value: \'a\', constant_value_type: String + JOIN TREE + ARRAY_JOIN id: 5, is_left: 0 + TABLE EXPRESSION + TABLE id: 7, alias: __table2, table_name: default.t_nested + JOIN EXPRESSIONS + LIST id: 8, nodes: 1 + COLUMN id: 9, alias: __array_join_exp_1, column_name: __array_join_exp_1, result_type: Tuple(a Int64), source_id: 5 + EXPRESSION + FUNCTION id: 10, function_name: nested, function_type: ordinary, result_type: Array(Tuple(a Int64)) + ARGUMENTS + LIST id: 11, nodes: 2 + CONSTANT id: 12, constant_value: Array_[\'a\'], constant_value_type: Array(String) + COLUMN id: 13, column_name: n.a, result_type: Array(Int64), source_id: 7 + ORDER BY + LIST id: 14, nodes: 1 + SORT id: 15, sort_direction: ASCENDING, with_fill: 0 + EXPRESSION + FUNCTION id: 16, function_name: tupleElement, function_type: ordinary, result_type: Int64 + ARGUMENTS + LIST id: 17, nodes: 2 + COLUMN id: 18, column_name: __array_join_exp_1, result_type: Tuple(a Int64), source_id: 5 + CONSTANT id: 19, constant_value: \'a\', constant_value_type: String +Expression (Project names) +Header: n.a Int64 + Sorting (Sorting for ORDER BY) + Header: tupleElement(__array_join_exp_1, \'a\'_String) Int64 + Expression ((Before ORDER BY + Projection)) + Header: tupleElement(__array_join_exp_1, \'a\'_String) Int64 + ArrayJoin (ARRAY JOIN) + Header: __array_join_exp_1 Tuple(a Int64) + Expression ((DROP unused columns before ARRAY JOIN + (ARRAY JOIN actions + Change column names to column identifiers))) + Header: __array_join_exp_1 Array(Tuple(a Int64)) + ReadFromMergeTree (default.t_nested) + Header: n.a Array(Int64) +1 3 +2 4 +QUERY id: 0 + PROJECTION COLUMNS + n.a Int64 + n.b Int64 + PROJECTION + LIST id: 1, nodes: 2 + FUNCTION id: 2, function_name: tupleElement, function_type: ordinary, result_type: Int64 + ARGUMENTS + LIST id: 3, nodes: 2 + COLUMN id: 4, column_name: __array_join_exp_1, result_type: Tuple(a Int64, b Int64), source_id: 5 + CONSTANT id: 6, constant_value: \'a\', constant_value_type: String + FUNCTION id: 7, function_name: tupleElement, function_type: ordinary, result_type: Int64 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: __array_join_exp_1, result_type: Tuple(a Int64, b Int64), source_id: 5 + CONSTANT id: 10, constant_value: \'b\', constant_value_type: String + JOIN TREE + ARRAY_JOIN id: 5, is_left: 0 + TABLE EXPRESSION + TABLE id: 11, alias: __table2, table_name: default.t_nested + JOIN EXPRESSIONS + LIST id: 12, nodes: 1 + COLUMN id: 13, alias: __array_join_exp_1, column_name: __array_join_exp_1, result_type: Tuple(a Int64, b Int64), source_id: 5 + EXPRESSION + FUNCTION id: 14, function_name: nested, function_type: ordinary, result_type: Array(Tuple(a Int64, b Int64)) + ARGUMENTS + LIST id: 15, nodes: 3 + CONSTANT id: 16, constant_value: Array_[\'a\', \'b\'], constant_value_type: Array(String) + COLUMN id: 17, column_name: n.a, result_type: Array(Int64), source_id: 11 + COLUMN id: 18, column_name: n.b, result_type: Array(Int64), source_id: 11 + ORDER BY + LIST id: 19, nodes: 1 + SORT id: 20, sort_direction: ASCENDING, with_fill: 0 + EXPRESSION + FUNCTION id: 21, function_name: tupleElement, function_type: ordinary, result_type: Int64 + ARGUMENTS + LIST id: 22, nodes: 2 + COLUMN id: 23, column_name: __array_join_exp_1, result_type: Tuple(a Int64, b Int64), source_id: 5 + CONSTANT id: 24, constant_value: \'a\', constant_value_type: String +Expression ((Project names + (Before ORDER BY + Projection) [lifted up part])) +Header: n.a Int64 + n.b Int64 + Sorting (Sorting for ORDER BY) + Header: __array_join_exp_1 Tuple(a Int64, b Int64) + tupleElement(__array_join_exp_1, \'a\'_String) Int64 + Expression ((Before ORDER BY + Projection)) + Header: __array_join_exp_1 Tuple(a Int64, b Int64) + tupleElement(__array_join_exp_1, \'a\'_String) Int64 + ArrayJoin (ARRAY JOIN) + Header: __array_join_exp_1 Tuple(a Int64, b Int64) + Expression ((DROP unused columns before ARRAY JOIN + (ARRAY JOIN actions + Change column names to column identifiers))) + Header: __array_join_exp_1 Array(Tuple(a Int64, b Int64)) + ReadFromMergeTree (default.t_nested) + Header: n.a Array(Int64) + n.b Array(Int64) +(1,3,5) +(2,4,6) +QUERY id: 0 + PROJECTION COLUMNS + n Tuple(a Int64, b Int64, c Int64) + PROJECTION + LIST id: 1, nodes: 1 + COLUMN id: 2, column_name: __array_join_exp_1, result_type: Tuple(a Int64, b Int64, c Int64), source_id: 3 + JOIN TREE + ARRAY_JOIN id: 3, is_left: 0 + TABLE EXPRESSION + TABLE id: 4, alias: __table2, table_name: default.t_nested + JOIN EXPRESSIONS + LIST id: 5, nodes: 1 + COLUMN id: 6, alias: __array_join_exp_1, column_name: __array_join_exp_1, result_type: Tuple(a Int64, b Int64, c Int64), source_id: 3 + EXPRESSION + FUNCTION id: 7, function_name: nested, function_type: ordinary, result_type: Array(Tuple(a Int64, b Int64, c Int64)) + ARGUMENTS + LIST id: 8, nodes: 4 + CONSTANT id: 9, constant_value: Array_[\'a\', \'b\', \'c\'], constant_value_type: Array(String) + COLUMN id: 10, column_name: n.a, result_type: Array(Int64), source_id: 4 + COLUMN id: 11, column_name: n.b, result_type: Array(Int64), source_id: 4 + COLUMN id: 12, column_name: n.c, result_type: Array(Int64), source_id: 4 + ORDER BY + LIST id: 13, nodes: 1 + SORT id: 14, sort_direction: ASCENDING, with_fill: 0 + EXPRESSION + FUNCTION id: 15, function_name: tupleElement, function_type: ordinary, result_type: Int64 + ARGUMENTS + LIST id: 16, nodes: 2 + COLUMN id: 17, column_name: __array_join_exp_1, result_type: Tuple(a Int64, b Int64, c Int64), source_id: 3 + CONSTANT id: 18, constant_value: \'a\', constant_value_type: String +Expression (Project names) +Header: n Tuple(a Int64, b Int64, c Int64) + Sorting (Sorting for ORDER BY) + Header: tupleElement(__array_join_exp_1, \'a\'_String) Int64 + __array_join_exp_1 Tuple(a Int64, b Int64, c Int64) + Expression ((Before ORDER BY + Projection)) + Header: tupleElement(__array_join_exp_1, \'a\'_String) Int64 + __array_join_exp_1 Tuple(a Int64, b Int64, c Int64) + ArrayJoin (ARRAY JOIN) + Header: __array_join_exp_1 Tuple(a Int64, b Int64, c Int64) + Expression ((DROP unused columns before ARRAY JOIN + (ARRAY JOIN actions + Change column names to column identifiers))) + Header: __array_join_exp_1 Array(Tuple(a Int64, b Int64, c Int64)) + ReadFromMergeTree (default.t_nested) + Header: n.a Array(Int64) + n.b Array(Int64) + n.c Array(Int64) +1 +1 +QUERY id: 0 + PROJECTION COLUMNS + 1 UInt8 + PROJECTION + LIST id: 1, nodes: 1 + CONSTANT id: 2, constant_value: UInt64_1, constant_value_type: UInt8 + JOIN TREE + ARRAY_JOIN id: 3, is_left: 0 + TABLE EXPRESSION + TABLE id: 4, alias: __table2, table_name: default.t_nested + JOIN EXPRESSIONS + LIST id: 5, nodes: 1 + COLUMN id: 6, alias: __array_join_exp_1, column_name: __array_join_exp_1, result_type: Tuple(a Int64), source_id: 3 + EXPRESSION + FUNCTION id: 7, function_name: nested, function_type: ordinary, result_type: Array(Tuple(a Int64)) + ARGUMENTS + LIST id: 8, nodes: 2 + CONSTANT id: 9, constant_value: Array_[\'a\'], constant_value_type: Array(String) + COLUMN id: 10, column_name: n.a, result_type: Array(Int64), source_id: 4 + WHERE + FUNCTION id: 11, function_name: greater, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 12, nodes: 2 + FUNCTION id: 13, function_name: tupleElement, function_type: ordinary, result_type: Int64 + ARGUMENTS + LIST id: 14, nodes: 2 + COLUMN id: 15, column_name: __array_join_exp_1, result_type: Tuple(a Int64), source_id: 3 + CONSTANT id: 16, constant_value: \'a\', constant_value_type: String + CONSTANT id: 17, constant_value: UInt64_0, constant_value_type: UInt8 +Expression ((Project names + Projection)) +Header: 1 UInt8 + Filter (WHERE) + Header: + ArrayJoin (ARRAY JOIN) + Header: __array_join_exp_1 Tuple(a Int64) + Expression ((DROP unused columns before ARRAY JOIN + (ARRAY JOIN actions + Change column names to column identifiers))) + Header: __array_join_exp_1 Array(Tuple(a Int64)) + ReadFromMergeTree (default.t_nested) + Header: n.a Array(Int64) +1 +2 +Expression ((Project names + (Before ORDER BY + Projection) [lifted up part])) +Header: tupleElement(n, 1) Int64 + Sorting (Sorting for ORDER BY) + Header: __array_join_exp_1 Tuple(a Int64) + tupleElement(__array_join_exp_1, \'a\'_String) Int64 + Expression ((Before ORDER BY + Projection)) + Header: __array_join_exp_1 Tuple(a Int64) + tupleElement(__array_join_exp_1, \'a\'_String) Int64 + ArrayJoin (ARRAY JOIN) + Header: __array_join_exp_1 Tuple(a Int64) + Expression ((DROP unused columns before ARRAY JOIN + (ARRAY JOIN actions + Change column names to column identifiers))) + Header: __array_join_exp_1 Array(Tuple(a Int64)) + ReadFromMergeTree (default.t_nested) + Header: n.a Array(Int64) +3 +4 +QUERY id: 0 + PROJECTION COLUMNS + b Int64 + PROJECTION + LIST id: 1, nodes: 1 + COLUMN id: 2, column_name: __array_join_exp_2, result_type: Int64, source_id: 3 + JOIN TREE + ARRAY_JOIN id: 3, is_left: 0 + TABLE EXPRESSION + TABLE id: 4, alias: __table2, table_name: default.t_two_arrays + JOIN EXPRESSIONS + LIST id: 5, nodes: 1 + COLUMN id: 6, alias: __array_join_exp_2, column_name: __array_join_exp_2, result_type: Int64, source_id: 3 + EXPRESSION + COLUMN id: 7, column_name: b, result_type: Array(Int64), source_id: 4 + ORDER BY + LIST id: 8, nodes: 1 + SORT id: 9, sort_direction: ASCENDING, with_fill: 0 + EXPRESSION + COLUMN id: 10, column_name: __array_join_exp_2, result_type: Int64, source_id: 3 +Expression (Project names) +Header: b Int64 + Sorting (Sorting for ORDER BY) + Header: __array_join_exp_2 Int64 + Expression ((Before ORDER BY + Projection)) + Header: __array_join_exp_2 Int64 + ArrayJoin (ARRAY JOIN) + Header: __array_join_exp_2 Int64 + Expression ((DROP unused columns before ARRAY JOIN + (ARRAY JOIN actions + Change column names to column identifiers))) + Header: __array_join_exp_2 Array(Int64) + ReadFromMergeTree (default.t_two_arrays) + Header: b Array(Int64) diff --git a/tests/queries/0_stateless/04039_prune_array_join_columns.sql b/tests/queries/0_stateless/04039_prune_array_join_columns.sql new file mode 100644 index 000000000000..399479d6f41b --- /dev/null +++ b/tests/queries/0_stateless/04039_prune_array_join_columns.sql @@ -0,0 +1,52 @@ +SET enable_analyzer = 1; +SET enable_parallel_replicas = 0; + +DROP TABLE IF EXISTS t_nested; +CREATE TABLE t_nested (`n.a` Array(Int64), `n.b` Array(Int64), `n.c` Array(Int64)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_nested VALUES ([1, 2], [3, 4], [5, 6]); + +-- Only n.a is used — n.b and n.c should not be read. +SELECT n.a FROM t_nested ARRAY JOIN n ORDER BY n.a; + +-- Verify nested() is pruned to only a. +EXPLAIN QUERY TREE SELECT n.a FROM t_nested ARRAY JOIN n ORDER BY n.a; +EXPLAIN header = 1 SELECT n.a FROM t_nested ARRAY JOIN n ORDER BY n.a; + +-- Both n.a and n.b used — n.c should not be read. +SELECT n.a, n.b FROM t_nested ARRAY JOIN n ORDER BY n.a; + +EXPLAIN QUERY TREE SELECT n.a, n.b FROM t_nested ARRAY JOIN n ORDER BY n.a; +EXPLAIN header = 1 SELECT n.a, n.b FROM t_nested ARRAY JOIN n ORDER BY n.a; + +-- Direct reference to n — all subcolumns needed. +SELECT n FROM t_nested ARRAY JOIN n ORDER BY n.a; + +EXPLAIN QUERY TREE SELECT n FROM t_nested ARRAY JOIN n ORDER BY n.a; +EXPLAIN header = 1 SELECT n FROM t_nested ARRAY JOIN n ORDER BY n.a; + +-- n used only in WHERE — should still be pruned to only n.a. +SELECT 1 FROM t_nested ARRAY JOIN n WHERE n.a > 0; + +EXPLAIN QUERY TREE SELECT 1 FROM t_nested ARRAY JOIN n WHERE n.a > 0; +EXPLAIN header = 1 SELECT 1 FROM t_nested ARRAY JOIN n WHERE n.a > 0; + +-- Numeric tupleElement index — should prune the same as string access. +SELECT tupleElement(n, 1) FROM t_nested ARRAY JOIN n ORDER BY n.a; +EXPLAIN header = 1 SELECT tupleElement(n, 1) FROM t_nested ARRAY JOIN n ORDER BY n.a; + +DROP TABLE t_nested; + +-- General case: ARRAY JOIN with two independent arrays, only one used. +DROP TABLE IF EXISTS t_two_arrays; +CREATE TABLE t_two_arrays (a Array(Int64), b Array(Int64)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_two_arrays VALUES ([1, 2], [3, 4]); + +SELECT b FROM t_two_arrays ARRAY JOIN a, b ORDER BY b; + +-- Verify: column a should be pruned from ARRAY JOIN, only b remains. +EXPLAIN QUERY TREE SELECT b FROM t_two_arrays ARRAY JOIN a, b ORDER BY b; + +-- Verify with EXPLAIN header=1 that only b is read from storage. +EXPLAIN header = 1 SELECT b FROM t_two_arrays ARRAY JOIN a, b ORDER BY b; + +DROP TABLE t_two_arrays; diff --git a/tests/queries/0_stateless/04041_variant_read_with_direct_io.reference b/tests/queries/0_stateless/04041_variant_read_with_direct_io.reference new file mode 100644 index 000000000000..3cd40e317c88 --- /dev/null +++ b/tests/queries/0_stateless/04041_variant_read_with_direct_io.reference @@ -0,0 +1,3 @@ +500000 +100000 +100000 diff --git a/tests/queries/0_stateless/04041_variant_read_with_direct_io.sh b/tests/queries/0_stateless/04041_variant_read_with_direct_io.sh new file mode 100755 index 000000000000..d50fa6d943b2 --- /dev/null +++ b/tests/queries/0_stateless/04041_variant_read_with_direct_io.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Tags: long + +# Regression test for incorrect seek in AsynchronousReadBufferFromFileDescriptor +# with O_DIRECT (min_bytes_to_use_direct_io=1). The bug was that getPosition() +# and seek NOOP/in-buffer checks did not account for bytes_to_ignore set by +# O_DIRECT alignment, causing corrupted reads of Variant subcolumns. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +CH_CLIENT="$CLICKHOUSE_CLIENT --allow_suspicious_variant_types=1 --max_threads 2 --min_bytes_to_use_direct_io 1" + +$CH_CLIENT -q "drop table if exists test_variant_direct_io;" + +$CH_CLIENT -q "create table test_variant_direct_io (id UInt64, v Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, index_granularity_bytes=10485760, index_granularity=8192;" + +$CH_CLIENT -mq "insert into test_variant_direct_io select number, NULL from numbers(100000); +insert into test_variant_direct_io select number + 100000, number from numbers(100000); +insert into test_variant_direct_io select number + 200000, ('str_' || toString(number))::Variant(String) from numbers(100000); +insert into test_variant_direct_io select number + 300000, ('lc_str_' || toString(number))::LowCardinality(String) from numbers(100000); +insert into test_variant_direct_io select number + 400000, tuple(number, number + 1)::Tuple(a UInt32, b UInt32) from numbers(100000); +insert into test_variant_direct_io select number + 500000, range(number % 20 + 1)::Array(UInt64) from numbers(100000);" + +$CH_CLIENT -q "optimize table test_variant_direct_io final settings mutations_sync=1;" + +# Without the fix, reading v.String here would fail with: +# "Size of deserialized variant column less than the limit" +$CH_CLIENT -q "select v.String from test_variant_direct_io format Null;" + +# Also check that subcolumn reads return the correct count +$CH_CLIENT -q "select count() from test_variant_direct_io where v is not null;" +$CH_CLIENT -q "select count() from test_variant_direct_io where v.String is not null;" +$CH_CLIENT -q "select count() from test_variant_direct_io where v.UInt64 is not null;" + +$CH_CLIENT -q "drop table test_variant_direct_io;" diff --git a/tests/queries/0_stateless/04042_hash_join_allocated_size_tracking.reference b/tests/queries/0_stateless/04042_hash_join_allocated_size_tracking.reference new file mode 100644 index 000000000000..86d9c30c0b0a --- /dev/null +++ b/tests/queries/0_stateless/04042_hash_join_allocated_size_tracking.reference @@ -0,0 +1 @@ +200000 2147483647 diff --git a/tests/queries/0_stateless/04042_hash_join_allocated_size_tracking.sql b/tests/queries/0_stateless/04042_hash_join_allocated_size_tracking.sql new file mode 100644 index 000000000000..54a787aa2ec3 --- /dev/null +++ b/tests/queries/0_stateless/04042_hash_join_allocated_size_tracking.sql @@ -0,0 +1,23 @@ +SET enable_analyzer = 1; +SET use_variant_as_common_type = 1; + +CREATE TABLE test__fuzz_1 (`id` Nullable(UInt64), `d` Dynamic(max_types = 133)) ENGINE = Memory; + +INSERT INTO test__fuzz_1 SETTINGS min_insert_block_size_rows = 50000 SELECT number, number FROM numbers(100000) SETTINGS min_insert_block_size_rows = 50000; + +INSERT INTO test__fuzz_1 SETTINGS min_insert_block_size_rows = 50000 SELECT number, concat('str_', toString(number)) FROM numbers(100000, 100000) SETTINGS min_insert_block_size_rows = 50000; + +INSERT INTO test__fuzz_1 SETTINGS min_insert_block_size_rows = 50000 SELECT number, arrayMap(x -> multiIf((number % 9) = 0, NULL, (number % 9) = 3, concat('str_', toString(number)), number), range((number % 10) + 1)) FROM numbers(200000, 100000) SETTINGS min_insert_block_size_rows = 50000; + +INSERT INTO test__fuzz_1 SETTINGS min_insert_block_size_rows = 50000 SELECT number, NULL FROM numbers(300000, 100000) SETTINGS min_insert_block_size_rows = 50000; + +INSERT INTO test__fuzz_1 SETTINGS min_insert_block_size_rows = 50000 SELECT number, multiIf((number % 4) = 3, concat('str_', toString(number)), (number % 4) = 2, NULL, (number % 4) = 1, number, arrayMap(x -> multiIf((number % 9) = 0, NULL, (number % 9) = 3, concat('str_', toString(number)), number), range((number % 10) + 1))) FROM numbers(400000, 400000) SETTINGS min_insert_block_size_rows = 50000; + +INSERT INTO test__fuzz_1 SETTINGS min_insert_block_size_rows = 50000 SELECT number, if((number % 5) = 1, CAST([range(CAST((number % 10) + 1, 'UInt64'))], 'Array(Array(Dynamic))'), number) FROM numbers(100000, 100000) SETTINGS min_insert_block_size_rows = 50000; + +INSERT INTO test__fuzz_1 SETTINGS min_insert_block_size_rows = 50000 SELECT number, if((number % 5) = 1, CAST(CAST(concat('str_', number), 'LowCardinality(String)'), 'Dynamic'), CAST(number, 'Dynamic')) FROM numbers(100000, 100000) SETTINGS min_insert_block_size_rows = 50000; + +SELECT count(equals(toInt256(257), intDiv(65536, -2147483649) AS alias144)), toInt128(2147483647) FROM test__fuzz_1 WHERE NOT empty((SELECT d.`Array(Variant(String, UInt64))`)); + +DROP TABLE test__fuzz_1; + diff --git a/tests/queries/0_stateless/04043_system_asynchronous_inserts_user_filter.reference b/tests/queries/0_stateless/04043_system_asynchronous_inserts_user_filter.reference new file mode 100644 index 000000000000..e9130c6855ee --- /dev/null +++ b/tests/queries/0_stateless/04043_system_asynchronous_inserts_user_filter.reference @@ -0,0 +1,6 @@ +restricted_user sees: +0 +secret_user sees: +1 +admin sees: +1 diff --git a/tests/queries/0_stateless/04043_system_asynchronous_inserts_user_filter.sh b/tests/queries/0_stateless/04043_system_asynchronous_inserts_user_filter.sh new file mode 100755 index 000000000000..9acc6585241c --- /dev/null +++ b/tests/queries/0_stateless/04043_system_asynchronous_inserts_user_filter.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# Regression test: system.asynchronous_inserts must not leak cross-user insert metadata. +# A user without SHOW_USERS privilege must only see their own pending inserts. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q " + DROP USER IF EXISTS secret_user_${CLICKHOUSE_DATABASE}; + DROP USER IF EXISTS restricted_user_${CLICKHOUSE_DATABASE}; + CREATE USER secret_user_${CLICKHOUSE_DATABASE}; + CREATE USER restricted_user_${CLICKHOUSE_DATABASE}; + DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.async_insert_test; + CREATE TABLE ${CLICKHOUSE_DATABASE}.async_insert_test (x UInt64) ENGINE=MergeTree ORDER BY x; + GRANT INSERT ON ${CLICKHOUSE_DATABASE}.async_insert_test TO secret_user_${CLICKHOUSE_DATABASE}; + GRANT SELECT ON system.asynchronous_inserts TO secret_user_${CLICKHOUSE_DATABASE}; + GRANT SELECT ON system.asynchronous_inserts TO restricted_user_${CLICKHOUSE_DATABASE}; +" + +# secret_user inserts with async_insert enabled and a very long flush timeout so the entry stays in the queue. +${CLICKHOUSE_CLIENT} \ + --user "secret_user_${CLICKHOUSE_DATABASE}" \ + --async_insert 1 \ + --async_insert_busy_timeout_max_ms 600000 \ + --async_insert_busy_timeout_min_ms 600000 \ + --wait_for_async_insert 0 \ + -q "INSERT INTO ${CLICKHOUSE_DATABASE}.async_insert_test VALUES (42)" + +# restricted_user must see 0 rows (no cross-user visibility). +echo "restricted_user sees:" +${CLICKHOUSE_CLIENT} \ + --user "restricted_user_${CLICKHOUSE_DATABASE}" \ + -q "SELECT count() FROM system.asynchronous_inserts WHERE table = 'async_insert_test' AND database = '${CLICKHOUSE_DATABASE}'" + +# secret_user must see their own row. +echo "secret_user sees:" +${CLICKHOUSE_CLIENT} \ + --user "secret_user_${CLICKHOUSE_DATABASE}" \ + -q "SELECT count() FROM system.asynchronous_inserts WHERE table = 'async_insert_test' AND database = '${CLICKHOUSE_DATABASE}'" + +# Admin (current session) must see all rows. +echo "admin sees:" +${CLICKHOUSE_CLIENT} \ + -q "SELECT count() FROM system.asynchronous_inserts WHERE table = 'async_insert_test' AND database = '${CLICKHOUSE_DATABASE}'" + +${CLICKHOUSE_CLIENT} -q " + DROP USER IF EXISTS secret_user_${CLICKHOUSE_DATABASE}; + DROP USER IF EXISTS restricted_user_${CLICKHOUSE_DATABASE}; + DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.async_insert_test; +" diff --git a/tests/queries/0_stateless/04049_aggregate_function_numeric_indexed_vector_self_merge.reference b/tests/queries/0_stateless/04049_aggregate_function_numeric_indexed_vector_self_merge.reference new file mode 100644 index 000000000000..156baf3abc90 --- /dev/null +++ b/tests/queries/0_stateless/04049_aggregate_function_numeric_indexed_vector_self_merge.reference @@ -0,0 +1,2 @@ +{100:2} +{100:4} diff --git a/tests/queries/0_stateless/04049_aggregate_function_numeric_indexed_vector_self_merge.sql b/tests/queries/0_stateless/04049_aggregate_function_numeric_indexed_vector_self_merge.sql new file mode 100644 index 000000000000..15eb95fe70bd --- /dev/null +++ b/tests/queries/0_stateless/04049_aggregate_function_numeric_indexed_vector_self_merge.sql @@ -0,0 +1,11 @@ +-- Test that self-merge of NumericIndexedVector aggregate states does not trigger +-- assertion failure in CRoaring (x1 != x2 in `roaring_bitmap_xor_inplace`). +-- https://github.com/ClickHouse/ClickHouse/issues/99704 + +-- `multiply` triggers self-merge via exponentiation by squaring (even branch). +SELECT arrayJoin([numericIndexedVectorToMap( + multiply(2, groupNumericIndexedVectorState(100, 1)))]); + +-- Power of 2 forces multiple self-merge iterations. +SELECT arrayJoin([numericIndexedVectorToMap( + multiply(4, groupNumericIndexedVectorState(100, 1)))]); diff --git a/tests/queries/0_stateless/04051_variant_filter_shared_columns_bug.reference b/tests/queries/0_stateless/04051_variant_filter_shared_columns_bug.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/04051_variant_filter_shared_columns_bug.sql b/tests/queries/0_stateless/04051_variant_filter_shared_columns_bug.sql new file mode 100644 index 000000000000..9afcef92c472 --- /dev/null +++ b/tests/queries/0_stateless/04051_variant_filter_shared_columns_bug.sql @@ -0,0 +1,20 @@ +-- Regression test for ColumnVariant::filter sharing variant column pointers +-- in the hasOnlyNulls() optimization path. The filter was copying ColumnPtr +-- shared pointers instead of cloning, causing two ColumnVariant objects to +-- share the same variant columns. When one was mutated via insertRangeFrom, +-- the other became inconsistent (discriminators_size=0 but variant_sizes>0), +-- leading to a LOGICAL_ERROR in compress(). + +SET cross_join_min_rows_to_compress = 1; +SET enable_analyzer = 1; +SET use_variant_as_common_type = 1; + +DROP TABLE IF EXISTS test_variant_filter; +CREATE TABLE test_variant_filter (`id` UInt64, `d` Dynamic) ENGINE = Memory; + +INSERT INTO test_variant_filter SELECT number, NULL FROM numbers(50000); +INSERT INTO test_variant_filter SELECT number, [number, 'str']::Array(Variant(String, UInt64)) FROM numbers(50000); + +SELECT 1 FROM test_variant_filter WHERE NOT empty((SELECT d.`Array(Variant(String, UInt64))`)) FORMAT Null; + +DROP TABLE test_variant_filter; diff --git a/tests/queries/0_stateless/04054_backup_restore_validate_entry_paths.reference b/tests/queries/0_stateless/04054_backup_restore_validate_entry_paths.reference new file mode 100644 index 000000000000..9e647571eac7 --- /dev/null +++ b/tests/queries/0_stateless/04054_backup_restore_validate_entry_paths.reference @@ -0,0 +1,2 @@ +OK: path traversal was blocked +1 hello diff --git a/tests/queries/0_stateless/04054_backup_restore_validate_entry_paths.sh b/tests/queries/0_stateless/04054_backup_restore_validate_entry_paths.sh new file mode 100755 index 000000000000..b1aa605bc9f3 --- /dev/null +++ b/tests/queries/0_stateless/04054_backup_restore_validate_entry_paths.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Test that RESTORE rejects backup entries with path traversal sequences (../) + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS tbl_backup_traversal" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE tbl_backup_traversal (id UInt64, data String) ENGINE = MergeTree ORDER BY id" +${CLICKHOUSE_CLIENT} --query "INSERT INTO tbl_backup_traversal VALUES (1, 'hello')" + +backups_disk_root=$(${CLICKHOUSE_CLIENT} --query "SELECT path FROM system.disks WHERE name='backups'" 2>/dev/null) + +if [ -z "${backups_disk_root}" ]; then + echo "backups disk is not configured, skipping test" + exit 0 +fi + +extra_content="EXTRA_FILE_CONTENT_HERE" +extra_size=${#extra_content} +extra_checksum=$(echo -n "${extra_content}" | md5sum | awk '{print $1}') +extra_data_path="data/default/tbl_backup_traversal/extra_payload.bin" + +# Creates a backup, injects an extra file entry into its .backup metadata, and +# attempts to restore. Expects the specified error. +# $1 - backup suffix +# $2 - injected value +# $3 - expected error code (e.g. INSECURE_PATH, BACKUP_DAMAGED) +# $4 - (optional) injected value; defaults to extra_data_path +inject_and_restore() { + local suffix="$1" + local injected_name="$2" + local expected_error="$3" + local injected_data_file="${4:-${extra_data_path}}" + local bname="${CLICKHOUSE_TEST_UNIQUE_NAME}_${suffix}" + + ${CLICKHOUSE_CLIENT} --query "BACKUP TABLE tbl_backup_traversal TO Disk('backups', '${bname}')" > /dev/null 2>&1 + + local bpath="${backups_disk_root}/${bname}" + mkdir -p "${bpath}/$(dirname "${extra_data_path}")" + echo -n "${extra_content}" > "${bpath}/${extra_data_path}" + + sed -i "s||${injected_name}${extra_size}${extra_checksum}${injected_data_file}|" "${bpath}/.backup" + + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS tbl_backup_traversal" + ${CLICKHOUSE_CLIENT} -m -q "RESTORE TABLE tbl_backup_traversal FROM Disk('backups', '${bname}'); -- { serverError ${expected_error} }" +} + +# Helper to recreate the table between tests. +recreate_table() { + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE IF NOT EXISTS tbl_backup_traversal (id UInt64, data String) ENGINE = MergeTree ORDER BY id" + ${CLICKHOUSE_CLIENT} --query "INSERT INTO tbl_backup_traversal VALUES (1, 'hello')" +} + +# Test 1: relative path traversal in . +inject_and_restore "rel" "data/default/tbl_backup_traversal/all_0_0_0/../../../../../../../tmp/backup_traversal_test_output.txt" INSECURE_PATH + +# Verify the file was NOT written outside the backup directory. +if [ -f "/tmp/backup_traversal_test_output.txt" ]; then + echo "FAIL: file written to /tmp/" + rm -f "/tmp/backup_traversal_test_output.txt" +else + echo "OK: path traversal was blocked" +fi + +# Test 2: absolute path in . +recreate_table +inject_and_restore "abs" "/tmp/backup_absolute_path_test_output.xml" INSECURE_PATH + +# Test 3: path traversal in (source path for reading from the backup). +recreate_table +inject_and_restore "datafile" "data/default/tbl_backup_traversal/extra_payload.bin" INSECURE_PATH "data/default/tbl_backup_traversal/all_0_0_0/../../../../../../../etc/passwd" + +# Test 4: empty should be rejected as damaged. +recreate_table +inject_and_restore "empty" "" BACKUP_DAMAGED + +# Test 5: "." as should be rejected as damaged. +recreate_table +inject_and_restore "dot" "." BACKUP_DAMAGED + +# Test 6: bare ".." as . +recreate_table +inject_and_restore "dotdot" ".." INSECURE_PATH + +# Test 7: absolute path in . +recreate_table +inject_and_restore "abs_datafile" "data/default/tbl_backup_traversal/extra_payload.bin" INSECURE_PATH "/etc/passwd" + +# Test 8: normal backup/restore still works after the validation was added. +recreate_table +normal_backup="${CLICKHOUSE_TEST_UNIQUE_NAME}_normal" +${CLICKHOUSE_CLIENT} --query "BACKUP TABLE tbl_backup_traversal TO Disk('backups', '${normal_backup}')" > /dev/null 2>&1 +${CLICKHOUSE_CLIENT} --query "DROP TABLE tbl_backup_traversal" +${CLICKHOUSE_CLIENT} --query "RESTORE TABLE tbl_backup_traversal FROM Disk('backups', '${normal_backup}')" > /dev/null 2>&1 +${CLICKHOUSE_CLIENT} --query "SELECT * FROM tbl_backup_traversal" + +# Clean up. +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS tbl_backup_traversal" +rm -rf "${backups_disk_root:?}/${CLICKHOUSE_TEST_UNIQUE_NAME}"_* 2>/dev/null || true diff --git a/tests/queries/0_stateless/04054_json_nested_shared_data_buckets_missing_stream_bug.reference b/tests/queries/0_stateless/04054_json_nested_shared_data_buckets_missing_stream_bug.reference new file mode 100644 index 000000000000..2fdc1deb963d --- /dev/null +++ b/tests/queries/0_stateless/04054_json_nested_shared_data_buckets_missing_stream_bug.reference @@ -0,0 +1,3 @@ +1 {"images":[{"url":"c","width":300}]} +2 {"images":[{"url":"d","width":400}]} +3 {"images":[{"url":"a","width":100}]} diff --git a/tests/queries/0_stateless/04054_json_nested_shared_data_buckets_missing_stream_bug.sql b/tests/queries/0_stateless/04054_json_nested_shared_data_buckets_missing_stream_bug.sql new file mode 100644 index 000000000000..9373a9fbaa57 --- /dev/null +++ b/tests/queries/0_stateless/04054_json_nested_shared_data_buckets_missing_stream_bug.sql @@ -0,0 +1,42 @@ +-- Tags: long + +SET allow_experimental_json_type = 1; + +-- Regression test for a bug where ColumnObject::index (and filter/replicate/scatter) +-- did not propagate statistics, causing a mismatch between the number of shared data +-- buckets chosen during stream creation vs serialization state creation for nested JSON +-- columns inside Array(JSON). This resulted in: +-- "Stream ... object_shared_data.1.size1 not found" (LOGICAL_ERROR) +-- +-- The bug requires: Wide parts, nested JSON with empty shared data, a non-trivial +-- permutation applied during INSERT, and optimize_on_insert=0 to prevent mergeBlock +-- from pre-sorting the block and nullifying the permutation. + +DROP TABLE IF EXISTS src; +DROP TABLE IF EXISTS dst; + +CREATE TABLE src (id UInt64, data JSON(max_dynamic_paths=256)) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part=0; + +INSERT INTO src VALUES + (3, '{"images": [{"url": "a", "width": 100}]}'), + (2, '{"images": [{"url": "d", "width": 400}]}'), + (1, '{"images": [{"url": "c", "width": 300}]}'); + +CREATE TABLE dst (id UInt64, data JSON(max_dynamic_paths=256)) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part=0; + +-- Data arrives at the MergeTree sink with statistics intact from reading the source part. +-- The sink applies a permutation to sort by id. With optimize_on_insert=0, the raw +-- permutation is passed to the writer. The inner JSON (inside Array) goes through +-- ColumnArray::permute -> ColumnArray::indexImpl -> ColumnObject::index. +-- Before the fix, ColumnObject::index dropped statistics, causing a bucket count mismatch. +INSERT INTO dst SELECT * FROM src +SETTINGS max_insert_threads=1, optimize_on_insert=0; + +SELECT id, data FROM dst ORDER BY id; + +DROP TABLE src; +DROP TABLE dst; diff --git a/tests/queries/0_stateless/04057_ulid_non_ascii_input.reference b/tests/queries/0_stateless/04057_ulid_non_ascii_input.reference new file mode 100644 index 000000000000..6358d6525e96 --- /dev/null +++ b/tests/queries/0_stateless/04057_ulid_non_ascii_input.reference @@ -0,0 +1 @@ +2023-03-28 01:16:44.000 diff --git a/tests/queries/0_stateless/04057_ulid_non_ascii_input.sql b/tests/queries/0_stateless/04057_ulid_non_ascii_input.sql new file mode 100644 index 000000000000..fe46f82e03e9 --- /dev/null +++ b/tests/queries/0_stateless/04057_ulid_non_ascii_input.sql @@ -0,0 +1,9 @@ +-- Tags: no-fasttest +-- Test that ULIDStringToDateTime properly rejects non-ASCII input without buffer overflow + +SELECT ULIDStringToDateTime(unhex(repeat('ff', 26))); -- { serverError BAD_ARGUMENTS } +SELECT ULIDStringToDateTime(unhex(repeat('80', 26))); -- { serverError BAD_ARGUMENTS } +SELECT ULIDStringToDateTime(unhex(repeat('fe', 26))); -- { serverError BAD_ARGUMENTS } + +-- Valid ULID should still work +SELECT ULIDStringToDateTime('01GWJWKW30MFPQJRYEAF4XFZ9E', 'UTC'); diff --git a/tests/queries/0_stateless/04059_has_lowcardinality_tuple_key_crash.reference b/tests/queries/0_stateless/04059_has_lowcardinality_tuple_key_crash.reference new file mode 100644 index 000000000000..bb5ee5c21ebe --- /dev/null +++ b/tests/queries/0_stateless/04059_has_lowcardinality_tuple_key_crash.reference @@ -0,0 +1,3 @@ +0 +0 +1 diff --git a/tests/queries/0_stateless/04059_has_lowcardinality_tuple_key_crash.sql b/tests/queries/0_stateless/04059_has_lowcardinality_tuple_key_crash.sql new file mode 100644 index 000000000000..353014c43496 --- /dev/null +++ b/tests/queries/0_stateless/04059_has_lowcardinality_tuple_key_crash.sql @@ -0,0 +1,38 @@ +-- Regression test for crash: "ColumnUnique can't contain null values" +-- when has() is used with PREWHERE on a Tuple key containing LowCardinality elements. +-- The crash occurred because tryPrepareSetColumnsForIndex passed the LowCardinality-wrapped +-- key type to castColumnAccurateOrNull, which produced null values for out-of-range casts +-- that were then inserted into a non-nullable LowCardinality dictionary. + +SET allow_suspicious_low_cardinality_types = 1; + +DROP TABLE IF EXISTS test_has_lc_tuple_crash; + +CREATE TABLE test_has_lc_tuple_crash +( + id UInt64, + key_tuple Tuple(LowCardinality(UInt32), UInt32), + payload UInt64 +) +ENGINE = MergeTree +ORDER BY key_tuple +SETTINGS index_granularity = 1000, allow_nullable_key = 1; + +INSERT INTO test_has_lc_tuple_crash SELECT number, (number, number % 10), number FROM numbers(1000); + +-- This query should not crash. The Int64 values (-2147483649, 9223372036854775806) +-- cannot be safely cast to (LowCardinality(UInt32), UInt32), so accurateOrNull produces nulls. +-- Previously, these nulls were inserted into the non-nullable LowCardinality dictionary, causing a crash. +SELECT count() FROM test_has_lc_tuple_crash + PREWHERE has((SELECT DISTINCT [(-2147483649, 9223372036854775806)]), key_tuple) + WHERE has((SELECT DISTINCT [(1, 10)]), key_tuple); + +-- Simpler variant: just PREWHERE with out-of-range values +SELECT count() FROM test_has_lc_tuple_crash + PREWHERE has([(-1, 0)], key_tuple); + +-- Verify correct results still work: (10, 0) matches row where number=10 (key_tuple=(10, 10%10)=(10,0)) +SELECT count() FROM test_has_lc_tuple_crash + WHERE has([(10, 0)], key_tuple); + +DROP TABLE test_has_lc_tuple_crash; diff --git a/tests/queries/0_stateless/04063_parse_datetime_best_effort_many_fractional_digits.reference b/tests/queries/0_stateless/04063_parse_datetime_best_effort_many_fractional_digits.reference new file mode 100644 index 000000000000..afadfedc5ba0 --- /dev/null +++ b/tests/queries/0_stateless/04063_parse_datetime_best_effort_many_fractional_digits.reference @@ -0,0 +1,9 @@ +2020-08-06 22:29:00.123456 +1973-03-03 09:46:40.123456 +2020-08-07 01:29:00.123456 +2020-08-06 22:29:00.999999 +1973-03-03 09:46:40.999999 +2020-08-07 01:29:00.999999 +\N +\N +\N diff --git a/tests/queries/0_stateless/04063_parse_datetime_best_effort_many_fractional_digits.sql b/tests/queries/0_stateless/04063_parse_datetime_best_effort_many_fractional_digits.sql new file mode 100644 index 000000000000..882256ae38d4 --- /dev/null +++ b/tests/queries/0_stateless/04063_parse_datetime_best_effort_many_fractional_digits.sql @@ -0,0 +1,23 @@ +-- Regression test: parsing datetime strings with many fractional digits must not cause +-- signed integer overflow (UB) in readDecimalNumber. Fractional digits are capped at +-- digits10 of the result type (Int64::digits10 = 18). + +-- 18 fractional digits (at the cap) - parses without truncation +SELECT parseDateTime64BestEffort('1596752940.123456789012345678', 6, 'UTC'); +SELECT parseDateTime64BestEffort('100000000.123456789012345678', 6, 'UTC'); +SELECT parseDateTime64BestEffort('2020-08-07 01:29:00.123456789012345678', 6, 'UTC'); + +-- 19 fractional digits with a value that overflows Int64 without the cap: +-- readDecimalNumber processes chunks 4+4+4+4+3; at the last step +-- 9999999999999999 * 1000 overflows Int64, previously causing UB. +-- With the fix the digit count is capped to 18 (chunk 4+4+4+4+2) and the +-- 19th digit is silently dropped. +SELECT parseDateTime64BestEffort('1596752940.9999999999999999999', 6, 'UTC'); +SELECT parseDateTime64BestEffort('100000000.9999999999999999999', 6, 'UTC'); +SELECT parseDateTime64BestEffort('2020-08-07 01:29:00.9999999999999999999', 6, 'UTC'); + +-- 20+ fractional digits: the 20th digit is left in the stream after readDigits +-- exhausts its buffer (UInt64::digits10 = 19), causing a parse error +SELECT parseDateTime64BestEffortOrNull('1596752940.12345678901234567890', 6, 'UTC'); +SELECT parseDateTime64BestEffortOrNull('100000000.12345678901234567890', 6, 'UTC'); +SELECT parseDateTime64BestEffortOrNull('2020-08-07 01:29:00.12345678901234567890', 6, 'UTC'); diff --git a/tests/queries/0_stateless/04076_json_in_min_max_index_bug.reference b/tests/queries/0_stateless/04076_json_in_min_max_index_bug.reference new file mode 100644 index 000000000000..702c2663b6d3 --- /dev/null +++ b/tests/queries/0_stateless/04076_json_in_min_max_index_bug.reference @@ -0,0 +1,51 @@ +3 +Expression (Project names) + Sorting (Sorting for ORDER BY) + Expression ((Before ORDER BY + Projection)) + Expression ((WHERE + Change column names to column identifiers)) + ReadFromMergeTree (default.t_json_minmax_idx) + Indexes: + PrimaryKey + Condition: true + Parts: 1/1 + Granules: 3/3 + Skip + Name: idx_j + Description: minmax GRANULARITY 1 + Parts: 1/1 + Granules: 1/3 + Ranges: 1 +2 +Expression (Project names) + Sorting (Sorting for ORDER BY) + Expression ((Before ORDER BY + Projection)) + Expression ((WHERE + Change column names to column identifiers)) + ReadFromMergeTree (default.t_json_minmax_idx) + Indexes: + PrimaryKey + Condition: true + Parts: 1/1 + Granules: 3/3 + Skip + Name: idx_j + Description: minmax GRANULARITY 1 + Parts: 1/1 + Granules: 1/3 + Ranges: 1 +1 +Expression (Project names) + Sorting (Sorting for ORDER BY) + Expression ((Before ORDER BY + Projection)) + Expression ((WHERE + Change column names to column identifiers)) + ReadFromMergeTree (default.t_json_minmax_idx) + Indexes: + PrimaryKey + Condition: true + Parts: 1/1 + Granules: 3/3 + Skip + Name: idx_j + Description: minmax GRANULARITY 1 + Parts: 1/1 + Granules: 1/3 + Ranges: 1 diff --git a/tests/queries/0_stateless/04076_json_in_min_max_index_bug.sql b/tests/queries/0_stateless/04076_json_in_min_max_index_bug.sql new file mode 100644 index 000000000000..cb754f3faa1e --- /dev/null +++ b/tests/queries/0_stateless/04076_json_in_min_max_index_bug.sql @@ -0,0 +1,21 @@ +-- Tags: no-parallel-replicas +-- Tag no-parallel-replicas: output of explain is different + +DROP TABLE IF EXISTS t_json_minmax_idx; + +CREATE TABLE t_json_minmax_idx (id UInt32, j JSON, INDEX idx_j j TYPE minmax GRANULARITY 1) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=1; + +INSERT INTO t_json_minmax_idx VALUES (1, '{"a":"1"}'), (2, '{"a":"2"}'), (3, '{"a":"3"}'); + +SET enable_analyzer = 1; +SET optimize_move_to_prewhere = 1; +SET query_plan_optimize_prewhere = 1; + +SELECT id FROM t_json_minmax_idx WHERE j > '{"a":"2"}'::JSON ORDER BY id; +EXPLAIN indexes=1 SELECT id FROM t_json_minmax_idx WHERE j > '{"a":"2"}'::JSON ORDER BY id; +SELECT id FROM t_json_minmax_idx WHERE j = '{"a":"2"}'::JSON ORDER BY id; +EXPLAIN indexes=1 SELECT id FROM t_json_minmax_idx WHERE j = '{"a":"2"}'::JSON ORDER BY id; +SELECT id FROM t_json_minmax_idx WHERE j < '{"a":"2"}'::JSON ORDER BY id; +EXPLAIN indexes=1 SELECT id FROM t_json_minmax_idx WHERE j < '{"a":"2"}'::JSON ORDER BY id; + +DROP TABLE IF EXISTS t_json_minmax_idx; diff --git a/tests/queries/0_stateless/04077_formatdatetime_w_is_a_variable_length_formatter.reference b/tests/queries/0_stateless/04077_formatdatetime_w_is_a_variable_length_formatter.reference new file mode 100644 index 000000000000..cc34d1fc736f --- /dev/null +++ b/tests/queries/0_stateless/04077_formatdatetime_w_is_a_variable_length_formatter.reference @@ -0,0 +1,64 @@ +--- Test with formatdatetime_parsedatetime_m_is_month_name: +Monday 06 +Tuesday 07 +Wednesday 08 +Thursday 09 +Friday 10 +Saturday 11 +Sunday 12 +--- +Monday 06 +Tuesday 07 +Wednesday 08 +Thursday 09 +Friday 10 +Saturday 11 +Sunday 12 +--- Test with formatdatetime_f_prints_single_zero: +Monday 06 +Tuesday 07 +Wednesday 08 +Thursday 09 +Friday 10 +Saturday 11 +Sunday 12 +--- +Monday 06 +Tuesday 07 +Wednesday 08 +Thursday 09 +Friday 10 +Saturday 11 +Sunday 12 +--- Test with formatdatetime_f_prints_scale_number_of_digits: +Monday 06 +Tuesday 07 +Wednesday 08 +Thursday 09 +Friday 10 +Saturday 11 +Sunday 12 +--- +Monday 06 +Tuesday 07 +Wednesday 08 +Thursday 09 +Friday 10 +Saturday 11 +Sunday 12 +--- Test with formatdatetime_format_without_leading_zeros: +Monday 06 +Tuesday 07 +Wednesday 08 +Thursday 09 +Friday 10 +Saturday 11 +Sunday 12 +--- +Monday 06 +Tuesday 07 +Wednesday 08 +Thursday 09 +Friday 10 +Saturday 11 +Sunday 12 diff --git a/tests/queries/0_stateless/04077_formatdatetime_w_is_a_variable_length_formatter.sql b/tests/queries/0_stateless/04077_formatdatetime_w_is_a_variable_length_formatter.sql new file mode 100644 index 000000000000..6f3b612563c7 --- /dev/null +++ b/tests/queries/0_stateless/04077_formatdatetime_w_is_a_variable_length_formatter.sql @@ -0,0 +1,30 @@ +-- Formatter %W in function 'formatDateTime' is a variable-length formatter +-- In Bug 101844, this was the case only for some combinations of extra formatting settings + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab (d Date) ENGINE = MergeTree ORDER BY d; + +INSERT INTO tab SELECT toDate('2026-04-06') + number FROM numbers(7); + +SELECT '--- Test with formatdatetime_parsedatetime_m_is_month_name:'; +SELECT formatDateTime(d, '%W %d') FROM tab ORDER BY d SETTINGS formatdatetime_parsedatetime_m_is_month_name = 1; +SELECT '---'; +SELECT formatDateTime(d, '%W %d') FROM tab ORDER BY d SETTINGS formatdatetime_parsedatetime_m_is_month_name = 0; + +SELECT '--- Test with formatdatetime_f_prints_single_zero:'; +SELECT formatDateTime(d, '%W %d') FROM tab ORDER BY d SETTINGS formatdatetime_f_prints_single_zero = 0; +SELECT '---'; +SELECT formatDateTime(d, '%W %d') FROM tab ORDER BY d SETTINGS formatdatetime_f_prints_single_zero = 1; + +SELECT '--- Test with formatdatetime_f_prints_scale_number_of_digits:'; +SELECT formatDateTime(d, '%W %d') FROM tab ORDER BY d SETTINGS formatdatetime_f_prints_scale_number_of_digits = 0; +SELECT '---'; +SELECT formatDateTime(d, '%W %d') FROM tab ORDER BY d SETTINGS formatdatetime_f_prints_scale_number_of_digits = 1; + +SELECT '--- Test with formatdatetime_format_without_leading_zeros:'; +SELECT formatDateTime(d, '%W %d') FROM tab ORDER BY d SETTINGS formatdatetime_format_without_leading_zeros = 0; +SELECT '---'; +SELECT formatDateTime(d, '%W %d') FROM tab ORDER BY d SETTINGS formatdatetime_format_without_leading_zeros = 1; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/04077_wide_part_writer_cancel_on_exception.reference b/tests/queries/0_stateless/04077_wide_part_writer_cancel_on_exception.reference new file mode 100644 index 000000000000..08839f6bb296 --- /dev/null +++ b/tests/queries/0_stateless/04077_wide_part_writer_cancel_on_exception.reference @@ -0,0 +1 @@ +200 diff --git a/tests/queries/0_stateless/04077_wide_part_writer_cancel_on_exception.sql b/tests/queries/0_stateless/04077_wide_part_writer_cancel_on_exception.sql new file mode 100644 index 000000000000..88a001fc33f0 --- /dev/null +++ b/tests/queries/0_stateless/04077_wide_part_writer_cancel_on_exception.sql @@ -0,0 +1,28 @@ +-- Tags: no-parallel, no-random-merge-tree-settings +-- Regression test: MergeTreeDataPartWriterWide::cancel must not SIGSEGV +-- when addStreams fails mid-way leaving no null entries in column_streams. + +DROP TABLE IF EXISTS t_wide_cancel; + +CREATE TABLE t_wide_cancel (a UInt64, b String, c Float64) +ENGINE = MergeTree ORDER BY a +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; + +-- Prevent background merges from racing with the failpoint. +SYSTEM STOP MERGES t_wide_cancel; + +INSERT INTO t_wide_cancel SELECT number, toString(number), number FROM numbers(100); +INSERT INTO t_wide_cancel SELECT number, toString(number), number FROM numbers(100, 100); + +-- Force the Wide writer's addStreams to throw during OPTIMIZE (merge). +SYSTEM ENABLE FAILPOINT wide_part_writer_fail_in_add_streams; +SYSTEM START MERGES t_wide_cancel; + +OPTIMIZE TABLE t_wide_cancel FINAL; -- {serverError FAULT_INJECTED} + +SYSTEM DISABLE FAILPOINT wide_part_writer_fail_in_add_streams; + +-- The server must still be alive and the table readable. +SELECT count() FROM t_wide_cancel; + +DROP TABLE t_wide_cancel; diff --git a/tests/queries/0_stateless/04093_best_effort_datetime_timezone_boundary_range_check.reference b/tests/queries/0_stateless/04093_best_effort_datetime_timezone_boundary_range_check.reference new file mode 100644 index 000000000000..2634ecb23695 --- /dev/null +++ b/tests/queries/0_stateless/04093_best_effort_datetime_timezone_boundary_range_check.reference @@ -0,0 +1,4 @@ +2106-02-07 07:28:15.000000000 Nullable(DateTime64(9)) +1969-12-31 23:00:00.000000000 Nullable(DateTime64(9)) +2106-02-07 05:28:15 Nullable(DateTime) +1970-01-01 00:00:00 Nullable(DateTime) diff --git a/tests/queries/0_stateless/04093_best_effort_datetime_timezone_boundary_range_check.sql b/tests/queries/0_stateless/04093_best_effort_datetime_timezone_boundary_range_check.sql new file mode 100644 index 000000000000..ec4567169f5b --- /dev/null +++ b/tests/queries/0_stateless/04093_best_effort_datetime_timezone_boundary_range_check.sql @@ -0,0 +1,20 @@ +-- Tags: no-fasttest +-- Test for missing range check after adjust_time_zone in parseDateTimeBestEffortImpl +-- https://github.com/ClickHouse/ClickHouse/issues/102601 + +SET session_timezone = 'UTC'; +SET date_time_input_format = 'best_effort'; + +-- Upper boundary: 2106-02-07 06:28:15 UTC is exactly UINT32_MAX. +-- With -01:00 offset, UTC time is 2106-02-07 07:28:15, which exceeds UINT32_MAX. +-- Should be inferred as DateTime64, not DateTime with wrap-around. +SELECT d, toTypeName(d) FROM format(JSONEachRow, '{"d" : "2106-02-07 06:28:15-01:00"}'); + +-- Lower boundary: 1970-01-01 00:00:00 UTC is timestamp 0. +-- With +01:00 offset, UTC time is 1969-12-31 23:00:00, which is negative. +-- Should be inferred as DateTime64, not DateTime with clamped value. +SELECT d, toTypeName(d) FROM format(JSONEachRow, '{"d" : "1970-01-01 00:00:00+01:00"}'); + +-- Control: values that SHOULD remain DateTime (timezone adjustment stays within range) +SELECT d, toTypeName(d) FROM format(JSONEachRow, '{"d" : "2106-02-07 06:28:15+01:00"}'); +SELECT d, toTypeName(d) FROM format(JSONEachRow, '{"d" : "1970-01-01 01:00:00+01:00"}'); diff --git a/tests/queries/0_stateless/04094_flattened_dynamic_native_encode_types_binary.reference b/tests/queries/0_stateless/04094_flattened_dynamic_native_encode_types_binary.reference new file mode 100644 index 000000000000..664388937d73 --- /dev/null +++ b/tests/queries/0_stateless/04094_flattened_dynamic_native_encode_types_binary.reference @@ -0,0 +1,3 @@ +42 Int64 +hello String +2020-01-01 Date diff --git a/tests/queries/0_stateless/04094_flattened_dynamic_native_encode_types_binary.sh b/tests/queries/0_stateless/04094_flattened_dynamic_native_encode_types_binary.sh new file mode 100755 index 000000000000..fa66b56af53b --- /dev/null +++ b/tests/queries/0_stateless/04094_flattened_dynamic_native_encode_types_binary.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Test: flattened Dynamic serialization with encode_types_in_binary_format=1. +# encodeDataType at SerializationDynamic.cpp must use the two-arg overload +# that writes to the stream, not the one-arg overload that returns a String. + +$CLICKHOUSE_LOCAL -m -q " + CREATE TABLE test (d Dynamic(max_types=2)) ENGINE=Memory; + INSERT INTO test VALUES (42::Int64), ('hello'), ('2020-01-01'::Date); + SELECT * FROM test FORMAT Native SETTINGS + output_format_native_use_flattened_dynamic_and_json_serialization=1, + output_format_native_encode_types_in_binary_format=1; +" | $CLICKHOUSE_LOCAL --table test --input-format Native --input_format_native_decode_types_in_binary_format=1 -q "SELECT d, dynamicType(d) FROM test" diff --git a/tests/queries/0_stateless/04098_row_policy_disjunction_optimization.reference b/tests/queries/0_stateless/04098_row_policy_disjunction_optimization.reference new file mode 100644 index 000000000000..13c8485ff153 --- /dev/null +++ b/tests/queries/0_stateless/04098_row_policy_disjunction_optimization.reference @@ -0,0 +1,8 @@ +old analyzer +Row level filter column: in(id, (1, 3, 5)) (removed) +new analyzer +Row level filter column: in(id, __set_UInt64_) (removed) +result +1 +3 +5 diff --git a/tests/queries/0_stateless/04098_row_policy_disjunction_optimization.sql b/tests/queries/0_stateless/04098_row_policy_disjunction_optimization.sql new file mode 100644 index 000000000000..d78888bcadcb --- /dev/null +++ b/tests/queries/0_stateless/04098_row_policy_disjunction_optimization.sql @@ -0,0 +1,36 @@ +-- Tags: no-parallel, no-parallel-replicas + +DROP TABLE IF EXISTS t_row_policy_or; +CREATE TABLE t_row_policy_or (id UInt64, value String) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_row_policy_or SELECT number, toString(number) FROM numbers(10); + +DROP ROW POLICY IF EXISTS 04098_p1 ON t_row_policy_or; +DROP ROW POLICY IF EXISTS 04098_p2 ON t_row_policy_or; +DROP ROW POLICY IF EXISTS 04098_p3 ON t_row_policy_or; + +CREATE ROW POLICY 04098_p1 ON t_row_policy_or USING id = 1 AS permissive TO ALL; +CREATE ROW POLICY 04098_p2 ON t_row_policy_or USING id = 3 AS permissive TO ALL; +CREATE ROW POLICY 04098_p3 ON t_row_policy_or USING id = 5 AS permissive TO ALL; + +-- With the old analyzer the OR chain is converted to IN by LogicalExpressionsOptimizer. +-- With the new analyzer the same should happen via LogicalExpressionOptimizerPass +-- applied to the row policy filter in buildFilterInfo. + +SET enable_analyzer = 0; +SELECT 'old analyzer'; +SELECT trim(BOTH ' ' FROM explain) FROM (EXPLAIN actions = 1 SELECT id FROM t_row_policy_or SETTINGS optimize_move_to_prewhere = 0) +WHERE explain LIKE '%Row level filter column: in(%'; + +SET enable_analyzer = 1; +SELECT 'new analyzer'; +SELECT trim(BOTH ' ' FROM replaceRegexpOne(explain, '__set_UInt64_\\d+_\\d+', '__set_UInt64_')) +FROM (EXPLAIN actions = 1 SELECT id FROM t_row_policy_or SETTINGS optimize_move_to_prewhere = 0) +WHERE explain LIKE '%Row level filter column: in(%'; + +SELECT 'result'; +SELECT id FROM t_row_policy_or ORDER BY id; + +DROP ROW POLICY 04098_p1 ON t_row_policy_or; +DROP ROW POLICY 04098_p2 ON t_row_policy_or; +DROP ROW POLICY 04098_p3 ON t_row_policy_or; +DROP TABLE t_row_policy_or; diff --git a/tests/queries/0_stateless/04108_dynamic_flattened_malformed_index.reference b/tests/queries/0_stateless/04108_dynamic_flattened_malformed_index.reference new file mode 100644 index 000000000000..336d78135719 --- /dev/null +++ b/tests/queries/0_stateless/04108_dynamic_flattened_malformed_index.reference @@ -0,0 +1 @@ +Incorrect index 10 in indexes column of flattened Dynamic column at row 2: the index should be in range [0, 2] (there are 2 types, index 2 is reserved for NULL values) diff --git a/tests/queries/0_stateless/04108_dynamic_flattened_malformed_index.sh b/tests/queries/0_stateless/04108_dynamic_flattened_malformed_index.sh new file mode 100755 index 000000000000..027009815edb --- /dev/null +++ b/tests/queries/0_stateless/04108_dynamic_flattened_malformed_index.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Test that reading a Native file with a corrupted index in flattened Dynamic +# serialization produces an informative error message instead of a crash. +# +# The file data_native/dynamic_flattened_bad_index.native is a pregenerated +# Native file with flattened Dynamic serialization for Dynamic(max_types=2) +# containing 3 rows: (42::Int64, 'hello', NULL). The NULL index byte (value 2) +# was changed to 10, which exceeds the valid range [0, 2]. + +$CLICKHOUSE_LOCAL --table test --input-format Native -q "SELECT * FROM test" < "${CUR_DIR}/data_native/dynamic_flattened_bad_index.native" 2>&1 | grep -o 'Incorrect index.*reserved for NULL values)' diff --git a/tests/queries/0_stateless/data_native/dynamic_flattened_bad_index.native b/tests/queries/0_stateless/data_native/dynamic_flattened_bad_index.native new file mode 100644 index 000000000000..787e73ad1e27 Binary files /dev/null and b/tests/queries/0_stateless/data_native/dynamic_flattened_bad_index.native differ