diff --git a/.gitmodules b/.gitmodules index 4bc8fb0a547b..d93a346a7c2b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -148,3 +148,6 @@ path = contrib/avro url = https://github.com/ClickHouse-Extras/avro.git ignore = untracked +[submodule "contrib/fmtlib"] + path = contrib/fmtlib + url = https://github.com/fmtlib/fmt.git diff --git a/base/common/CMakeLists.txt b/base/common/CMakeLists.txt index 682e622bb77d..68b77b68c7c3 100644 --- a/base/common/CMakeLists.txt +++ b/base/common/CMakeLists.txt @@ -90,6 +90,7 @@ target_link_libraries (common ${Poco_Foundation_LIBRARY} ${CITYHASH_LIBRARIES} ${Boost_SYSTEM_LIBRARY} + fmt ) if (ENABLE_TESTS) diff --git a/base/common/LineReader.cpp b/base/common/LineReader.cpp index c69690e34200..3a3e54dc94e5 100644 --- a/base/common/LineReader.cpp +++ b/base/common/LineReader.cpp @@ -67,8 +67,8 @@ LineReader::Suggest::WordsRange LineReader::Suggest::getCompletions(const String }); } -LineReader::LineReader(const String & history_file_path_, char extender_, char delimiter_) - : history_file_path(history_file_path_), extender(extender_), delimiter(delimiter_) +LineReader::LineReader(const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_) + : history_file_path(history_file_path_), multiline(multiline_), extenders(std::move(extenders_)), delimiters(std::move(delimiters_)) { /// FIXME: check extender != delimiter } @@ -76,38 +76,60 @@ LineReader::LineReader(const String & history_file_path_, char extender_, char d String LineReader::readLine(const String & first_prompt, const String & second_prompt) { String line; - bool is_multiline = false; + bool need_next_line = false; - while (auto status = readOneLine(is_multiline ? second_prompt : first_prompt)) + while (auto status = readOneLine(need_next_line ? second_prompt : first_prompt)) { if (status == RESET_LINE) { line.clear(); - is_multiline = false; + need_next_line = false; continue; } if (input.empty()) { - if (!line.empty() && !delimiter && !hasInputData()) + if (!line.empty() && !multiline && !hasInputData()) break; else continue; } - is_multiline = (input.back() == extender) || (delimiter && input.back() != delimiter) || hasInputData(); +#if !defined(ARCADIA_BUILD) /// C++20 + const char * has_extender = nullptr; + for (const auto * extender : extenders) + { + if (input.ends_with(extender)) + { + has_extender = extender; + break; + } + } - if (input.back() == extender) + const char * has_delimiter = nullptr; + for (const auto * delimiter : delimiters) { - input = input.substr(0, input.size() - 1); + if (input.ends_with(delimiter)) + { + has_delimiter = delimiter; + break; + } + } + + need_next_line = has_extender || (multiline && !has_delimiter) || hasInputData(); + + if (has_extender) + { + input.resize(input.size() - strlen(has_extender)); trim(input); if (input.empty()) continue; } +#endif line += (line.empty() ? "" : " ") + input; - if (!is_multiline) + if (!need_next_line) break; } @@ -127,7 +149,7 @@ LineReader::InputStatus LineReader::readOneLine(const String & prompt) #ifdef OS_LINUX if (!readline_ptr) { - for (auto name : {"libreadline.so", "libreadline.so.0", "libeditline.so", "libeditline.so.0"}) + for (const auto * name : {"libreadline.so", "libreadline.so.0", "libeditline.so", "libeditline.so.0"}) { void * dl_handle = dlopen(name, RTLD_LAZY); if (dl_handle) diff --git a/base/common/LineReader.h b/base/common/LineReader.h index 66de46d5fcbd..99dd8d0824b7 100644 --- a/base/common/LineReader.h +++ b/base/common/LineReader.h @@ -23,7 +23,9 @@ class LineReader bool case_insensitive = false; }; - LineReader(const String & history_file_path, char extender, char delimiter = 0); /// if delimiter != 0, then it's multiline mode + using Patterns = std::vector; + + LineReader(const String & history_file_path, bool multiline, Patterns extenders, Patterns delimiters); virtual ~LineReader() {} /// Reads the whole line until delimiter (in multiline mode) or until the last line without extender. @@ -53,8 +55,10 @@ class LineReader String input; private: - const char extender; - const char delimiter; + bool multiline; + + Patterns extenders; + Patterns delimiters; String prev_line; diff --git a/base/common/ReadlineLineReader.cpp b/base/common/ReadlineLineReader.cpp index fdbb929be79e..2c8b62ea3dda 100644 --- a/base/common/ReadlineLineReader.cpp +++ b/base/common/ReadlineLineReader.cpp @@ -56,8 +56,9 @@ static char * generate(const char * text, int state) return nextMatch(); }; -ReadlineLineReader::ReadlineLineReader(const Suggest & suggest_, const String & history_file_path_, char extender_, char delimiter_) - : LineReader(history_file_path_, extender_, delimiter_) +ReadlineLineReader::ReadlineLineReader( + const Suggest & suggest_, const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_) + : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_)) { suggest = &suggest_; diff --git a/base/common/ReadlineLineReader.h b/base/common/ReadlineLineReader.h index 395ae56c7242..95bd23b46347 100644 --- a/base/common/ReadlineLineReader.h +++ b/base/common/ReadlineLineReader.h @@ -8,7 +8,7 @@ class ReadlineLineReader : public LineReader { public: - ReadlineLineReader(const Suggest & suggest, const String & history_file_path, char extender, char delimiter = 0); + ReadlineLineReader(const Suggest & suggest, const String & history_file_path, bool multiline, Patterns extenders_, Patterns delimiters_); ~ReadlineLineReader() override; void enableBracketedPaste() override; diff --git a/base/common/ReplxxLineReader.cpp b/base/common/ReplxxLineReader.cpp index 40fcc04dedce..141237d5d945 100644 --- a/base/common/ReplxxLineReader.cpp +++ b/base/common/ReplxxLineReader.cpp @@ -16,8 +16,9 @@ void trim(String & s) } -ReplxxLineReader::ReplxxLineReader(const Suggest & suggest, const String & history_file_path_, char extender_, char delimiter_) - : LineReader(history_file_path_, extender_, delimiter_) +ReplxxLineReader::ReplxxLineReader( + const Suggest & suggest, const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_) + : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_)) { using namespace std::placeholders; using Replxx = replxx::Replxx; @@ -37,13 +38,13 @@ ReplxxLineReader::ReplxxLineReader(const Suggest & suggest, const String & histo /// By default C-p/C-n binded to COMPLETE_NEXT/COMPLETE_PREV, /// bind C-p/C-n to history-previous/history-next like readline. - rx.bind_key(Replxx::KEY::control('N'), std::bind(&Replxx::invoke, &rx, Replxx::ACTION::HISTORY_NEXT, _1)); - rx.bind_key(Replxx::KEY::control('P'), std::bind(&Replxx::invoke, &rx, Replxx::ACTION::HISTORY_PREVIOUS, _1)); + rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); }); + rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); }); /// By default COMPLETE_NEXT/COMPLETE_PREV was binded to C-p/C-n, re-bind /// to M-P/M-N (that was used for HISTORY_COMMON_PREFIX_SEARCH before, but /// it also binded to M-p/M-n). - rx.bind_key(Replxx::KEY::meta('N'), std::bind(&Replxx::invoke, &rx, Replxx::ACTION::COMPLETE_NEXT, _1)); - rx.bind_key(Replxx::KEY::meta('P'), std::bind(&Replxx::invoke, &rx, Replxx::ACTION::COMPLETE_PREVIOUS, _1)); + rx.bind_key(Replxx::KEY::meta('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_NEXT, code); }); + rx.bind_key(Replxx::KEY::meta('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_PREVIOUS, code); }); } ReplxxLineReader::~ReplxxLineReader() diff --git a/base/common/ReplxxLineReader.h b/base/common/ReplxxLineReader.h index e7821f54ad3c..472198bcfafa 100644 --- a/base/common/ReplxxLineReader.h +++ b/base/common/ReplxxLineReader.h @@ -7,7 +7,7 @@ class ReplxxLineReader : public LineReader { public: - ReplxxLineReader(const Suggest & suggest, const String & history_file_path, char extender, char delimiter = 0); + ReplxxLineReader(const Suggest & suggest, const String & history_file_path, bool multiline, Patterns extenders_, Patterns delimiters_); ~ReplxxLineReader() override; void enableBracketedPaste() override; diff --git a/base/common/argsToConfig.cpp b/base/common/argsToConfig.cpp index b0ec29002684..13dba86973e9 100644 --- a/base/common/argsToConfig.cpp +++ b/base/common/argsToConfig.cpp @@ -11,7 +11,7 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::Laye /// Test: -- --1=1 --1=2 --3 5 7 8 -9 10 -11=12 14= 15== --16==17 --=18 --19= --20 21 22 --23 --24 25 --26 -27 28 ---29=30 -- ----31 32 --33 3-4 Poco::AutoPtr map_config = new Poco::Util::MapConfiguration; std::string key; - for (auto & arg : argv) + for (const auto & arg : argv) { auto key_start = arg.find_first_not_of('-'); auto pos_minus = arg.find('-'); diff --git a/base/common/phdr_cache.cpp b/base/common/phdr_cache.cpp index fc81c20e8dd2..cef386a85b87 100644 --- a/base/common/phdr_cache.cpp +++ b/base/common/phdr_cache.cpp @@ -70,7 +70,7 @@ extern "C" #endif int dl_iterate_phdr(int (*callback) (dl_phdr_info * info, size_t size, void * data), void * data) { - auto current_phdr_cache = phdr_cache.load(); + auto * current_phdr_cache = phdr_cache.load(); if (!current_phdr_cache) { // Cache is not yet populated, pass through to the original function. diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt new file mode 100644 index 000000000000..506313696ea3 --- /dev/null +++ b/cmake/autogenerated_versions.txt @@ -0,0 +1,9 @@ +# This strings autochanged from release_lib.sh: +SET(VERSION_REVISION 54433) +SET(VERSION_MAJOR 20) +SET(VERSION_MINOR 3) +SET(VERSION_PATCH 14) +SET(VERSION_GITHASH 37bdc0588422d170f8bc36f152f4ec3964b07a9b) +SET(VERSION_DESCRIBE v20.3.14.130-lts) +SET(VERSION_STRING 20.3.14.130) +# end of autochange diff --git a/cmake/version.cmake b/cmake/version.cmake index 8286c52b510f..eea17f68c47c 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -1,12 +1,4 @@ -# This strings autochanged from release_lib.sh: -set(VERSION_REVISION 54433) -set(VERSION_MAJOR 20) -set(VERSION_MINOR 3) -set(VERSION_PATCH 10) -set(VERSION_GITHASH 257cd6cadf7380ee6936d9c00922176e5d268992) -set(VERSION_DESCRIBE v20.3.10.75-lts) -set(VERSION_STRING 20.3.10.75) -# end of autochange +include(${CMAKE_SOURCE_DIR}/cmake/autogenerated_versions.txt) set(VERSION_EXTRA "" CACHE STRING "") set(VERSION_TWEAK "" CACHE STRING "") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 0208b4e859af..4dbf6d079c90 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -335,3 +335,6 @@ endif() add_subdirectory(grpc-cmake) add_subdirectory(replxx-cmake) + +add_subdirectory (fmtlib-cmake) + diff --git a/contrib/cppkafka b/contrib/cppkafka index 9b184d881c15..f555ee36aaa7 160000 --- a/contrib/cppkafka +++ b/contrib/cppkafka @@ -1 +1 @@ -Subproject commit 9b184d881c15cc50784b28688c7c99d3d764db24 +Subproject commit f555ee36aaa74d17ca0dab3ce472070a610b2966 diff --git a/contrib/fmtlib b/contrib/fmtlib new file mode 160000 index 000000000000..297c3b2ed551 --- /dev/null +++ b/contrib/fmtlib @@ -0,0 +1 @@ +Subproject commit 297c3b2ed551a4989826fc8c4780bf533e964bd9 diff --git a/contrib/fmtlib-cmake/CMakeLists.txt b/contrib/fmtlib-cmake/CMakeLists.txt new file mode 100644 index 000000000000..f3bf73d7dbc5 --- /dev/null +++ b/contrib/fmtlib-cmake/CMakeLists.txt @@ -0,0 +1,20 @@ +set (SRCS + ../fmtlib/src/format.cc + ../fmtlib/src/os.cc + + ../fmtlib/include/fmt/chrono.h + ../fmtlib/include/fmt/color.h + ../fmtlib/include/fmt/compile.h + ../fmtlib/include/fmt/core.h + ../fmtlib/include/fmt/format.h + ../fmtlib/include/fmt/format-inl.h + ../fmtlib/include/fmt/locale.h + ../fmtlib/include/fmt/os.h + ../fmtlib/include/fmt/ostream.h + ../fmtlib/include/fmt/posix.h + ../fmtlib/include/fmt/printf.h + ../fmtlib/include/fmt/ranges.h +) + +add_library(fmt ${SRCS}) +target_include_directories(fmt SYSTEM PUBLIC ../fmtlib/include) diff --git a/debian/changelog b/debian/changelog index 7ba58111b81c..a1f0871fcfca 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (20.3.10.1) unstable; urgency=low +clickhouse (20.3.14.1) unstable; urgency=low * Modified source code - -- clickhouse-release Tue, 19 May 2020 02:44:39 +0300 + -- clickhouse-release Wed, 15 Jul 2020 11:53:50 +0300 diff --git a/debian/rules b/debian/rules index dabebb516cd4..7218e196baa2 100755 --- a/debian/rules +++ b/debian/rules @@ -24,6 +24,10 @@ DEB_BUILD_OPTIONS+=parallel=$(THREADS_COUNT) ifndef ENABLE_TESTS CMAKE_FLAGS += -DENABLE_TESTS=0 +else +# To export binaries and from deb build we do not strip them. No need to run tests in deb build as we run them in CI + DEB_BUILD_OPTIONS+= nocheck + DEB_BUILD_OPTIONS+= nostrip endif ifndef MAKE_TARGET @@ -88,14 +92,19 @@ override_dh_auto_build: $(MAKE) $(THREADS_COUNT) -C $(BUILDDIR) $(MAKE_TARGET) override_dh_auto_test: +ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) cd $(BUILDDIR) && ctest $(THREADS_COUNT) -V -E with_server +endif override_dh_clean: rm -rf debian/copyright debian/clickhouse-client.docs debian/clickhouse-common-static.docs - dh_clean -X contrib + dh_clean # -X contrib override_dh_strip: +#https://www.debian.org/doc/debian-policy/ch-source.html#debian-rules-and-deb-build-options +ifeq (,$(filter nostrip,$(DEB_BUILD_OPTIONS))) dh_strip -pclickhouse-common-static --dbg-package=clickhouse-common-static-dbg +endif override_dh_install: # Making docs diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index e0433fc03562..a4f27c797bf8 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=20.3.10.* +ARG version=20.3.14.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/images.json b/docker/images.json index c7dfc82d9060..23f8cc0d9fdd 100644 --- a/docker/images.json +++ b/docker/images.json @@ -1,18 +1,83 @@ { - "docker/packager/deb": "yandex/clickhouse-deb-builder", - "docker/packager/binary": "yandex/clickhouse-binary-builder", - "docker/test/compatibility/centos": "yandex/clickhouse-test-old-centos", - "docker/test/compatibility/ubuntu": "yandex/clickhouse-test-old-ubuntu", - "docker/test/integration": "yandex/clickhouse-integration-test", - "docker/test/performance": "yandex/clickhouse-performance-test", - "docker/test/performance-comparison": "yandex/clickhouse-performance-comparison", - "docker/test/pvs": "yandex/clickhouse-pvs-test", - "docker/test/stateful": "yandex/clickhouse-stateful-test", - "docker/test/stateful_with_coverage": "yandex/clickhouse-stateful-test-with-coverage", - "docker/test/stateless": "yandex/clickhouse-stateless-test", - "docker/test/stateless_with_coverage": "yandex/clickhouse-stateless-test-with-coverage", - "docker/test/unit": "yandex/clickhouse-unit-test", - "docker/test/stress": "yandex/clickhouse-stress-test", - "docker/test/split_build_smoke_test": "yandex/clickhouse-split-build-smoke-test", - "tests/integration/image": "yandex/clickhouse-integration-tests-runner" + "docker/packager/deb": { + "name": "yandex/clickhouse-deb-builder", + "dependent": [ + "docker/test/stateless", + "docker/test/stateless_with_coverage", + "docker/test/stateless_pytest", + "docker/test/coverage" + ] + }, + "docker/packager/binary": { + "name": "yandex/clickhouse-binary-builder", + "dependent": [ + "docker/test/split_build_smoke_test", + "docker/test/pvs" + ] + }, + "docker/test/coverage": { + "name": "yandex/clickhouse-coverage", + "dependent": [] + }, + "docker/test/compatibility/centos": { + "name": "yandex/clickhouse-test-old-centos", + "dependent": [] + }, + "docker/test/compatibility/ubuntu": { + "name": "yandex/clickhouse-test-old-ubuntu", + "dependent": [] + }, + "docker/test/integration/base": { + "name": "yandex/clickhouse-integration-test", + "dependent": [] + }, + "docker/test/performance-comparison": { + "name": "yandex/clickhouse-performance-comparison", + "dependent": [] + }, + "docker/test/stateful": { + "name": "yandex/clickhouse-stateful-test", + "dependent": [ + "docker/test/stress" + ] + }, + "docker/test/stateful_with_coverage": { + "name": "yandex/clickhouse-stateful-test-with-coverage", + "dependent": [] + }, + "docker/test/stateless": { + "name": "yandex/clickhouse-stateless-test", + "dependent": [ + "docker/test/stateful", + "docker/test/stateful_with_coverage" + ] + }, + "docker/test/stateless_pytest": { + "name": "yandex/clickhouse-stateless-pytest", + "dependent": [] + }, + "docker/test/stateless_with_coverage": { + "name": "yandex/clickhouse-stateless-test-with-coverage", + "dependent": [] + }, + "docker/test/unit": { + "name": "yandex/clickhouse-unit-test", + "dependent": [] + }, + "docker/test/stress": { + "name": "yandex/clickhouse-stress-test", + "dependent": [] + }, + "docker/test/split_build_smoke_test": { + "name": "yandex/clickhouse-split-build-smoke-test", + "dependent": [] + }, + "docker/test/codebrowser": { + "name": "yandex/clickhouse-codebrowser", + "dependent": [] + }, + "docker/test/integration/runner": { + "name": "yandex/clickhouse-integration-tests-runner", + "dependent": [] + } } diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh index 6d5144266ae7..707d71b1d209 100755 --- a/docker/packager/deb/build.sh +++ b/docker/packager/deb/build.sh @@ -10,4 +10,15 @@ mv *.changes /output mv *.buildinfo /output mv /*.rpm /output ||: # if exists mv /*.tgz /output ||: # if exists + +if [ -n "$BINARY_OUTPUT" ] && { [ "$BINARY_OUTPUT" = "programs" ] || [ "$BINARY_OUTPUT" = "tests" ] ;} +then + echo Place $BINARY_OUTPUT to output + mkdir /output/binary ||: # if exists + mv /build/obj-*/programs/clickhouse* /output/binary + if [ "$BINARY_OUTPUT" = "tests" ] + then + mv /build/obj-*/src/unit_tests_dbms /output/binary + fi +fi ccache --show-stats ||: diff --git a/docker/packager/packager b/docker/packager/packager index cb79a6297e6c..7583a53d64c8 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -11,48 +11,8 @@ SCRIPT_PATH = os.path.realpath(__file__) IMAGE_MAP = { "deb": "yandex/clickhouse-deb-builder", "binary": "yandex/clickhouse-binary-builder", - "freebsd": os.path.join(os.path.dirname(SCRIPT_PATH), "freebsd"), } -class Vagrant(object): - def __init__(self, path_to_vagrant_file): - self.prefix = "VAGRANT_CWD=" + path_to_vagrant_file - - def __enter__(self): - subprocess.check_call("{} vagrant up".format(self.prefix), shell=True) - self.ssh_path = "/tmp/vagrant-ssh" - subprocess.check_call("{} vagrant ssh-config > {}".format(self.prefix, self.ssh_path), shell=True) - return self - - def copy_to_image(self, local_path, remote_path): - cmd = "scp -F {ssh} -r {lpath} default:{rpath}".format(ssh=self.ssh_path, lpath=local_path, rpath=remote_path) - logging.info("Copying to image %s", cmd) - subprocess.check_call( - cmd, - shell=True - ) - - def copy_from_image(self, remote_path, local_path): - cmd = "scp -F {ssh} -r default:{rpath} {lpath}".format(ssh=self.ssh_path, rpath=remote_path, lpath=local_path) - logging.info("Copying from image %s", cmd) - subprocess.check_call( - cmd, - shell=True - ) - - def execute_cmd(self, cmd): - cmd = '{} vagrant ssh -c "{}"'.format(self.prefix, cmd) - logging.info("Executin cmd %s", cmd) - subprocess.check_call( - cmd, - shell=True - ) - - def __exit__(self, exc_type, exc_val, exc_tb): - logging.info("Destroying image") - subprocess.check_call("{} vagrant destroy --force".format(self.prefix), shell=True) - - def check_image_exists_locally(image_name): try: output = subprocess.check_output("docker images -q {} 2> /dev/null".format(image_name), shell=True) @@ -94,19 +54,11 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache subprocess.check_call(cmd, shell=True) -def run_vagrant_box_with_env(image_path, output_dir, ch_root): - with Vagrant(image_path) as vagrant: - logging.info("Copying folder to vagrant machine") - vagrant.copy_to_image(ch_root, "~/ClickHouse") - logging.info("Running build") - vagrant.execute_cmd("cd ~/ClickHouse && cmake . && ninja") - logging.info("Copying binary back") - vagrant.copy_from_image("~/ClickHouse/programs/clickhouse", output_dir) - -def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, version, author, official, alien_pkgs, with_coverage): +def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage, with_binaries): CLANG_PREFIX = "clang" DARWIN_SUFFIX = "-darwin" ARM_SUFFIX = "-aarch64" + FREEBSD_SUFFIX = "-freebsd" result = [] cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1'] @@ -114,7 +66,8 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ is_clang = compiler.startswith(CLANG_PREFIX) is_cross_darwin = compiler.endswith(DARWIN_SUFFIX) is_cross_arm = compiler.endswith(ARM_SUFFIX) - is_cross_compile = is_cross_darwin or is_cross_arm + is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX) + is_cross_compile = is_cross_darwin or is_cross_arm or is_cross_freebsd # Explicitly use LLD with Clang by default. # Don't force linker for cross-compilation. @@ -131,6 +84,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ elif is_cross_arm: cc = compiler[:-len(ARM_SUFFIX)] cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake") + elif is_cross_freebsd: + cc = compiler[:-len(FREEBSD_SUFFIX)] + cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake") else: cc = compiler @@ -175,13 +131,24 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if alien_pkgs: result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'") + if with_binaries == "programs": + result.append('BINARY_OUTPUT=programs') + elif with_binaries == "tests": + result.append('ENABLE_TESTS=1') + result.append('BINARY_OUTPUT=tests') + cmake_flags.append('-DENABLE_TESTS=1') + cmake_flags.append('-DUSE_GTEST=1') + if unbundled: # TODO: fix build with ENABLE_RDKAFKA - cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0') + cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0') if split_binary: cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') + if clang_tidy: + cmake_flags.append('-DENABLE_CLANG_TIDY=1') + if with_coverage: cmake_flags.append('-DWITH_COVERAGE=1') @@ -202,14 +169,15 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') parser = argparse.ArgumentParser(description="ClickHouse building script using prebuilt Docker image") # 'performance' creates a combined .tgz with server and configs to be used for performance test. - parser.add_argument("--package-type", choices=['deb', 'binary', 'performance', 'freebsd'], required=True) - parser.add_argument("--clickhouse-repo-path", default="../../") + parser.add_argument("--package-type", choices=['deb', 'binary', 'performance'], required=True) + parser.add_argument("--clickhouse-repo-path", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) parser.add_argument("--output-dir", required=True) parser.add_argument("--build-type", choices=("debug", ""), default="") parser.add_argument("--compiler", choices=("clang-8", "clang-8-darwin", "clang-9-aarch64", "clang-9-freebsd", "gcc-9", "clang-9", "clang-10"), default="gcc-9") parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") parser.add_argument("--unbundled", action="store_true") parser.add_argument("--split-binary", action="store_true") + parser.add_argument("--clang-tidy", action="store_true") parser.add_argument("--cache", choices=("", "ccache", "distcc"), default="") parser.add_argument("--ccache_dir", default= os.getenv("HOME", "") + '/.ccache') parser.add_argument("--distcc-hosts", nargs="+") @@ -219,6 +187,7 @@ if __name__ == "__main__": parser.add_argument("--official", action="store_true") parser.add_argument("--alien-pkgs", nargs='+', default=[]) parser.add_argument("--with-coverage", action="store_true") + parser.add_argument("--with-binaries", choices=("programs", "tests", ""), default="") args = parser.parse_args() if not os.path.isabs(args.output_dir): @@ -235,17 +204,19 @@ if __name__ == "__main__": if args.alien_pkgs and not image_type == "deb": raise Exception("Can add alien packages only in deb build") + if args.with_binaries != "" and not image_type == "deb": + raise Exception("Can add additional binaries only in deb build") + + if args.with_binaries != "" and image_type == "deb": + logging.info("Should place {} to output".format(args.with_binaries)) + dockerfile = os.path.join(ch_root, "docker/packager", image_type, "Dockerfile") if image_type != "freebsd" and not check_image_exists_locally(image_name) or args.force_build_image: if not pull_image(image_name) or args.force_build_image: build_image(image_name, dockerfile) env_prepared = parse_env_variables( args.build_type, args.compiler, args.sanitizer, args.package_type, image_type, - args.cache, args.distcc_hosts, args.unbundled, args.split_binary, - args.version, args.author, args.official, args.alien_pkgs, args.with_coverage) - if image_type != "freebsd": - run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir) - else: - logging.info("Running freebsd build, arguments will be ignored") - run_vagrant_box_with_env(image_name, args.output_dir, ch_root) + args.cache, args.distcc_hosts, args.unbundled, args.split_binary, args.clang_tidy, + args.version, args.author, args.official, args.alien_pkgs, args.with_coverage, args.with_binaries) + run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir) logging.info("Output placed into {}".format(args.output_dir)) diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 67aaed587883..febb0945c90d 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=20.3.10.* +ARG version=20.3.14.* ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index fbab44de2b70..5e52ece9f006 100644 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -69,7 +69,7 @@ if [ -n "$(ls /docker-entrypoint-initdb.d/)" ]; then # check if clickhouse is ready to accept connections # will try to send ping clickhouse via http_port (max 12 retries, with 1 sec delay) - if ! wget --spider --quiet --tries=12 --waitretry=1 --retry-connrefused "http://localhost:$HTTP_PORT/ping" ; then + if ! wget --spider --quiet --prefer-family=IPv6 --tries=12 --waitretry=1 --retry-connrefused "http://localhost:$HTTP_PORT/ping" ; then echo >&2 'ClickHouse init process failed.' exit 1 fi diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index c21370a7d787..b0cd53fdb899 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=20.3.10.* +ARG version=20.3.14.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docs/en/operations/table_engines/kafka.md b/docs/en/operations/table_engines/kafka.md index 5731d68a61d9..e9c3e781d187 100644 --- a/docs/en/operations/table_engines/kafka.md +++ b/docs/en/operations/table_engines/kafka.md @@ -26,21 +26,26 @@ SETTINGS [kafka_row_delimiter = 'delimiter_symbol',] [kafka_schema = '',] [kafka_num_consumers = N,] - [kafka_skip_broken_messages = N] + [kafka_max_block_size = 0,] + [kafka_skip_broken_messages = N,] + [kafka_commit_every_batch = 0] ``` + Required parameters: - `kafka_broker_list` – A comma-separated list of brokers (for example, `localhost:9092`). - `kafka_topic_list` – A list of Kafka topics. - `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. -- `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` function, such as ` JSONEachRow`. For more information, see the [Formats](../../interfaces/formats.md) section. +- `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` function, such as `JSONEachRow`. For more information, see the [Formats](../../interfaces/formats.md) section. Optional parameters: - `kafka_row_delimiter` – Delimiter character, which ends the message. -- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object. +- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object. - `kafka_num_consumers` – The number of consumers per table. Default: `1`. Specify more consumers if the throughput of one consumer is insufficient. The total number of consumers should not exceed the number of partitions in the topic, since only one consumer can be assigned per partition. +- `kafka_max_block_size` - The maximum batch size (in messages) for poll (default: `max_block_size`). - `kafka_skip_broken_messages` – Kafka message parser tolerance to schema-incompatible messages per block. Default: `0`. If `kafka_skip_broken_messages = N` then the engine skips *N* Kafka messages that cannot be parsed (a message equals a row of data). +- `kafka_commit_every_batch` - Commit every consumed and handled batch instead of a single commit after writing a whole block (default: `0`). Examples: diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index eea7a5f4fbea..a313dd69909c 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -212,3 +212,9 @@ endif () if (TARGET clickhouse-server AND TARGET copy-headers) add_dependencies(clickhouse-server copy-headers) endif () + +if (ENABLE_TESTS AND USE_GTEST) + set (CLICKHOUSE_ALL_TESTS_TARGETS local_date_time_comparison unit_tests_libcommon unit_tests_dbms hashing_write_buffer hashing_read_buffer in_join_subqueries_preprocessor expression_analyzer) + add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_ALL_TESTS_TARGETS}) + add_dependencies(clickhouse-bundle clickhouse-tests) +endif() diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 5737493b13f7..62494a60c563 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -502,12 +502,15 @@ class Client : public Poco::Util::Application if (!history_file.empty() && !Poco::File(history_file).exists()) Poco::File(history_file).createFile(); + LineReader::Patterns query_extenders = {"\\"}; + LineReader::Patterns query_delimiters = {";", "\\G"}; + #if USE_REPLXX - ReplxxLineReader lr(Suggest::instance(), history_file, '\\', config().has("multiline") ? ';' : 0); -#elif USE_READLINE - ReadlineLineReader lr(Suggest::instance(), history_file, '\\', config().has("multiline") ? ';' : 0); + ReplxxLineReader lr(Suggest::instance(), history_file, config().has("multiline"), query_extenders, query_delimiters); +#elif defined(USE_READLINE) && USE_READLINE + ReadlineLineReader lr(Suggest::instance(), history_file, config().has("multiline"), query_extenders, query_delimiters); #else - LineReader lr(history_file, '\\', config().has("multiline") ? ';' : 0); + LineReader lr(history_file, config().has("multiline"), query_extenders, query_delimiters); #endif /// Enable bracketed-paste-mode only when multiquery is enabled and multiline is @@ -1361,7 +1364,8 @@ class Client : public Poco::Util::Application } else { - out_logs_buf = std::make_unique(server_logs_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT); + out_logs_buf = std::make_unique( + server_logs_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT); wb = out_logs_buf.get(); } } @@ -1559,6 +1563,11 @@ class Client : public Poco::Util::Application if (std::string::npos != embedded_stack_trace_pos && !config().getBool("stacktrace", false)) text.resize(embedded_stack_trace_pos); + /// If we probably have progress bar, we should add additional newline, + /// otherwise exception may display concatenated with the progress bar. + if (need_render_progress) + std::cerr << '\n'; + std::cerr << "Received exception from server (version " << server_version << "):" << std::endl << "Code: " << e.code() << ". " << text << std::endl; } @@ -1794,7 +1803,11 @@ class Client : public Poco::Util::Application std::string text = e.displayText(); std::cerr << "Code: " << e.code() << ". " << text << std::endl; std::cerr << "Table №" << i << std::endl << std::endl; - exit(e.code()); + /// Avoid the case when error exit code can possibly overflow to normal (zero). + auto exit_code = e.code() % 256; + if (exit_code == 0) + exit_code = 255; + exit(exit_code); } } diff --git a/programs/server/HTTPHandler.cpp b/programs/server/HTTPHandler.cpp index 63018cda3504..c2795dbda87c 100644 --- a/programs/server/HTTPHandler.cpp +++ b/programs/server/HTTPHandler.cpp @@ -209,15 +209,12 @@ HTTPHandler::HTTPHandler(IServer & server_) void HTTPHandler::processQuery( + Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params, Poco::Net::HTTPServerResponse & response, Output & used_output) { - Context context = server.context(); - - CurrentThread::QueryScope query_scope(context); - LOG_TRACE(log, "Request URI: " << request.getURI()); std::istream & istr = request.stream(); @@ -686,6 +683,11 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne setThreadName("HTTPHandler"); ThreadStatus thread_status; + /// Should be initialized before anything, + /// For correct memory accounting. + Context context = server.context(); + CurrentThread::QueryScope query_scope(context); + Output used_output; /// In case of exception, send stack trace to client. @@ -709,7 +711,7 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne throw Exception("The Transfer-Encoding is not chunked and there is no Content-Length header for POST request", ErrorCodes::HTTP_LENGTH_REQUIRED); } - processQuery(request, params, response, used_output); + processQuery(context, request, params, response, used_output); LOG_INFO(log, "Done processing query"); } catch (...) diff --git a/programs/server/HTTPHandler.h b/programs/server/HTTPHandler.h index fb6c9fb532c9..613da029497e 100644 --- a/programs/server/HTTPHandler.h +++ b/programs/server/HTTPHandler.h @@ -65,6 +65,7 @@ class HTTPHandler : public Poco::Net::HTTPRequestHandler /// Also initializes 'used_output'. void processQuery( + Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params, Poco::Net::HTTPServerResponse & response, diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 609306091fe8..9dc0b30fdf28 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -535,7 +535,7 @@ int Server::main(const std::vector & /*args*/) if (mark_cache_size > max_cache_size) { mark_cache_size = max_cache_size; - LOG_INFO(log, "Mark cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) + LOG_INFO(log, "Mark cache size was lowered to " << formatReadableSizeWithBinarySuffix(mark_cache_size) << " because the system has low amount of memory"); } global_context->setMarkCache(mark_cache_size); @@ -641,7 +641,7 @@ int Server::main(const std::vector & /*args*/) if (!hasLinuxCapability(CAP_SYS_NICE)) { - LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect." + LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_priority' will have no effect." " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep " << executable_path << "'." " Note that it will not work on 'nosuid' mounted filesystems."); diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index 317e766bc0fe..c67257fe4bf9 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -299,17 +299,17 @@ void TCPHandler::runImpl() * We will try to send exception to the client in any case - see below. */ state.io.onException(); - exception.emplace(Exception::CreateFromPoco, e); + exception.emplace(Exception::CreateFromPocoTag{}, e); } catch (const Poco::Exception & e) { state.io.onException(); - exception.emplace(Exception::CreateFromPoco, e); + exception.emplace(Exception::CreateFromPocoTag{}, e); } catch (const std::exception & e) { state.io.onException(); - exception.emplace(Exception::CreateFromSTD, e); + exception.emplace(Exception::CreateFromSTDTag{}, e); } catch (...) { diff --git a/programs/server/users.d/access_management.xml b/programs/server/users.d/access_management.xml new file mode 100644 index 000000000000..7e799cb7b10d --- /dev/null +++ b/programs/server/users.d/access_management.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/src/Access/AccessRightsContext.cpp b/src/Access/AccessRightsContext.cpp index 833ad2cd74d9..e9660ad95208 100644 --- a/src/Access/AccessRightsContext.cpp +++ b/src/Access/AccessRightsContext.cpp @@ -156,9 +156,9 @@ void AccessRightsContext::setUser(const UserPtr & user_) const current_roles.reserve(params.current_roles.size()); for (const auto & id : params.current_roles) { - if (user->granted_roles.contains(id)) + if (user->granted_roles.count(id)) current_roles.push_back(id); - if (user->granted_roles_with_admin_option.contains(id)) + if (user->granted_roles_with_admin_option.count(id)) current_roles_with_admin_option.push_back(id); } } @@ -380,7 +380,7 @@ void AccessRightsContext::checkAdminOption(const UUID & role_id) const } } - if (enabled_roles->contains(role_id)) + if (enabled_roles->count(role_id)) return; std::optional role_name = manager->readName(role_id); diff --git a/src/Access/GenericRoleSet.cpp b/src/Access/GenericRoleSet.cpp index 1e751f995c1f..c28e153480ad 100644 --- a/src/Access/GenericRoleSet.cpp +++ b/src/Access/GenericRoleSet.cpp @@ -251,44 +251,44 @@ void GenericRoleSet::add(const boost::container::flat_set & ids_) bool GenericRoleSet::match(const UUID & id) const { - return (all || ids.contains(id)) && !except_ids.contains(id); + return (all || ids.count(id)) && !except_ids.count(id); } bool GenericRoleSet::match(const UUID & user_id, const std::vector & enabled_roles) const { - if (!all && !ids.contains(user_id)) + if (!all && !ids.count(user_id)) { bool found_enabled_role = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.contains(enabled_role); }); + enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.count(enabled_role); }); if (!found_enabled_role) return false; } - if (except_ids.contains(user_id)) + if (except_ids.count(user_id)) return false; bool in_except_list = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.contains(enabled_role); }); + enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.count(enabled_role); }); return !in_except_list; } bool GenericRoleSet::match(const UUID & user_id, const boost::container::flat_set & enabled_roles) const { - if (!all && !ids.contains(user_id)) + if (!all && !ids.count(user_id)) { bool found_enabled_role = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.contains(enabled_role); }); + enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.count(enabled_role); }); if (!found_enabled_role) return false; } - if (except_ids.contains(user_id)) + if (except_ids.count(user_id)) return false; bool in_except_list = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.contains(enabled_role); }); + enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.count(enabled_role); }); return !in_except_list; } diff --git a/src/Access/RoleContextFactory.cpp b/src/Access/RoleContextFactory.cpp index 3356bc238db3..ea22e9f21df7 100644 --- a/src/Access/RoleContextFactory.cpp +++ b/src/Access/RoleContextFactory.cpp @@ -29,7 +29,7 @@ RoleContextPtr RoleContextFactory::createContext( boost::container::flat_set roles_with_admin_option_set{roles_with_admin_option.begin(), roles_with_admin_option.end()}; for (const auto & role : roles) { - if (!roles_with_admin_option_set.contains(role)) + if (!roles_with_admin_option_set.count(role)) children.push_back(createContextImpl(role, false)); } diff --git a/src/Access/RowPolicyContext.cpp b/src/Access/RowPolicyContext.cpp index 661a6cb4b5f6..432031626eb1 100644 --- a/src/Access/RowPolicyContext.cpp +++ b/src/Access/RowPolicyContext.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -8,6 +9,26 @@ namespace DB { +namespace +{ + bool tryGetLiteralBool(const IAST & ast, bool & value) + { + try + { + if (const ASTLiteral * literal = ast.as()) + { + value = !literal->value.isNull() && applyVisitor(FieldVisitorConvertToNumber(), literal->value); + return true; + } + return false; + } + catch (...) + { + return false; + } + } +} + size_t RowPolicyContext::Hash::operator()(const DatabaseAndTableNameRef & database_and_table_name) const { return std::hash{}(database_and_table_name.first) - std::hash{}(database_and_table_name.second); @@ -35,15 +56,24 @@ ASTPtr RowPolicyContext::getCondition(const String & database, const String & ta auto it = loaded->find({database, table_name}); if (it == loaded->end()) return {}; - return it->second.mixed_conditions[index]; + auto ast = it->second.mixed_conditions[index]; + bool value; + if (ast && tryGetLiteralBool(*ast, value) && value) + ast = nullptr; /// The condition is always true, no need to check it. + return ast; } ASTPtr RowPolicyContext::combineConditionsUsingAnd(const ASTPtr & lhs, const ASTPtr & rhs) { - if (!lhs) + bool lhs_value, rhs_value; + if (!lhs || (tryGetLiteralBool(*lhs, lhs_value) && lhs_value)) + { + if (!rhs || (tryGetLiteralBool(*rhs, rhs_value) && rhs_value)) + return nullptr; return rhs; - if (!rhs) + } + if (!rhs || (tryGetLiteralBool(*rhs, rhs_value) && rhs_value)) return lhs; auto function = std::make_shared(); auto exp_list = std::make_shared(); diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 20ee2a628a6a..f9c8db7dc13e 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -55,18 +55,20 @@ namespace String user_config = "users." + user_name; - bool has_password = config.has(user_config + ".password"); + bool has_no_password = config.has(user_config + ".no_password"); + bool has_password_plaintext = config.has(user_config + ".password"); bool has_password_sha256_hex = config.has(user_config + ".password_sha256_hex"); bool has_password_double_sha1_hex = config.has(user_config + ".password_double_sha1_hex"); - if (has_password + has_password_sha256_hex + has_password_double_sha1_hex > 1) - throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex' is used to specify password for user " + user_name + ". Must be only one of them.", + size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex; + if (num_password_fields > 1) + throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex', 'no_password' are used to specify password for user " + user_name + ". Must be only one of them.", ErrorCodes::BAD_ARGUMENTS); - if (!has_password && !has_password_sha256_hex && !has_password_double_sha1_hex) - throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS); + if (num_password_fields < 1) + throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' or 'no_password' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS); - if (has_password) + if (has_password_plaintext) { user->authentication = Authentication{Authentication::PLAINTEXT_PASSWORD}; user->authentication.setPassword(config.getString(user_config + ".password")); diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.cpp b/src/AggregateFunctions/AggregateFunctionQuantile.cpp index 993b25d6a6e3..816fabad2222 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.cpp +++ b/src/AggregateFunctions/AggregateFunctionQuantile.cpp @@ -70,17 +70,17 @@ AggregateFunctionPtr createAggregateFunctionQuantile(const std::string & name, c WhichDataType which(argument_type); #define DISPATCH(TYPE) \ - if (which.idx == TypeIndex::TYPE) return std::make_shared>(argument_type, params); + if (which.idx == TypeIndex::TYPE) return std::make_shared>(argument_types, params); FOR_NUMERIC_TYPES(DISPATCH) #undef DISPATCH - if (which.idx == TypeIndex::Date) return std::make_shared>(argument_type, params); - if (which.idx == TypeIndex::DateTime) return std::make_shared>(argument_type, params); + if (which.idx == TypeIndex::Date) return std::make_shared>(argument_types, params); + if (which.idx == TypeIndex::DateTime) return std::make_shared>(argument_types, params); if constexpr (supportDecimal()) { - if (which.idx == TypeIndex::Decimal32) return std::make_shared>(argument_type, params); - if (which.idx == TypeIndex::Decimal64) return std::make_shared>(argument_type, params); - if (which.idx == TypeIndex::Decimal128) return std::make_shared>(argument_type, params); + if (which.idx == TypeIndex::Decimal32) return std::make_shared>(argument_types, params); + if (which.idx == TypeIndex::Decimal64) return std::make_shared>(argument_types, params); + if (which.idx == TypeIndex::Decimal128) return std::make_shared>(argument_types, params); } throw Exception("Illegal type " + argument_type->getName() + " of argument for aggregate function " + name, diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h index f85eb15b3ab9..3c83c091878d 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -78,8 +78,8 @@ class AggregateFunctionQuantile final : public IAggregateFunctionDataHelper>({argument_type_}, params) + AggregateFunctionQuantile(const DataTypes & argument_types_, const Array & params) + : IAggregateFunctionDataHelper>(argument_types_, params) , levels(params, returns_many), level(levels.levels[0]), argument_type(this->argument_types[0]) { if (!returns_many && levels.size() > 1) diff --git a/src/AggregateFunctions/AggregateFunctionTopK.cpp b/src/AggregateFunctions/AggregateFunctionTopK.cpp index 7f2da260c2df..344ab340d62b 100644 --- a/src/AggregateFunctions/AggregateFunctionTopK.cpp +++ b/src/AggregateFunctions/AggregateFunctionTopK.cpp @@ -100,7 +100,8 @@ AggregateFunctionPtr createAggregateFunctionTopK(const std::string & name, const threshold = k; } - AggregateFunctionPtr res(createWithNumericType(*argument_types[0], threshold, load_factor, argument_types, params)); + AggregateFunctionPtr res(createWithNumericType( + *argument_types[0], threshold, load_factor, argument_types, params)); if (!res) res = AggregateFunctionPtr(createWithExtraTypes(argument_types[0], threshold, load_factor, params)); diff --git a/src/AggregateFunctions/AggregateFunctionTopK.h b/src/AggregateFunctions/AggregateFunctionTopK.h index dec6baf6ed30..a40ca1d97943 100644 --- a/src/AggregateFunctions/AggregateFunctionTopK.h +++ b/src/AggregateFunctions/AggregateFunctionTopK.h @@ -52,7 +52,7 @@ class AggregateFunctionTopK DataTypePtr getReturnType() const override { - return std::make_shared(std::make_shared>()); + return std::make_shared(this->argument_types[0]); } void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override diff --git a/src/AggregateFunctions/parseAggregateFunctionParameters.cpp b/src/AggregateFunctions/parseAggregateFunctionParameters.cpp index bcb73f1e9d9b..3be2bf88e1f9 100644 --- a/src/AggregateFunctions/parseAggregateFunctionParameters.cpp +++ b/src/AggregateFunctions/parseAggregateFunctionParameters.cpp @@ -26,8 +26,12 @@ Array getAggregateFunctionParametersArray(const ASTPtr & expression_list, const const auto * literal = parameters[i]->as(); if (!literal) { - throw Exception("Parameters to aggregate functions must be literals" + (error_context.empty() ? "" : " (in " + error_context +")"), - ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS); + throw Exception( + ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS, + "Parameters to aggregate functions must be literals. " + "Got parameter '{}'{}", + parameters[i]->formatForErrorMessage(), + (error_context.empty() ? "" : " (in " + error_context +")")); } params_row[i] = literal->value; diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 2c98e873cb67..e489a82b5c08 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -24,7 +24,6 @@ namespace DB namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; extern const int NOT_IMPLEMENTED; extern const int BAD_ARGUMENTS; extern const int PARAMETER_OUT_OF_BOUND; @@ -36,8 +35,18 @@ namespace ErrorCodes ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column) : data(std::move(nested_column)), offsets(std::move(offsets_column)) { - if (!typeid_cast(offsets.get())) - throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::ILLEGAL_COLUMN); + const ColumnOffsets * offsets_concrete = typeid_cast(offsets.get()); + + if (!offsets_concrete) + throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::LOGICAL_ERROR); + + size_t size = offsets_concrete->size(); + if (size != 0 && nested_column) + { + /// This will also prevent possible overflow in offset. + if (nested_column->size() != offsets_concrete->getData()[size - 1]) + throw Exception("offsets_column has data inconsistent with nested_column", ErrorCodes::LOGICAL_ERROR); + } /** NOTE * Arrays with constant value are possible and used in implementation of higher order functions (see FunctionReplicate). @@ -49,7 +58,7 @@ ColumnArray::ColumnArray(MutableColumnPtr && nested_column) : data(std::move(nested_column)) { if (!data->empty()) - throw Exception("Not empty data passed to ColumnArray, but no offsets passed", ErrorCodes::ILLEGAL_COLUMN); + throw Exception("Not empty data passed to ColumnArray, but no offsets passed", ErrorCodes::LOGICAL_ERROR); offsets = ColumnOffsets::create(); } diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 575a9f4ac833..f8801ff5ff9a 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -464,7 +464,6 @@ void ColumnNullable::applyNullMap(const ColumnNullable & other) applyNullMap(other.getNullMapColumn()); } - void ColumnNullable::checkConsistency() const { if (null_map->size() != getNestedColumn().size()) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index a1c86953dc98..c2fab1b60cb9 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -309,6 +309,31 @@ ColumnPtr ColumnVector::filter(const IColumn::Filter & filt, ssize_t result_s return res; } +template +void ColumnVector::applyZeroMap(const IColumn::Filter & filt, bool inverted) +{ + size_t size = data.size(); + if (size != filt.size()) + throw Exception("Size of filter doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); + + const UInt8 * filt_pos = filt.data(); + const UInt8 * filt_end = filt_pos + size; + T * data_pos = data.data(); + + if (inverted) + { + for (; filt_pos < filt_end; ++filt_pos, ++data_pos) + if (!*filt_pos) + *data_pos = 0; + } + else + { + for (; filt_pos < filt_end; ++filt_pos, ++data_pos) + if (*filt_pos) + *data_pos = 0; + } +} + template ColumnPtr ColumnVector::permute(const IColumn::Permutation & perm, size_t limit) const { diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 00c98ed9c88d..9bd4e7b12bc3 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -269,6 +269,9 @@ class ColumnVector final : public COWHelper> return typeid(rhs) == typeid(ColumnVector); } + /// Replace elements that match the filter with zeroes. If inverted replaces not matched elements. + void applyZeroMap(const IColumn::Filter & filt, bool inverted = false); + /** More efficient methods of manipulation - to manipulate with data directly. */ Container & getData() { diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index dfb39f1624e9..36b4a11eb582 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -489,6 +489,8 @@ namespace ErrorCodes extern const int INCORRECT_ACCESS_ENTITY_DEFINITION = 515; extern const int AUTHENTICATION_FAILED = 516; extern const int CANNOT_ASSIGN_ALTER = 517; + extern const int CANNOT_COMMIT_OFFSET = 518; + extern const int CANNOT_FSTAT = 531; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 5b2023aed06a..e39d7bd3151d 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -8,6 +8,8 @@ #include +#include + namespace Poco { class Logger; } @@ -24,8 +26,14 @@ class Exception : public Poco::Exception Exception() = default; Exception(const std::string & msg, int code); - enum CreateFromPocoTag { CreateFromPoco }; - enum CreateFromSTDTag { CreateFromSTD }; + // Format message with fmt::format, like the logging functions. + template + Exception(int code, Fmt&&... fmt) + : Exception(fmt::format(std::forward(fmt)...), code) + {} + + struct CreateFromPocoTag {}; + struct CreateFromSTDTag {}; Exception(CreateFromPocoTag, const Poco::Exception & exc); Exception(CreateFromSTDTag, const std::exception & exc); diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 3753e4430c4e..4892bece65d4 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -120,7 +120,6 @@ class MemoryTracker DB::SimpleActionBlocker blocker; }; - /// Convenience methods, that use current thread's memory_tracker if it is available. namespace CurrentMemoryTracker { diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp index ce660c1e2ab3..f1799b3691a9 100644 --- a/src/Common/OptimizedRegularExpression.cpp +++ b/src/Common/OptimizedRegularExpression.cpp @@ -38,6 +38,7 @@ void OptimizedRegularExpressionImpl::analyze( required_substring_is_prefix = false; required_substring.clear(); bool has_alternative_on_depth_0 = false; + bool has_case_insensitive_flag = false; /// Substring with a position. using Substring = std::pair; @@ -65,7 +66,17 @@ void OptimizedRegularExpressionImpl::analyze( switch (*pos) { - case '|': case '(': case ')': case '^': case '$': case '.': case '[': case '?': case '*': case '+': case '{': + case '|': + case '(': + case ')': + case '^': + case '$': + case '.': + case '[': + case '?': + case '*': + case '+': + case '{': if (depth == 0 && !in_curly_braces && !in_square_braces) { if (last_substring->first.empty()) @@ -110,6 +121,28 @@ void OptimizedRegularExpressionImpl::analyze( trivial_substrings.resize(trivial_substrings.size() + 1); last_substring = &trivial_substrings.back(); } + + /// Check for case-insensitive flag. + if (pos + 1 < end && pos[1] == '?') + { + for (size_t offset = 2; pos + offset < end; ++offset) + { + if (pos[offset] == '-' /// it means flag negation + /// various possible flags, actually only imsU are supported by re2 + || (pos[offset] >= 'a' && pos[offset] <= 'z') + || (pos[offset] >= 'A' && pos[offset] <= 'Z')) + { + if (pos[offset] == 'i') + { + /// Actually it can be negated case-insensitive flag. But we don't care. + has_case_insensitive_flag = true; + break; + } + } + else + break; + } + } } ++pos; break; @@ -209,7 +242,7 @@ void OptimizedRegularExpressionImpl::analyze( if (!is_trivial) { - if (!has_alternative_on_depth_0) + if (!has_alternative_on_depth_0 && !has_case_insensitive_flag) { /// We choose the non-alternative substring of the maximum length for first search. diff --git a/src/Common/ProfilingScopedRWLock.h b/src/Common/ProfilingScopedRWLock.h index c537bfa85865..f5753f9ce46c 100644 --- a/src/Common/ProfilingScopedRWLock.h +++ b/src/Common/ProfilingScopedRWLock.h @@ -8,16 +8,11 @@ namespace DB { -class ProfilingScopedWriteUnlocker; - class ProfilingScopedWriteRWLock { public: - friend class ProfilingScopedWriteUnlocker; - ProfilingScopedWriteRWLock(std::shared_mutex & rwl_, ProfileEvents::Event event_) : - watch(), - event(event_), + ProfilingScopedWriteRWLock(std::shared_mutex & rwl_, ProfileEvents::Event event) : scoped_write_lock(rwl_) { ProfileEvents::increment(event, watch.elapsed()); @@ -25,38 +20,14 @@ class ProfilingScopedWriteRWLock private: Stopwatch watch; - ProfileEvents::Event event; std::unique_lock scoped_write_lock; }; -/// Inversed RAII -/// Used to unlock current writelock for various purposes. -class ProfilingScopedWriteUnlocker -{ -public: - ProfilingScopedWriteUnlocker() = delete; - - ProfilingScopedWriteUnlocker(ProfilingScopedWriteRWLock & parent_lock_) : parent_lock(parent_lock_) - { - parent_lock.scoped_write_lock.unlock(); - } - - ~ProfilingScopedWriteUnlocker() - { - Stopwatch watch; - parent_lock.scoped_write_lock.lock(); - ProfileEvents::increment(parent_lock.event, watch.elapsed()); - } - -private: - ProfilingScopedWriteRWLock & parent_lock; -}; class ProfilingScopedReadRWLock { public: ProfilingScopedReadRWLock(std::shared_mutex & rwl, ProfileEvents::Event event) : - watch(), scoped_read_lock(rwl) { ProfileEvents::increment(event, watch.elapsed()); diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 2b57ee5dbb63..a2ac45cbae6b 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -234,14 +234,6 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ --scheduled_jobs; } - DB::tryLogCurrentException("ThreadPool", - std::string("Exception in ThreadPool(") + - "max_threads: " + std::to_string(max_threads) - + ", max_free_threads: " + std::to_string(max_free_threads) - + ", queue_size: " + std::to_string(queue_size) - + ", shutdown_on_exception: " + std::to_string(shutdown_on_exception) - + ")."); - job_finished.notify_all(); new_job_or_shutdown.notify_all(); return; diff --git a/src/Common/XDBCBridgeHelper.h b/src/Common/XDBCBridgeHelper.h index 613d1bed8d7e..34c70e226923 100644 --- a/src/Common/XDBCBridgeHelper.h +++ b/src/Common/XDBCBridgeHelper.h @@ -105,7 +105,8 @@ class XDBCBridgeHelper : public IXDBCBridgeHelper uri.setPath(IDENTIFIER_QUOTE_HANDLER); uri.addQueryParameter("connection_string", getConnectionString()); - ReadWriteBufferFromHTTP buf(uri, Poco::Net::HTTPRequest::HTTP_POST, nullptr); + ReadWriteBufferFromHTTP buf( + uri, Poco::Net::HTTPRequest::HTTP_POST, {}, ConnectionTimeouts::getHTTPTimeouts(context)); std::string character; readStringBinary(character, buf); if (character.length() > 1) @@ -198,7 +199,8 @@ class XDBCBridgeHelper : public IXDBCBridgeHelper { try { - ReadWriteBufferFromHTTP buf(ping_url, Poco::Net::HTTPRequest::HTTP_GET, nullptr); + ReadWriteBufferFromHTTP buf( + ping_url, Poco::Net::HTTPRequest::HTTP_GET, {}, ConnectionTimeouts::getHTTPTimeouts(context)); return checkString(XDBCBridgeHelper::PING_OK_ANSWER, buf); } catch (...) diff --git a/src/Common/intExp.h b/src/Common/intExp.h index 0212eb4c084d..a6889c96047e 100644 --- a/src/Common/intExp.h +++ b/src/Common/intExp.h @@ -20,14 +20,14 @@ inline NO_SANITIZE_UNDEFINED uint64_t intExp2(int x) return 1ULL << x; } -inline uint64_t intExp10(int x) +constexpr inline uint64_t intExp10(int x) { if (x < 0) return 0; if (x > 19) return std::numeric_limits::max(); - static const uint64_t table[20] = + constexpr uint64_t table[20] = { 1ULL, 10ULL, 100ULL, 1000ULL, 10000ULL, 100000ULL, @@ -44,9 +44,10 @@ inline uint64_t intExp10(int x) namespace common { -inline int exp10_i32(int x) +constexpr inline int exp10_i32(int x) { - static const int values[] = { + constexpr int values[] = + { 1, 10, 100, @@ -61,76 +62,88 @@ inline int exp10_i32(int x) return values[x]; } -inline int64_t exp10_i64(int x) +constexpr inline int64_t exp10_i64(int x) { - static const int64_t values[] = { - 1ll, - 10ll, - 100ll, - 1000ll, - 10000ll, - 100000ll, - 1000000ll, - 10000000ll, - 100000000ll, - 1000000000ll, - 10000000000ll, - 100000000000ll, - 1000000000000ll, - 10000000000000ll, - 100000000000000ll, - 1000000000000000ll, - 10000000000000000ll, - 100000000000000000ll, - 1000000000000000000ll + constexpr int64_t values[] = + { + 1LL, + 10LL, + 100LL, + 1000LL, + 10000LL, + 100000LL, + 1000000LL, + 10000000LL, + 100000000LL, + 1000000000LL, + 10000000000LL, + 100000000000LL, + 1000000000000LL, + 10000000000000LL, + 100000000000000LL, + 1000000000000000LL, + 10000000000000000LL, + 100000000000000000LL, + 1000000000000000000LL }; return values[x]; } -inline __int128 exp10_i128(int x) +constexpr inline __int128 exp10_i128(int x) { - static const __int128 values[] = { - static_cast<__int128>(1ll), - static_cast<__int128>(10ll), - static_cast<__int128>(100ll), - static_cast<__int128>(1000ll), - static_cast<__int128>(10000ll), - static_cast<__int128>(100000ll), - static_cast<__int128>(1000000ll), - static_cast<__int128>(10000000ll), - static_cast<__int128>(100000000ll), - static_cast<__int128>(1000000000ll), - static_cast<__int128>(10000000000ll), - static_cast<__int128>(100000000000ll), - static_cast<__int128>(1000000000000ll), - static_cast<__int128>(10000000000000ll), - static_cast<__int128>(100000000000000ll), - static_cast<__int128>(1000000000000000ll), - static_cast<__int128>(10000000000000000ll), - static_cast<__int128>(100000000000000000ll), - static_cast<__int128>(1000000000000000000ll), - static_cast<__int128>(1000000000000000000ll) * 10ll, - static_cast<__int128>(1000000000000000000ll) * 100ll, - static_cast<__int128>(1000000000000000000ll) * 1000ll, - static_cast<__int128>(1000000000000000000ll) * 10000ll, - static_cast<__int128>(1000000000000000000ll) * 100000ll, - static_cast<__int128>(1000000000000000000ll) * 1000000ll, - static_cast<__int128>(1000000000000000000ll) * 10000000ll, - static_cast<__int128>(1000000000000000000ll) * 100000000ll, - static_cast<__int128>(1000000000000000000ll) * 1000000000ll, - static_cast<__int128>(1000000000000000000ll) * 10000000000ll, - static_cast<__int128>(1000000000000000000ll) * 100000000000ll, - static_cast<__int128>(1000000000000000000ll) * 1000000000000ll, - static_cast<__int128>(1000000000000000000ll) * 10000000000000ll, - static_cast<__int128>(1000000000000000000ll) * 100000000000000ll, - static_cast<__int128>(1000000000000000000ll) * 1000000000000000ll, - static_cast<__int128>(1000000000000000000ll) * 10000000000000000ll, - static_cast<__int128>(1000000000000000000ll) * 100000000000000000ll, - static_cast<__int128>(1000000000000000000ll) * 100000000000000000ll * 10ll, - static_cast<__int128>(1000000000000000000ll) * 100000000000000000ll * 100ll, - static_cast<__int128>(1000000000000000000ll) * 100000000000000000ll * 1000ll + constexpr __int128 values[] = + { + static_cast<__int128>(1LL), + static_cast<__int128>(10LL), + static_cast<__int128>(100LL), + static_cast<__int128>(1000LL), + static_cast<__int128>(10000LL), + static_cast<__int128>(100000LL), + static_cast<__int128>(1000000LL), + static_cast<__int128>(10000000LL), + static_cast<__int128>(100000000LL), + static_cast<__int128>(1000000000LL), + static_cast<__int128>(10000000000LL), + static_cast<__int128>(100000000000LL), + static_cast<__int128>(1000000000000LL), + static_cast<__int128>(10000000000000LL), + static_cast<__int128>(100000000000000LL), + static_cast<__int128>(1000000000000000LL), + static_cast<__int128>(10000000000000000LL), + static_cast<__int128>(100000000000000000LL), + static_cast<__int128>(1000000000000000000LL), + static_cast<__int128>(1000000000000000000LL) * 10LL, + static_cast<__int128>(1000000000000000000LL) * 100LL, + static_cast<__int128>(1000000000000000000LL) * 1000LL, + static_cast<__int128>(1000000000000000000LL) * 10000LL, + static_cast<__int128>(1000000000000000000LL) * 100000LL, + static_cast<__int128>(1000000000000000000LL) * 1000000LL, + static_cast<__int128>(1000000000000000000LL) * 10000000LL, + static_cast<__int128>(1000000000000000000LL) * 100000000LL, + static_cast<__int128>(1000000000000000000LL) * 1000000000LL, + static_cast<__int128>(1000000000000000000LL) * 10000000000LL, + static_cast<__int128>(1000000000000000000LL) * 100000000000LL, + static_cast<__int128>(1000000000000000000LL) * 1000000000000LL, + static_cast<__int128>(1000000000000000000LL) * 10000000000000LL, + static_cast<__int128>(1000000000000000000LL) * 100000000000000LL, + static_cast<__int128>(1000000000000000000LL) * 1000000000000000LL, + static_cast<__int128>(1000000000000000000LL) * 10000000000000000LL, + static_cast<__int128>(1000000000000000000LL) * 100000000000000000LL, + static_cast<__int128>(1000000000000000000LL) * 100000000000000000LL * 10LL, + static_cast<__int128>(1000000000000000000LL) * 100000000000000000LL * 100LL, + static_cast<__int128>(1000000000000000000LL) * 100000000000000000LL * 1000LL }; return values[x]; } } + +/// intExp10 returning the type T. +template +constexpr inline T intExp10OfSize(int x) +{ + if constexpr (sizeof(T) <= 8) + return intExp10(x); + else + return common::exp10_i128(x); +} diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp index 2369e2ca2327..6c7cf92a41df 100644 --- a/src/Compression/CompressionCodecDelta.cpp +++ b/src/Compression/CompressionCodecDelta.cpp @@ -166,6 +166,9 @@ void registerCodecDelta(CompressionCodecFactory & factory) const auto children = arguments->children; const auto * literal = children[0]->as(); + if (!literal) + throw Exception("Delta codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + size_t user_bytes_size = literal->value.safeGet(); if (user_bytes_size != 1 && user_bytes_size != 2 && user_bytes_size != 4 && user_bytes_size != 8) throw Exception("Delta value for delta codec can be 1, 2, 4 or 8, given " + toString(user_bytes_size), ErrorCodes::ILLEGAL_CODEC_PARAMETER); diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index 95fa51d1bd09..845e21e62d8a 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -166,6 +166,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) throw Exception("Cannot compress, data size " + toString(source_size) + " is not aligned to " + toString(sizeof(ValueType)), ErrorCodes::CANNOT_COMPRESS); const char * source_end = source + source_size; + const char * dest_start = dest; const UInt32 items_count = source_size / sizeof(ValueType); unalignedStore(dest, items_count); @@ -229,7 +230,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) writer.flush(); - return sizeof(items_count) + sizeof(prev_value) + sizeof(prev_delta) + writer.count() / 8; + return (dest - dest_start) + (writer.count() + 7) / 8; } template diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index ab1d9614d993..30be656e1c04 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -90,6 +90,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest, if (source_size % sizeof(T) != 0) throw Exception("Cannot compress, data size " + toString(source_size) + " is not aligned to " + toString(sizeof(T)), ErrorCodes::CANNOT_COMPRESS); const char * source_end = source + source_size; + const char * dest_start = dest; const char * dest_end = dest + dest_size; const UInt32 items_count = source_size / sizeof(T); @@ -145,7 +146,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest, writer.flush(); - return sizeof(items_count) + sizeof(prev_value) + writer.count() / 8; + return (dest - dest_start) + (writer.count() + 7) / 8; } template diff --git a/src/Compression/CompressionCodecLZ4.cpp b/src/Compression/CompressionCodecLZ4.cpp index cf8f8e976ea6..32c3958e65e2 100644 --- a/src/Compression/CompressionCodecLZ4.cpp +++ b/src/Compression/CompressionCodecLZ4.cpp @@ -19,6 +19,7 @@ namespace ErrorCodes { extern const int CANNOT_COMPRESS; extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; +extern const int ILLEGAL_CODEC_PARAMETER; } @@ -84,6 +85,9 @@ void registerCodecLZ4HC(CompressionCodecFactory & factory) const auto children = arguments->children; const auto * literal = children[0]->as(); + if (!literal) + throw Exception("LZ4HC codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + level = literal->value.safeGet(); } diff --git a/src/Compression/CompressionCodecZSTD.cpp b/src/Compression/CompressionCodecZSTD.cpp index e5f950347e1f..4fff0de78d0a 100644 --- a/src/Compression/CompressionCodecZSTD.cpp +++ b/src/Compression/CompressionCodecZSTD.cpp @@ -74,6 +74,9 @@ void registerCodecZSTD(CompressionCodecFactory & factory) const auto children = arguments->children; const auto * literal = children[0]->as(); + if (!literal) + throw Exception("ZSTD codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + level = literal->value.safeGet(); if (level > ZSTD_maxCLevel()) throw Exception("ZSTD codec can't have level more that " + toString(ZSTD_maxCLevel()) + ", given " + toString(level), ErrorCodes::ILLEGAL_CODEC_PARAMETER); diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a22afd9edcd9..7950f4eeda89 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -294,7 +294,7 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_execution_speed, 0, "Maximum number of execution rows per second.", 0) \ M(SettingUInt64, min_execution_speed_bytes, 0, "Minimum number of execution bytes per second.", 0) \ M(SettingUInt64, max_execution_speed_bytes, 0, "Maximum number of execution bytes per second.", 0) \ - M(SettingSeconds, timeout_before_checking_execution_speed, 0, "Check that the speed is not too low after the specified time has elapsed.", 0) \ + M(SettingSeconds, timeout_before_checking_execution_speed, 10, "Check that the speed is not too low after the specified time has elapsed.", 0) \ \ M(SettingUInt64, max_columns_to_read, 0, "", 0) \ M(SettingUInt64, max_temporary_columns, 0, "", 0) \ diff --git a/src/Core/SettingsCollection.cpp b/src/Core/SettingsCollection.cpp index f2ac331b0283..5ae885272b6a 100644 --- a/src/Core/SettingsCollection.cpp +++ b/src/Core/SettingsCollection.cpp @@ -246,7 +246,7 @@ void SettingTimespan::set(const Field & x) if (x.getType() == Field::Types::String) set(get(x)); else - set(safeGet(x)); + set(applyVisitor(FieldVisitorConvertToNumber(), x)); } template diff --git a/src/DataStreams/ExecutionSpeedLimits.cpp b/src/DataStreams/ExecutionSpeedLimits.cpp index ca79138655a7..3eab00ee8cc8 100644 --- a/src/DataStreams/ExecutionSpeedLimits.cpp +++ b/src/DataStreams/ExecutionSpeedLimits.cpp @@ -30,7 +30,8 @@ static void limitProgressingSpeed(size_t total_progress_size, size_t max_speed_i { UInt64 sleep_microseconds = desired_microseconds - total_elapsed_microseconds; - /// Never sleep more than one second (it should be enough to limit speed for a reasonable amount, and otherwise it's too easy to make query hang). + /// Never sleep more than one second (it should be enough to limit speed for a reasonable amount, + /// and otherwise it's too easy to make query hang). sleep_microseconds = std::min(UInt64(1000000), sleep_microseconds); sleepForMicroseconds(sleep_microseconds); @@ -45,14 +46,14 @@ void ExecutionSpeedLimits::throttle( { if ((min_execution_rps != 0 || max_execution_rps != 0 || min_execution_bps != 0 || max_execution_bps != 0 - || (total_rows_to_read != 0 && timeout_before_checking_execution_speed != 0)) && - (static_cast(total_elapsed_microseconds) > timeout_before_checking_execution_speed.totalMicroseconds())) + || (total_rows_to_read != 0 && timeout_before_checking_execution_speed != 0)) + && (static_cast(total_elapsed_microseconds) > timeout_before_checking_execution_speed.totalMicroseconds())) { /// Do not count sleeps in throttlers UInt64 throttler_sleep_microseconds = CurrentThread::getProfileEvents()[ProfileEvents::ThrottlerSleepMicroseconds]; double elapsed_seconds = 0; - if (throttler_sleep_microseconds > total_elapsed_microseconds) + if (total_elapsed_microseconds > throttler_sleep_microseconds) elapsed_seconds = static_cast(total_elapsed_microseconds - throttler_sleep_microseconds) / 1000000.0; if (elapsed_seconds > 0) diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index f30b570ac4f9..05f157f72c31 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -139,7 +139,7 @@ void PushingToViewsBlockOutputStream::write(const Block & block) const Settings & settings = context.getSettingsRef(); if (settings.parallel_view_processing && views.size() > 1) { - // Push to views concurrently if enabled, and more than one view is attached + // Push to views concurrently if enabled and more than one view is attached ThreadPool pool(std::min(size_t(settings.max_threads), views.size())); for (size_t view_num = 0; view_num < views.size(); ++view_num) { @@ -194,6 +194,45 @@ void PushingToViewsBlockOutputStream::writeSuffix() std::exception_ptr first_exception; + const Settings & settings = context.getSettingsRef(); + bool parallel_processing = false; + + /// Run writeSuffix() for views in separate thread pool. + /// In could have been done in PushingToViewsBlockOutputStream::process, however + /// it is not good if insert into main table fail but into view succeed. + if (settings.parallel_view_processing && views.size() > 1) + { + parallel_processing = true; + + // Push to views concurrently if enabled and more than one view is attached + ThreadPool pool(std::min(size_t(settings.max_threads), views.size())); + auto thread_group = CurrentThread::getGroup(); + + for (auto & view : views) + { + if (view.exception) + continue; + + pool.scheduleOrThrowOnError([thread_group, &view] + { + setThreadName("PushingToViews"); + if (thread_group) + CurrentThread::attachToIfDetached(thread_group); + + try + { + view.out->writeSuffix(); + } + catch (...) + { + view.exception = std::current_exception(); + } + }); + } + // Wait for concurrent view processing + pool.wait(); + } + for (auto & view : views) { if (view.exception) @@ -204,6 +243,9 @@ void PushingToViewsBlockOutputStream::writeSuffix() continue; } + if (parallel_processing) + continue; + try { view.out->writeSuffix(); diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index 3fb380eac0f9..e94d761dc874 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -362,8 +362,11 @@ static DataTypePtr create(const ASTPtr & arguments) { const auto * literal = parameters[i]->as(); if (!literal) - throw Exception("Parameters to aggregate functions must be literals", - ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS); + throw Exception( + ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS, + "Parameters to aggregate functions must be literals. " + "Got parameter '{}' for function '{}'", + parameters[i]->formatForErrorMessage(), function_name); params_row[i] = literal->value; } diff --git a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp index 3072e4a40c13..058d6b79f861 100644 --- a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp +++ b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp @@ -82,8 +82,11 @@ static std::pair create(const ASTPtr & argum { const ASTLiteral * lit = parameters[i]->as(); if (!lit) - throw Exception("Parameters to aggregate functions must be literals", - ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS); + throw Exception( + ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS, + "Parameters to aggregate functions must be literals. " + "Got parameter '{}' for function '{}'", + parameters[i]->formatForErrorMessage(), function_name); params_row[i] = lit->value; } diff --git a/src/DataTypes/DataTypeDateTime64.cpp b/src/DataTypes/DataTypeDateTime64.cpp index d790dd60ce41..7c8bdefed5c8 100644 --- a/src/DataTypes/DataTypeDateTime64.cpp +++ b/src/DataTypes/DataTypeDateTime64.cpp @@ -20,19 +20,34 @@ #include #include + namespace DB { +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; +} + +static constexpr UInt32 max_scale = 9; + DataTypeDateTime64::DataTypeDateTime64(UInt32 scale_, const std::string & time_zone_name) : DataTypeDecimalBase(DecimalUtils::maxPrecision(), scale_), TimezoneMixin(time_zone_name) { + if (scale > max_scale) + throw Exception("Scale " + std::to_string(scale) + " is too large for DateTime64. Maximum is up to nanoseconds (9).", + ErrorCodes::ARGUMENT_OUT_OF_BOUND); } DataTypeDateTime64::DataTypeDateTime64(UInt32 scale_, const TimezoneMixin & time_zone_info) - : DataTypeDecimalBase(DecimalUtils::maxPrecision() - scale_, scale_), + : DataTypeDecimalBase(DecimalUtils::maxPrecision(), scale_), TimezoneMixin(time_zone_info) -{} +{ + if (scale > max_scale) + throw Exception("Scale " + std::to_string(scale) + " is too large for DateTime64. Maximum is up to nanoseconds (9).", + ErrorCodes::ARGUMENT_OUT_OF_BOUND); +} std::string DataTypeDateTime64::doGetName() const { diff --git a/src/DataTypes/DataTypeDecimalBase.h b/src/DataTypes/DataTypeDecimalBase.h index bb5b0ca4ee8a..676438800a2a 100644 --- a/src/DataTypes/DataTypeDecimalBase.h +++ b/src/DataTypes/DataTypeDecimalBase.h @@ -73,7 +73,7 @@ class DataTypeDecimalBase : public DataTypeWithSimpleSerialization { if (unlikely(precision < 1 || precision > maxPrecision())) throw Exception("Precision " + std::to_string(precision) + " is out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND); - if (unlikely(scale < 0 || static_cast(scale) > maxPrecision())) + if (unlikely(scale > maxPrecision())) throw Exception("Scale " + std::to_string(scale) + " is out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND); } diff --git a/src/DataTypes/DataTypeNullable.h b/src/DataTypes/DataTypeNullable.h index 1766b399c2a8..22d403da6c43 100644 --- a/src/DataTypes/DataTypeNullable.h +++ b/src/DataTypes/DataTypeNullable.h @@ -90,7 +90,7 @@ class DataTypeNullable final : public IDataType bool canBeComparedWithCollation() const override { return nested_data_type->canBeComparedWithCollation(); } bool canBeUsedAsVersion() const override { return false; } bool isSummable() const override { return nested_data_type->isSummable(); } - bool canBeUsedInBooleanContext() const override { return nested_data_type->canBeUsedInBooleanContext(); } + bool canBeUsedInBooleanContext() const override { return nested_data_type->canBeUsedInBooleanContext() || onlyNull(); } bool haveMaximumSizeOfValue() const override { return nested_data_type->haveMaximumSizeOfValue(); } size_t getMaximumSizeOfValueInMemory() const override { return 1 + nested_data_type->getMaximumSizeOfValueInMemory(); } bool isNullable() const override { return true; } diff --git a/src/DataTypes/getLeastSupertype.cpp b/src/DataTypes/getLeastSupertype.cpp index cfe88987c2d1..ef8e8aff2a04 100644 --- a/src/DataTypes/getLeastSupertype.cpp +++ b/src/DataTypes/getLeastSupertype.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -160,6 +161,39 @@ DataTypePtr getLeastSupertype(const DataTypes & types) } } + /// For LowCardinality. This is above Nullable, because LowCardinality can contain Nullable but cannot be inside Nullable. + { + bool have_low_cardinality = false; + bool have_not_low_cardinality = false; + + DataTypes nested_types; + nested_types.reserve(types.size()); + + for (const auto & type : types) + { + if (const DataTypeLowCardinality * type_low_cardinality = typeid_cast(type.get())) + { + have_low_cardinality = true; + nested_types.emplace_back(type_low_cardinality->getDictionaryType()); + } + else + { + have_not_low_cardinality = true; + nested_types.emplace_back(type); + } + } + + /// All LowCardinality gives LowCardinality. + /// LowCardinality with high cardinality gives high cardinality. + if (have_low_cardinality) + { + if (have_not_low_cardinality) + return getLeastSupertype(nested_types); + else + return std::make_shared(getLeastSupertype(nested_types)); + } + } + /// For Nullable { bool have_nullable = false; @@ -208,7 +242,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types) } } - /// For Date and DateTime, the common type is DateTime. No other types are compatible. + /// For Date and DateTime/DateTime64, the common type is DateTime/DateTime64. No other types are compatible. { UInt32 have_date = type_ids.count(TypeIndex::Date); UInt32 have_datetime = type_ids.count(TypeIndex::DateTime); @@ -218,40 +252,25 @@ DataTypePtr getLeastSupertype(const DataTypes & types) { bool all_date_or_datetime = type_ids.size() == (have_date + have_datetime + have_datetime64); if (!all_date_or_datetime) - throw Exception(getExceptionMessagePrefix(types) + " because some of them are Date/DateTime and some of them are not", ErrorCodes::NO_COMMON_TYPE); + throw Exception(getExceptionMessagePrefix(types) + " because some of them are Date/DateTime/DateTime64 and some of them are not", + ErrorCodes::NO_COMMON_TYPE); if (have_datetime64 == 0) - { return std::make_shared(); - } - // When DateTime64 involved, make sure that supertype has whole-part precision - // big enough to hold max whole-value of any type from `types`. - // That would sacrifice scale when comparing DateTime64 of different scales. + UInt8 max_scale = 0; - UInt32 max_datetime64_whole_precision = 0; for (const auto & t : types) { if (auto dt64 = typeid_cast(t.get())) { - const auto whole_precision = dt64->getPrecision() - dt64->getScale(); - max_datetime64_whole_precision = std::max(whole_precision, max_datetime64_whole_precision); + const auto scale = dt64->getScale(); + if (scale > max_scale) + max_scale = scale; } } - UInt32 least_decimal_precision = 0; - if (have_datetime) - { - least_decimal_precision = leastDecimalPrecisionFor(TypeIndex::UInt32); - } - else if (have_date) - { - least_decimal_precision = leastDecimalPrecisionFor(TypeIndex::UInt16); - } - max_datetime64_whole_precision = std::max(least_decimal_precision, max_datetime64_whole_precision); - - const UInt32 scale = DataTypeDateTime64::maxPrecision() - max_datetime64_whole_precision; - return std::make_shared(scale); + return std::make_shared(max_scale); } } diff --git a/src/DataTypes/tests/gtest_data_type_get_common_type.cpp b/src/DataTypes/tests/gtest_data_type_get_common_type.cpp index f334ab222a4a..47ad4d29a36a 100644 --- a/src/DataTypes/tests/gtest_data_type_get_common_type.cpp +++ b/src/DataTypes/tests/gtest_data_type_get_common_type.cpp @@ -92,7 +92,7 @@ TEST_P(LeastSuperTypeTest, getLeastSupertype) class MostSubtypeTest : public TypeTest {}; -TEST_P(MostSubtypeTest, getLeastSupertype) +TEST_P(MostSubtypeTest, getMostSubtype) { if (this->expected_type) { @@ -130,9 +130,7 @@ INSTANTIATE_TEST_SUITE_P(data_type, {"Date DateTime64(3)", "DateTime64(3)"}, {"DateTime DateTime64(3)", "DateTime64(3)"}, {"DateTime DateTime64(0)", "DateTime64(0)"}, - {"DateTime64(9) DateTime64(3)", "DateTime64(3)"}, - {"DateTime DateTime64(12)", "DateTime64(8)"}, - {"Date DateTime64(15)", "DateTime64(13)"}, + {"DateTime64(9) DateTime64(3)", "DateTime64(9)"}, {"String FixedString(32) FixedString(8)", "String"}, diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 902933c58e1b..c09f86678f17 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -236,7 +236,7 @@ void DatabaseOnDisk::renameTable( catch (const Poco::Exception & e) { /// Better diagnostics. - throw Exception{Exception::CreateFromPoco, e}; + throw Exception{Exception::CreateFromPocoTag{}, e}; } /// NOTE Non-atomic. @@ -308,6 +308,27 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & object_n void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const IteratingFunction & iterating_function) const { + auto process_tmp_drop_metadata_file = [&](const String & file_name) + { + assert(getEngineName() != "Atomic"); + static const char * tmp_drop_ext = ".sql.tmp_drop"; + const std::string object_name = file_name.substr(0, file_name.size() - strlen(tmp_drop_ext)); + if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists()) + { + Poco::File(getMetadataPath() + file_name).renameTo(getMetadataPath() + object_name + ".sql"); + LOG_WARNING(log, "Object " << backQuote(object_name) << " was not dropped previously and will be restored"); + iterating_function(object_name + ".sql"); + } + else + { + LOG_INFO(log, "Removing file " << getMetadataPath() << file_name); + Poco::File(getMetadataPath() + file_name).remove(); + } + }; + + /// Metadata files to load: name and flag for .tmp_drop files + std::set> metadata_files; + Poco::DirectoryIterator dir_end; for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) { @@ -323,19 +344,7 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati static const char * tmp_drop_ext = ".sql.tmp_drop"; if (endsWith(dir_it.name(), tmp_drop_ext)) { - const std::string object_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext)); - if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists()) - { - /// TODO maybe complete table drop and remove all table data (including data on other volumes and metadata in ZK) - Poco::File(dir_it->path()).renameTo(getMetadataPath() + object_name + ".sql"); - LOG_WARNING(log, "Object " << backQuote(object_name) << " was not dropped previously and will be restored"); - iterating_function(object_name + ".sql"); - } - else - { - LOG_INFO(log, "Removing file " << dir_it->path()); - Poco::File(dir_it->path()).remove(); - } + metadata_files.emplace(dir_it.name(), false); continue; } @@ -350,12 +359,27 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati /// The required files have names like `table_name.sql` if (endsWith(dir_it.name(), ".sql")) { - iterating_function(dir_it.name()); + /// The required files have names like `table_name.sql` + metadata_files.emplace(dir_it.name(), true); } else throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(), ErrorCodes::INCORRECT_FILE_NAME); } + + /// Read and parse metadata in parallel + ThreadPool pool(SettingMaxThreads().getAutoValue()); + for (const auto & file : metadata_files) + { + pool.scheduleOrThrowOnError([&]() + { + if (file.second) + iterating_function(file.first); + else + process_tmp_drop_metadata_file(file.first); + }); + } + pool.wait(); } ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const Context & context, const String & metadata_file_path, bool throw_on_error /*= true*/, bool remove_empty /*= false*/) const diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index e8fcf19641d4..4d3de5e08283 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -118,10 +118,11 @@ void DatabaseOrdinary::loadStoredObjects( * which does not correspond to order tables creation and does not correspond to order of their location on disk. */ using FileNames = std::map; + std::mutex file_names_mutex; FileNames file_names; size_t total_dictionaries = 0; - iterateMetadataFiles(context, [&context, &file_names, &total_dictionaries, this](const String & file_name) + iterateMetadataFiles(context, [&context, &file_names, &total_dictionaries, &file_names_mutex, this](const String & file_name) { String full_path = getMetadataPath() + file_name; try @@ -130,6 +131,7 @@ void DatabaseOrdinary::loadStoredObjects( if (ast) { auto * create_query = ast->as(); + std::lock_guard lock{file_names_mutex}; file_names[file_name] = ast; total_dictionaries += create_query->is_dictionary; } diff --git a/src/Dictionaries/CacheDictionary.cpp b/src/Dictionaries/CacheDictionary.cpp index 2294c99c111b..b58a74ca737d 100644 --- a/src/Dictionaries/CacheDictionary.cpp +++ b/src/Dictionaries/CacheDictionary.cpp @@ -46,6 +46,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int UNSUPPORTED_METHOD; extern const int TOO_SMALL_BUFFER_SIZE; + extern const int TIMEOUT_EXCEEDED; } @@ -63,10 +64,12 @@ CacheDictionary::CacheDictionary( const DictionaryStructure & dict_struct_, DictionarySourcePtr source_ptr_, DictionaryLifetime dict_lifetime_, + size_t strict_max_lifetime_seconds_, size_t size_, bool allow_read_expired_keys_, size_t max_update_queue_size_, size_t update_queue_push_timeout_milliseconds_, + size_t query_wait_timeout_milliseconds_, size_t max_threads_for_updates_) : database(database_) , name(name_) @@ -74,9 +77,11 @@ CacheDictionary::CacheDictionary( , dict_struct(dict_struct_) , source_ptr{std::move(source_ptr_)} , dict_lifetime(dict_lifetime_) + , strict_max_lifetime_seconds(strict_max_lifetime_seconds_) , allow_read_expired_keys(allow_read_expired_keys_) , max_update_queue_size(max_update_queue_size_) , update_queue_push_timeout_milliseconds(update_queue_push_timeout_milliseconds_) + , query_wait_timeout_milliseconds(query_wait_timeout_milliseconds_) , max_threads_for_updates(max_threads_for_updates_) , log(&Logger::get("ExternalDictionaries")) , size{roundUpToPowerOfTwoOrZero(std::max(size_, size_t(max_collision_length)))} @@ -86,7 +91,7 @@ CacheDictionary::CacheDictionary( , update_queue(max_update_queue_size_) , update_pool(max_threads_for_updates) { - if (!this->source_ptr->supportsSelectiveLoad()) + if (!source_ptr->supportsSelectiveLoad()) throw Exception{full_name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD}; createAttributes(); @@ -332,6 +337,13 @@ void CacheDictionary::has(const PaddedPODArray & ids, PaddedPODArray { if (find_result.outdated) { + /// Protection of reading very expired keys. + if (now > cells[find_result.cell_idx].strict_max) + { + cache_not_found_ids[id].push_back(row); + continue; + } + cache_expired_ids[id].push_back(row); if (allow_read_expired_keys) @@ -693,6 +705,9 @@ void registerDictionaryCache(DictionaryFactory & factory) const String name = config.getString(config_prefix + ".name"); const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; + const size_t strict_max_lifetime_seconds = + config.getUInt64(layout_prefix + ".cache.strict_max_lifetime_seconds", static_cast(dict_lifetime.max_sec)); + const size_t max_update_queue_size = config.getUInt64(layout_prefix + ".cache.max_update_queue_size", 100000); if (max_update_queue_size == 0) @@ -708,6 +723,9 @@ void registerDictionaryCache(DictionaryFactory & factory) throw Exception{name + ": dictionary of layout 'cache' have too little update_queue_push_timeout", ErrorCodes::BAD_ARGUMENTS}; + const size_t query_wait_timeout_milliseconds = + config.getUInt64(layout_prefix + ".cache.query_wait_timeout_milliseconds", 60000); + const size_t max_threads_for_updates = config.getUInt64(layout_prefix + ".max_threads_for_updates", 4); if (max_threads_for_updates == 0) @@ -715,8 +733,17 @@ void registerDictionaryCache(DictionaryFactory & factory) ErrorCodes::BAD_ARGUMENTS}; return std::make_unique( - database, name, dict_struct, std::move(source_ptr), dict_lifetime, size, - allow_read_expired_keys, max_update_queue_size, update_queue_push_timeout_milliseconds, + database, + name, + dict_struct, + std::move(source_ptr), + dict_lifetime, + strict_max_lifetime_seconds, + size, + allow_read_expired_keys, + max_update_queue_size, + update_queue_push_timeout_milliseconds, + query_wait_timeout_milliseconds, max_threads_for_updates); }; factory.registerLayout("cache", create_layout, false); @@ -782,20 +809,32 @@ void CacheDictionary::updateThreadFunction() void CacheDictionary::waitForCurrentUpdateFinish(UpdateUnitPtr & update_unit_ptr) const { - std::unique_lock lock(update_mutex); - - /* - * We wait here without any timeout to avoid SEGFAULT's. - * Consider timeout for wait had expired and main query's thread ended with exception - * or some other error. But the UpdateUnit with callbacks is left in the queue. - * It has these callback that capture god knows what from the current thread - * (most of the variables lies on the stack of finished thread) that - * intended to do a synchronous update. AsyncUpdate thread can touch deallocated memory and explode. - * */ - is_update_finished.wait( - lock, + std::unique_lock update_lock(update_mutex); + + size_t timeout_for_wait = 100000; + bool result = is_update_finished.wait_for( + update_lock, + std::chrono::milliseconds(timeout_for_wait), [&] {return update_unit_ptr->is_done || update_unit_ptr->current_exception; }); + if (!result) + { + std::lock_guard callback_lock(update_unit_ptr->callback_mutex); + /* + * We acquire a lock here and store false to special variable to avoid SEGFAULT's. + * Consider timeout for wait had expired and main query's thread ended with exception + * or some other error. But the UpdateUnit with callbacks is left in the queue. + * It has these callback that capture god knows what from the current thread + * (most of the variables lies on the stack of finished thread) that + * intended to do a synchronous update. AsyncUpdate thread can touch deallocated memory and explode. + * */ + update_unit_ptr->can_use_callback = false; + throw DB::Exception( + "Dictionary " + getName() + " source seems unavailable, because " + + toString(timeout_for_wait) + " timeout exceeded.", ErrorCodes::TIMEOUT_EXCEEDED); + } + + if (update_unit_ptr->current_exception) std::rethrow_exception(update_unit_ptr->current_exception); } @@ -820,40 +859,24 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const const auto now = std::chrono::system_clock::now(); - /// Non const because it will be unlocked. - ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs}; if (now > backoff_end_time.load()) { try { - if (error_count) - { - /// Recover after error: we have to clone the source here because - /// it could keep connections which should be reset after error. - source_ptr = source_ptr->clone(); - } + auto current_source_ptr = getSourceAndUpdateIfNeeded(); Stopwatch watch; - /// To perform parallel loading. - BlockInputStreamPtr stream = nullptr; - { - ProfilingScopedWriteUnlocker unlocker(write_lock); - stream = source_ptr->loadIds(bunch_update_unit.getRequestedIds()); - } - + BlockInputStreamPtr stream = current_source_ptr->loadIds(bunch_update_unit.getRequestedIds()); stream->readPrefix(); + while (true) { - Block block; - { - ProfilingScopedWriteUnlocker unlocker(write_lock); - block = stream->read(); - if (!block) - break; - } + Block block = stream->read(); + if (!block) + break; const auto id_column = typeid_cast(block.safeGetByPosition(0).column.get()); if (!id_column) @@ -867,6 +890,8 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const for (const auto i : ext::range(0, ids.size())) { + /// Modifying cache with write lock + ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs}; const auto id = ids[i]; const auto find_result = findCellIdx(id, now); @@ -904,6 +929,9 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const stream->readSuffix(); + /// Lock just for last_exception safety + ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs}; + error_count = 0; last_exception = std::exception_ptr{}; backoff_end_time = std::chrono::system_clock::time_point{}; @@ -912,6 +940,8 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const } catch (...) { + /// Lock just for last_exception safety + ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs}; ++error_count; last_exception = std::current_exception(); backoff_end_time = now + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); @@ -921,7 +951,10 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const } } - size_t not_found_num = 0, found_num = 0; + /// Modifying cache state again with write lock + ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs}; + size_t not_found_num = 0; + size_t found_num = 0; /// Check which ids have not been found and require setting null_value for (const auto & id_found_pair : remaining_ids) @@ -968,9 +1001,14 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const { std::uniform_int_distribution distribution{dict_lifetime.min_sec, dict_lifetime.max_sec}; cell.setExpiresAt(now + std::chrono::seconds{distribution(rnd_engine)}); + cell.strict_max = now + std::chrono::seconds{strict_max_lifetime_seconds}; } else + { cell.setExpiresAt(std::chrono::time_point::max()); + cell.strict_max = now + std::chrono::seconds{strict_max_lifetime_seconds}; + } + /// Set null_value for each attribute cell.setDefault(); diff --git a/src/Dictionaries/CacheDictionary.h b/src/Dictionaries/CacheDictionary.h index e425b9c391aa..0b77e1bde512 100644 --- a/src/Dictionaries/CacheDictionary.h +++ b/src/Dictionaries/CacheDictionary.h @@ -55,10 +55,12 @@ class CacheDictionary final : public IDictionary const DictionaryStructure & dict_struct_, DictionarySourcePtr source_ptr_, DictionaryLifetime dict_lifetime_, + size_t strict_max_lifetime_seconds, size_t size_, bool allow_read_expired_keys_, size_t max_update_queue_size_, size_t update_queue_push_timeout_milliseconds_, + size_t query_wait_timeout_milliseconds, size_t max_threads_for_updates); ~CacheDictionary() override; @@ -87,9 +89,18 @@ class CacheDictionary final : public IDictionary std::shared_ptr clone() const override { return std::make_shared( - database, name, dict_struct, source_ptr->clone(), dict_lifetime, size, - allow_read_expired_keys, max_update_queue_size, - update_queue_push_timeout_milliseconds, max_threads_for_updates); + database, + name, + dict_struct, + getSourceAndUpdateIfNeeded()->clone(), + dict_lifetime, + strict_max_lifetime_seconds, + size, + allow_read_expired_keys, + max_update_queue_size, + update_queue_push_timeout_milliseconds, + query_wait_timeout_milliseconds, + max_threads_for_updates); } const IDictionarySource * getSource() const override { return source_ptr.get(); } @@ -206,6 +217,8 @@ class CacheDictionary final : public IDictionary /// Stores both expiration time and `is_default` flag in the most significant bit time_point_urep_t data; + time_point_t strict_max; + /// Sets expiration time, resets `is_default` flag to false time_point_t expiresAt() const { return ext::safe_bit_cast(data & EXPIRES_AT_MASK); } void setExpiresAt(const time_point_t & t) { data = ext::safe_bit_cast(t); } @@ -276,6 +289,26 @@ class CacheDictionary final : public IDictionary Attribute & getAttribute(const std::string & attribute_name) const; + using SharedDictionarySourcePtr = std::shared_ptr; + + /// Update dictionary source pointer if required and return it. Thread safe. + /// MultiVersion is not used here because it works with constant pointers. + /// For some reason almost all methods in IDictionarySource interface are + /// not constant. + SharedDictionarySourcePtr getSourceAndUpdateIfNeeded() const + { + std::lock_guard lock(source_mutex); + if (error_count) + { + /// Recover after error: we have to clone the source here because + /// it could keep connections which should be reset after error. + auto new_source_ptr = source_ptr->clone(); + source_ptr = std::move(new_source_ptr); + } + + return source_ptr; + } + struct FindResult { const size_t cell_idx; @@ -292,15 +325,24 @@ class CacheDictionary final : public IDictionary const std::string name; const std::string full_name; const DictionaryStructure dict_struct; - mutable DictionarySourcePtr source_ptr; + + /// Dictionary source should be used with mutex + mutable std::mutex source_mutex; + mutable SharedDictionarySourcePtr source_ptr; + const DictionaryLifetime dict_lifetime; + const size_t strict_max_lifetime_seconds; const bool allow_read_expired_keys; const size_t max_update_queue_size; const size_t update_queue_push_timeout_milliseconds; + const size_t query_wait_timeout_milliseconds; const size_t max_threads_for_updates; Logger * const log; + /// This lock is used for the inner cache state update function lock it for + /// write, when it need to update cache state all other functions just + /// readers. Suprisingly this lock is also used for last_exception pointer. mutable std::shared_mutex rw_lock; /// Actual size will be increased to match power of 2 @@ -366,6 +408,12 @@ class CacheDictionary final : public IDictionary alive_keys(CurrentMetrics::CacheDictionaryUpdateQueueKeys, requested_ids.size()){} std::vector requested_ids; + + /// It might seem that it is a leak of performance. + /// But aquiring a mutex without contention is rather cheap. + std::mutex callback_mutex; + bool can_use_callback{true}; + PresentIdHandler present_id_handler; AbsentIdHandler absent_id_handler; @@ -412,6 +460,7 @@ class CacheDictionary final : public IDictionary helper.push_back(unit_ptr->requested_ids.size() + helper.back()); present_id_handlers.emplace_back(unit_ptr->present_id_handler); absent_id_handlers.emplace_back(unit_ptr->absent_id_handler); + update_units.emplace_back(unit_ptr); } concatenated_requested_ids.reserve(total_requested_keys_count); @@ -428,31 +477,51 @@ class CacheDictionary final : public IDictionary void informCallersAboutPresentId(Key id, size_t cell_idx) { - for (size_t i = 0; i < concatenated_requested_ids.size(); ++i) + for (size_t position = 0; position < concatenated_requested_ids.size(); ++position) { - auto & curr = concatenated_requested_ids[i]; - if (curr == id) - getPresentIdHandlerForPosition(i)(id, cell_idx); + if (concatenated_requested_ids[position] == id) + { + auto unit_number = getUpdateUnitNumberForRequestedIdPosition(position); + auto lock = getLockToCurrentUnit(unit_number); + if (canUseCallback(unit_number)) + getPresentIdHandlerForPosition(unit_number)(id, cell_idx); + } } } void informCallersAboutAbsentId(Key id, size_t cell_idx) { - for (size_t i = 0; i < concatenated_requested_ids.size(); ++i) - if (concatenated_requested_ids[i] == id) - getAbsentIdHandlerForPosition(i)(id, cell_idx); + for (size_t position = 0; position < concatenated_requested_ids.size(); ++position) + if (concatenated_requested_ids[position] == id) + { + auto unit_number = getUpdateUnitNumberForRequestedIdPosition(position); + auto lock = getLockToCurrentUnit(unit_number); + if (canUseCallback(unit_number)) + getAbsentIdHandlerForPosition(unit_number)(id, cell_idx); + } } private: - PresentIdHandler & getPresentIdHandlerForPosition(size_t position) + /// Needed for control the usage of callback to avoid SEGFAULTs. + bool canUseCallback(size_t unit_number) + { + return update_units[unit_number].get()->can_use_callback; + } + + std::unique_lock getLockToCurrentUnit(size_t unit_number) { - return present_id_handlers[getUpdateUnitNumberForRequestedIdPosition(position)]; + return std::unique_lock(update_units[unit_number].get()->callback_mutex); } - AbsentIdHandler & getAbsentIdHandlerForPosition(size_t position) + PresentIdHandler & getPresentIdHandlerForPosition(size_t unit_number) { - return absent_id_handlers[getUpdateUnitNumberForRequestedIdPosition((position))]; + return update_units[unit_number].get()->present_id_handler; + } + + AbsentIdHandler & getAbsentIdHandlerForPosition(size_t unit_number) + { + return update_units[unit_number].get()->absent_id_handler; } size_t getUpdateUnitNumberForRequestedIdPosition(size_t position) @@ -464,6 +533,8 @@ class CacheDictionary final : public IDictionary std::vector present_id_handlers; std::vector absent_id_handlers; + std::vector> update_units; + std::vector helper; }; diff --git a/src/Dictionaries/CacheDictionary.inc.h b/src/Dictionaries/CacheDictionary.inc.h index 7b108438f76d..6b8734100f06 100644 --- a/src/Dictionaries/CacheDictionary.inc.h +++ b/src/Dictionaries/CacheDictionary.inc.h @@ -75,6 +75,13 @@ void CacheDictionary::getItemsNumberImpl( if (find_result.outdated) { + /// Protection of reading very expired keys. + if (now > cells[find_result.cell_idx].strict_max) + { + cache_not_found_ids[id].push_back(row); + continue; + } + cache_expired_ids[id].push_back(row); if (allow_read_expired_keys) update_routine(); @@ -249,6 +256,13 @@ void CacheDictionary::getItemsString( { if (find_result.outdated) { + /// Protection of reading very expired keys. + if (now > cells[find_result.cell_idx].strict_max) + { + cache_not_found_ids[id].push_back(row); + continue; + } + cache_expired_ids[id].push_back(row); if (allow_read_expired_keys) @@ -290,37 +304,33 @@ void CacheDictionary::getItemsString( /// Request new values sync. /// We have request both cache_not_found_ids and cache_expired_ids. - if (!cache_not_found_ids.empty()) - { - std::vector required_ids; - required_ids.reserve(cache_not_found_ids.size() + cache_expired_ids.size()); - std::transform( - std::begin(cache_not_found_ids), std::end(cache_not_found_ids), - std::back_inserter(required_ids), [](auto & pair) { return pair.first; }); - std::transform( - std::begin(cache_expired_ids), std::end(cache_expired_ids), - std::back_inserter(required_ids), [](auto & pair) { return pair.first; }); - - auto on_cell_updated = [&] (const auto id, const auto cell_idx) - { - const auto attribute_value = attribute_array[cell_idx]; + std::vector required_ids; + required_ids.reserve(cache_not_found_ids.size() + cache_expired_ids.size()); + std::transform( + std::begin(cache_not_found_ids), std::end(cache_not_found_ids), + std::back_inserter(required_ids), [](auto & pair) { return pair.first; }); + std::transform( + std::begin(cache_expired_ids), std::end(cache_expired_ids), + std::back_inserter(required_ids), [](auto & pair) { return pair.first; }); - map[id] = String{attribute_value}; - total_length += (attribute_value.size + 1) * cache_not_found_ids[id].size(); - }; + auto on_cell_updated = [&] (const auto id, const auto cell_idx) + { + const auto attribute_value = attribute_array[cell_idx]; - auto on_id_not_found = [&] (const auto id, const auto) - { - for (const auto row : cache_not_found_ids[id]) - total_length += get_default(row).size + 1; - }; + map[id] = String{attribute_value}; + total_length += (attribute_value.size + 1) * cache_not_found_ids[id].size(); + }; - auto update_unit_ptr = std::make_shared(required_ids, on_cell_updated, on_id_not_found); + auto on_id_not_found = [&] (const auto id, const auto) + { + for (const auto row : cache_not_found_ids[id]) + total_length += get_default(row).size + 1; + }; - tryPushToUpdateQueueOrThrow(update_unit_ptr); - waitForCurrentUpdateFinish(update_unit_ptr); - } + auto update_unit_ptr = std::make_shared(required_ids, on_cell_updated, on_id_not_found); + tryPushToUpdateQueueOrThrow(update_unit_ptr); + waitForCurrentUpdateFinish(update_unit_ptr); out->getChars().reserve(total_length); for (const auto row : ext::range(0, ext::size(ids))) diff --git a/src/Dictionaries/TrieDictionary.cpp b/src/Dictionaries/TrieDictionary.cpp index dcefc873b4f1..a6885db683ff 100644 --- a/src/Dictionaries/TrieDictionary.cpp +++ b/src/Dictionaries/TrieDictionary.cpp @@ -52,18 +52,8 @@ TrieDictionary::TrieDictionary( { createAttributes(); trie = btrie_create(); - - try - { - loadData(); - calculateBytesAllocated(); - } - catch (...) - { - creation_exception = std::current_exception(); - } - - creation_time = std::chrono::system_clock::now(); + loadData(); + calculateBytesAllocated(); } TrieDictionary::~TrieDictionary() diff --git a/src/Dictionaries/TrieDictionary.h b/src/Dictionaries/TrieDictionary.h index 81f5a02a00b8..4b5c49db121c 100644 --- a/src/Dictionaries/TrieDictionary.h +++ b/src/Dictionaries/TrieDictionary.h @@ -251,11 +251,7 @@ class TrieDictionary final : public IDictionaryBase size_t bucket_count = 0; mutable std::atomic query_count{0}; - std::chrono::time_point creation_time; - - std::exception_ptr creation_exception; - - Logger * logger; + Poco::Logger * logger; }; } diff --git a/src/Formats/ProtobufReader.cpp b/src/Formats/ProtobufReader.cpp index 3874ec3e4476..0924900f3cc5 100644 --- a/src/Formats/ProtobufReader.cpp +++ b/src/Formats/ProtobufReader.cpp @@ -34,11 +34,12 @@ namespace BITS32 = 5, }; - // The following condition must always be true: - // any_cursor_position < min(END_OF_VARINT, END_OF_GROUP) - // This inequation helps to check conditions in SimpleReader. - constexpr UInt64 END_OF_VARINT = static_cast(-1); - constexpr UInt64 END_OF_GROUP = static_cast(-2); + // The following conditions must always be true: + // any_cursor_position > END_OF_VARINT + // any_cursor_position > END_OF_GROUP + // Those inequations helps checking conditions in ProtobufReader::SimpleReader. + constexpr Int64 END_OF_VARINT = -1; + constexpr Int64 END_OF_GROUP = -2; Int64 decodeZigZag(UInt64 n) { return static_cast((n >> 1) ^ (~(n & 1) + 1)); } @@ -79,7 +80,7 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors) if (!current_message_level) return; - UInt64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front(); + Int64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front(); if (cursor != root_message_end) { if (cursor < root_message_end) @@ -97,6 +98,9 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors) void ProtobufReader::SimpleReader::startNestedMessage() { assert(current_message_level >= 1); + if ((cursor > field_end) && (field_end != END_OF_GROUP)) + throwUnknownFormat(); + // Start reading a nested message which is located inside a length-delimited field // of another message. parent_message_ends.emplace_back(current_message_end); @@ -148,7 +152,7 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number) throwUnknownFormat(); } - if (cursor >= current_message_end) + if ((cursor >= current_message_end) && (current_message_end != END_OF_GROUP)) return false; UInt64 varint = readVarint(); @@ -198,11 +202,17 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number) bool ProtobufReader::SimpleReader::readUInt(UInt64 & value) { + if (field_end == END_OF_VARINT) + { + value = readVarint(); + field_end = cursor; + return true; + } + if (unlikely(cursor >= field_end)) return false; + value = readVarint(); - if (field_end == END_OF_VARINT) - field_end = cursor; return true; } @@ -229,6 +239,7 @@ bool ProtobufReader::SimpleReader::readFixed(T & value) { if (unlikely(cursor >= field_end)) return false; + readBinary(&value, sizeof(T)); return true; } diff --git a/src/Formats/ProtobufReader.h b/src/Formats/ProtobufReader.h index 393b15eb343b..6670279ee067 100644 --- a/src/Formats/ProtobufReader.h +++ b/src/Formats/ProtobufReader.h @@ -122,12 +122,12 @@ class ProtobufReader : private boost::noncopyable void ignoreGroup(); ReadBuffer & in; - UInt64 cursor; + Int64 cursor; size_t current_message_level; - UInt64 current_message_end; - std::vector parent_message_ends; - UInt64 field_end; - UInt64 last_string_pos; + Int64 current_message_end; + std::vector parent_message_ends; + Int64 field_end; + Int64 last_string_pos; }; class IConverter diff --git a/src/Functions/DivisionUtils.h b/src/Functions/DivisionUtils.h index 9bf153d4d6ed..7b137c957e46 100644 --- a/src/Functions/DivisionUtils.h +++ b/src/Functions/DivisionUtils.h @@ -43,6 +43,13 @@ inline bool divisionLeadsToFPE(A a, B b) return false; } +template +inline auto checkedDivision(A a, B b) +{ + throwIfDivisionLeadsToFPE(a, b); + return a / b; +} + #pragma GCC diagnostic pop @@ -55,14 +62,13 @@ struct DivideIntegralImpl template static inline Result apply(A a, B b) { - throwIfDivisionLeadsToFPE(a, b); - /// Otherwise overflow may occur due to integer promotion. Example: int8_t(-1) / uint64_t(2). /// NOTE: overflow is still possible when dividing large signed number to large unsigned number or vice-versa. But it's less harmful. if constexpr (is_integral_v && is_integral_v && (is_signed_v || is_signed_v)) - return std::make_signed_t(a) / std::make_signed_t(b); + return checkedDivision(std::make_signed_t(a), + sizeof(A) > sizeof(B) ? std::make_signed_t(b) : std::make_signed_t(b)); else - return a / b; + return checkedDivision(a, b); } #if USE_EMBEDDED_COMPILER diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 7021479eed62..7eb66770d66c 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -978,6 +979,8 @@ class FunctionComparison : public IFunction if (0 == tuple_size) throw Exception("Comparison of zero-sized tuples is not implemented.", ErrorCodes::NOT_IMPLEMENTED); + ColumnsWithTypeAndName convolution_types(tuple_size); + Block tmp_block; for (size_t i = 0; i < tuple_size; ++i) { @@ -985,9 +988,10 @@ class FunctionComparison : public IFunction tmp_block.insert(y[i]); auto impl = func_compare->build({x[i], y[i]}); + convolution_types[i].type = impl->getReturnType(); /// Comparison of the elements. - tmp_block.insert({ nullptr, std::make_shared(), "" }); + tmp_block.insert({ nullptr, impl->getReturnType(), "" }); impl->execute(tmp_block, {i * 3, i * 3 + 1}, i * 3 + 2, input_rows_count); } @@ -999,14 +1003,13 @@ class FunctionComparison : public IFunction } /// Logical convolution. - tmp_block.insert({ nullptr, std::make_shared(), "" }); ColumnNumbers convolution_args(tuple_size); for (size_t i = 0; i < tuple_size; ++i) convolution_args[i] = i * 3 + 2; - ColumnsWithTypeAndName convolution_types(convolution_args.size(), { nullptr, std::make_shared(), "" }); auto impl = func_convolution->build(convolution_types); + tmp_block.insert({ nullptr, impl->getReturnType(), "" }); impl->execute(tmp_block, convolution_args, tuple_size * 3, input_rows_count); block.getByPosition(result).column = tmp_block.getByPosition(tuple_size * 3).column; @@ -1025,49 +1028,71 @@ class FunctionComparison : public IFunction size_t tuple_size, size_t input_rows_count) { - ColumnsWithTypeAndName bin_args = {{ nullptr, std::make_shared(), "" }, - { nullptr, std::make_shared(), "" }}; - - auto func_and_adaptor = func_and->build(bin_args); - auto func_or_adaptor = func_or->build(bin_args); - Block tmp_block; /// Pairwise comparison of the inequality of all elements; on the equality of all elements except the last. + /// (x[i], y[i], x[i] < y[i], x[i] == y[i]) for (size_t i = 0; i < tuple_size; ++i) { tmp_block.insert(x[i]); tmp_block.insert(y[i]); - tmp_block.insert({ nullptr, std::make_shared(), "" }); + tmp_block.insert(ColumnWithTypeAndName()); // pos == i * 4 + 2 if (i + 1 != tuple_size) { auto impl_head = func_compare_head->build({x[i], y[i]}); + tmp_block.getByPosition(i * 4 + 2).type = impl_head->getReturnType(); impl_head->execute(tmp_block, {i * 4, i * 4 + 1}, i * 4 + 2, input_rows_count); - tmp_block.insert({ nullptr, std::make_shared(), "" }); + tmp_block.insert(ColumnWithTypeAndName()); // i * 4 + 3 auto impl_equals = func_equals->build({x[i], y[i]}); + tmp_block.getByPosition(i * 4 + 3).type = impl_equals->getReturnType(); impl_equals->execute(tmp_block, {i * 4, i * 4 + 1}, i * 4 + 3, input_rows_count); } else { auto impl_tail = func_compare_tail->build({x[i], y[i]}); + tmp_block.getByPosition(i * 4 + 2).type = impl_tail->getReturnType(); impl_tail->execute(tmp_block, {i * 4, i * 4 + 1}, i * 4 + 2, input_rows_count); } } /// Combination. Complex code - make a drawing. It can be replaced by a recursive comparison of tuples. + /// Last column contains intermediate result. + /// Code is generally equivalent to: + /// res = `x < y`[tuple_size - 1]; + /// for (int i = tuple_size - 2; i >= 0; --i) + /// res = (res && `x == y`[i]) || `x < y`[i]; size_t i = tuple_size - 1; while (i > 0) { - tmp_block.insert({ nullptr, std::make_shared(), "" }); - func_and_adaptor->execute(tmp_block, {tmp_block.columns() - 2, (i - 1) * 4 + 3}, tmp_block.columns() - 1, input_rows_count); - tmp_block.insert({ nullptr, std::make_shared(), "" }); - func_or_adaptor->execute(tmp_block, {tmp_block.columns() - 2, (i - 1) * 4 + 2}, tmp_block.columns() - 1, input_rows_count); --i; + + size_t and_lhs_pos = tmp_block.columns() - 1; // res + size_t and_rhs_pos = i * 4 + 3; // `x == y`[i] + tmp_block.insert(ColumnWithTypeAndName()); + + ColumnsWithTypeAndName and_args = {{ nullptr, tmp_block.getByPosition(and_lhs_pos).type, "" }, + { nullptr, tmp_block.getByPosition(and_rhs_pos).type, "" }}; + + auto func_and_adaptor = func_and->build(and_args); + tmp_block.getByPosition(tmp_block.columns() - 1).type = func_and_adaptor->getReturnType(); + func_and_adaptor->execute(tmp_block, {and_lhs_pos, and_rhs_pos}, tmp_block.columns() - 1, input_rows_count); + + size_t or_lhs_pos = tmp_block.columns() - 1; // (res && `x == y`[i]) + size_t or_rhs_pos = i * 4 + 2; // `x < y`[i] + tmp_block.insert(ColumnWithTypeAndName()); + + ColumnsWithTypeAndName or_args = {{ nullptr, tmp_block.getByPosition(or_lhs_pos).type, "" }, + { nullptr, tmp_block.getByPosition(or_rhs_pos).type, "" }}; + + auto func_or_adaptor = func_or->build(or_args); + tmp_block.getByPosition(tmp_block.columns() - 1).type = func_or_adaptor->getReturnType(); + func_or_adaptor->execute(tmp_block, {or_lhs_pos, or_rhs_pos}, tmp_block.columns() - 1, input_rows_count); + } block.getByPosition(result).column = tmp_block.getByPosition(tmp_block.columns() - 1).column; @@ -1161,13 +1186,20 @@ class FunctionComparison : public IFunction { auto adaptor = FunctionOverloadResolverAdaptor(std::make_unique(FunctionComparison::create(context))); + bool has_nullable = false; + size_t size = left_tuple->getElements().size(); for (size_t i = 0; i < size; ++i) { ColumnsWithTypeAndName args = {{nullptr, left_tuple->getElements()[i], ""}, {nullptr, right_tuple->getElements()[i], ""}}; - adaptor.build(args); + has_nullable = has_nullable || adaptor.build(args)->getReturnType()->isNullable(); } + + /// If any element comparison is nullable, return type will also be nullable. + /// We useDefaultImplementationForNulls, but it doesn't work for tuples. + if (has_nullable) + return std::make_shared(std::make_shared()); } return std::make_shared(); @@ -1186,7 +1218,7 @@ class FunctionComparison : public IFunction /// The case when arguments are the same (tautological comparison). Return constant. /// NOTE: Nullable types are special case. (BTW, this function use default implementation for Nullable, so Nullable types cannot be here. Check just in case.) /// NOTE: We consider NaN comparison to be implementation specific (and in our implementation NaNs are sometimes equal sometimes not). - if (left_type->equals(*right_type) && !left_type->isNullable() && col_left_untyped == col_right_untyped) + if (left_type->equals(*right_type) && !left_type->isNullable() && !isTuple(left_type) && col_left_untyped == col_right_untyped) { /// Always true: =, <=, >= if constexpr (std::is_same_v, EqualsOp> diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 0af5526ff064..aa3c57fcfef7 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -145,6 +145,8 @@ struct ConvertImpl vec_to[i] = convertFromDecimal(vec_from[i], vec_from.getScale()); else if constexpr (IsDataTypeNumber && IsDataTypeDecimal) vec_to[i] = convertToDecimal(vec_from[i], vec_to.getScale()); + else + throw Exception("Unsupported data type in conversion function", ErrorCodes::CANNOT_CONVERT_TYPE); } else vec_to[i] = static_cast(vec_from[i]); diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index e1c89dd7d530..687d0500afa3 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -53,6 +53,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } + /** Functions that use plug-ins (external) dictionaries_loader. * * Get the value of the attribute of the specified type. @@ -68,6 +69,55 @@ namespace ErrorCodes */ +class FunctionDictHelper +{ +public: + FunctionDictHelper(const Context & context_) : context(context_), external_loader(context.getExternalDictionariesLoader()) {} + + std::shared_ptr getDictionary(const String & dictionary_name) + { + auto dict = std::atomic_load(&dictionary); + if (dict) + return dict; + dict = external_loader.getDictionary(dictionary_name); + context.checkAccess(AccessType::dictGet, dict->getDatabaseOrNoDatabaseTag(), dict->getName()); + std::atomic_store(&dictionary, dict); + return dict; + } + + std::shared_ptr getDictionary(const ColumnWithTypeAndName & column) + { + const auto dict_name_col = checkAndGetColumnConst(column.column.get()); + return getDictionary(dict_name_col->getValue()); + } + + bool isDictGetFunctionInjective(const Block & sample_block) + { + /// Assume non-injective by default + if (!sample_block) + return false; + + if (sample_block.columns() != 3 && sample_block.columns() != 4) + throw Exception{"Function dictGet... takes 3 or 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; + + const auto dict_name_col = checkAndGetColumnConst(sample_block.getByPosition(0).column.get()); + if (!dict_name_col) + throw Exception{"First argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; + + const auto attr_name_col = checkAndGetColumnConst(sample_block.getByPosition(1).column.get()); + if (!attr_name_col) + throw Exception{"Second argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; + + return getDictionary(dict_name_col->getValue())->isInjective(attr_name_col->getValue()); + } + +private: + const Context & context; + const ExternalDictionariesLoader & external_loader; + mutable std::shared_ptr dictionary; +}; + + class FunctionDictHas final : public IFunction { public: @@ -75,12 +125,10 @@ class FunctionDictHas final : public IFunction static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictHas(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictHas(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -108,10 +156,6 @@ class FunctionDictHas final : public IFunction void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - /** Do not require existence of the dictionary if the function is called for empty block. * This is needed to allow successful query analysis on a server, * that is the initiator of a distributed query, @@ -126,25 +170,23 @@ class FunctionDictHas final : public IFunction return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictHas, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); - - if (!executeDispatchSimple(block, arguments, result, dict_ptr) && - !executeDispatchSimple(block, arguments, result, dict_ptr) && - !executeDispatchSimple(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); + + if (!executeDispatchSimple(block, arguments, result, dict) && + !executeDispatchSimple(block, arguments, result, dict) && + !executeDispatchSimple(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatchSimple( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -165,9 +207,9 @@ class FunctionDictHas final : public IFunction template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -189,28 +231,11 @@ class FunctionDictHas final : public IFunction return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; +private: + mutable FunctionDictHelper helper; }; -static bool isDictGetFunctionInjective(const ExternalDictionariesLoader & dictionaries_loader, const Block & sample_block) -{ - if (sample_block.columns() != 3 && sample_block.columns() != 4) - throw Exception{"Function dictGet... takes 3 or 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const auto dict_name_col = checkAndGetColumnConst(sample_block.getByPosition(0).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - const auto attr_name_col = checkAndGetColumnConst(sample_block.getByPosition(1).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - return dictionaries_loader.getDictionary(dict_name_col->getValue())->isInjective(attr_name_col->getValue()); -} - - /** For ColumnVector. Either returns a reference to internal data, * or convert it to T type, stores the result in backup_storage and returns a reference to it. */ @@ -225,12 +250,10 @@ class FunctionDictGetString final : public IFunction static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetString(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictGetString(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -243,7 +266,7 @@ class FunctionDictGetString final : public IFunction bool isInjective(const Block & sample_block) override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -286,10 +309,6 @@ class FunctionDictGetString final : public IFunction void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -297,26 +316,24 @@ class FunctionDictGetString final : public IFunction return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); - - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchRange(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); + + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchRange(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -345,9 +362,9 @@ class FunctionDictGetString final : public IFunction template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -382,9 +399,9 @@ class FunctionDictGetString final : public IFunction template bool executeDispatchRange( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -413,8 +430,8 @@ class FunctionDictGetString final : public IFunction return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; +private: + mutable FunctionDictHelper helper; }; @@ -425,12 +442,10 @@ class FunctionDictGetStringOrDefault final : public IFunction static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetStringOrDefault(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictGetStringOrDefault(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -468,10 +483,6 @@ class FunctionDictGetStringOrDefault final : public IFunction void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -479,25 +490,23 @@ class FunctionDictGetStringOrDefault final : public IFunction return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); - - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); + + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -520,7 +529,7 @@ class FunctionDictGetStringOrDefault final : public IFunction template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnUInt64 * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -530,7 +539,7 @@ class FunctionDictGetStringOrDefault final : public IFunction /// vector ids, vector defaults auto out = ColumnString::create(); const auto & ids = id_col->getData(); - dictionary->getString(attr_name, ids, default_col, out.get()); + dict->getString(attr_name, ids, default_col, out.get()); block.getByPosition(result).column = std::move(out); } else if (const auto default_col_const = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) @@ -539,7 +548,7 @@ class FunctionDictGetStringOrDefault final : public IFunction auto out = ColumnString::create(); const auto & ids = id_col->getData(); String def = default_col_const->getValue(); - dictionary->getString(attr_name, ids, def, out.get()); + dict->getString(attr_name, ids, def, out.get()); block.getByPosition(result).column = std::move(out); } else @@ -548,7 +557,7 @@ class FunctionDictGetStringOrDefault final : public IFunction template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnConst * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -558,11 +567,11 @@ class FunctionDictGetStringOrDefault final : public IFunction /// const ids, vector defaults const PaddedPODArray ids(1, id_col->getValue()); PaddedPODArray flags(1); - dictionary->has(ids, flags); + dict->has(ids, flags); if (flags.front()) { auto out = ColumnString::create(); - dictionary->getString(attr_name, ids, String(), out.get()); + dict->getString(attr_name, ids, String(), out.get()); block.getByPosition(result).column = DataTypeString().createColumnConst(id_col->size(), out->getDataAt(0).toString()); } else @@ -574,7 +583,7 @@ class FunctionDictGetStringOrDefault final : public IFunction const PaddedPODArray ids(1, id_col->getValue()); auto out = ColumnString::create(); String def = default_col_const->getValue(); - dictionary->getString(attr_name, ids, def, out.get()); + dict->getString(attr_name, ids, def, out.get()); block.getByPosition(result).column = DataTypeString().createColumnConst(id_col->size(), out->getDataAt(0).toString()); } else @@ -583,9 +592,9 @@ class FunctionDictGetStringOrDefault final : public IFunction template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -621,8 +630,7 @@ class FunctionDictGetStringOrDefault final : public IFunction return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; }; @@ -745,12 +753,11 @@ class FunctionDictGet final : public IFunction static FunctionPtr create(const Context & context, UInt32 dec_scale = 0) { - return std::make_shared(context.getExternalDictionariesLoader(), context, dec_scale); + return std::make_shared(context, dec_scale); } - FunctionDictGet(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_, UInt32 dec_scale = 0) - : dictionaries_loader(dictionaries_loader_) - , context(context_) + FunctionDictGet(const Context & context_, UInt32 dec_scale = 0) + : helper(context_) , decimal_scale(dec_scale) {} @@ -765,7 +772,7 @@ class FunctionDictGet final : public IFunction bool isInjective(const Block & sample_block) override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -806,10 +813,6 @@ class FunctionDictGet final : public IFunction void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -817,26 +820,23 @@ class FunctionDictGet final : public IFunction return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); - - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchRange(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); + + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchRange(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -890,9 +890,9 @@ class FunctionDictGet final : public IFunction template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -933,9 +933,9 @@ class FunctionDictGet final : public IFunction template bool executeDispatchRange( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -969,8 +969,7 @@ class FunctionDictGet final : public IFunction return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; UInt32 decimal_scale; }; @@ -1020,12 +1019,11 @@ class FunctionDictGetOrDefault final : public IFunction static FunctionPtr create(const Context & context, UInt32 dec_scale = 0) { - return std::make_shared(context.getExternalDictionariesLoader(), context, dec_scale); + return std::make_shared(context, dec_scale); } - FunctionDictGetOrDefault(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_, UInt32 dec_scale = 0) - : dictionaries_loader(dictionaries_loader_) - , context(context_) + FunctionDictGetOrDefault(const Context & context_, UInt32 dec_scale = 0) + : helper(context_) , decimal_scale(dec_scale) {} @@ -1066,10 +1064,6 @@ class FunctionDictGetOrDefault final : public IFunction void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -1077,25 +1071,22 @@ class FunctionDictGetOrDefault final : public IFunction return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); - - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); + + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1118,7 +1109,7 @@ class FunctionDictGetOrDefault final : public IFunction template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnUInt64 * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -1134,7 +1125,7 @@ class FunctionDictGetOrDefault final : public IFunction const auto & ids = id_col->getData(); auto & data = out->getData(); const auto & defs = default_col->getData(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, defs, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, defs, data); block.getByPosition(result).column = std::move(out); } else if (const auto default_col_const = checkAndGetColumnConst(default_col_untyped)) @@ -1148,7 +1139,7 @@ class FunctionDictGetOrDefault final : public IFunction const auto & ids = id_col->getData(); auto & data = out->getData(); const auto def = default_col_const->template getValue(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, def, data); block.getByPosition(result).column = std::move(out); } else @@ -1157,7 +1148,7 @@ class FunctionDictGetOrDefault final : public IFunction template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnConst * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -1167,13 +1158,13 @@ class FunctionDictGetOrDefault final : public IFunction /// const ids, vector defaults const PaddedPODArray ids(1, id_col->getValue()); PaddedPODArray flags(1); - dictionary->has(ids, flags); + dict->has(ids, flags); if (flags.front()) { if constexpr (IsDataTypeDecimal) { DecimalPaddedPODArray data(1, decimal_scale); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, Type(), data); + DictGetTraits::getOrDefault(dict, attr_name, ids, Type(), data); block.getByPosition(result).column = DataType(DataType::maxPrecision(), decimal_scale).createColumnConst( id_col->size(), toField(data.front(), decimal_scale)); @@ -1181,7 +1172,7 @@ class FunctionDictGetOrDefault final : public IFunction else { PaddedPODArray data(1); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, Type(), data); + DictGetTraits::getOrDefault(dict, attr_name, ids, Type(), data); block.getByPosition(result).column = DataType().createColumnConst(id_col->size(), toField(data.front())); } } @@ -1197,7 +1188,7 @@ class FunctionDictGetOrDefault final : public IFunction { DecimalPaddedPODArray data(1, decimal_scale); const auto & def = default_col_const->template getValue(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, def, data); block.getByPosition(result).column = DataType(DataType::maxPrecision(), decimal_scale).createColumnConst( id_col->size(), toField(data.front(), decimal_scale)); @@ -1206,7 +1197,7 @@ class FunctionDictGetOrDefault final : public IFunction { PaddedPODArray data(1); const auto & def = default_col_const->template getValue(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, def, data); block.getByPosition(result).column = DataType().createColumnConst(id_col->size(), toField(data.front())); } } @@ -1216,9 +1207,9 @@ class FunctionDictGetOrDefault final : public IFunction template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1266,8 +1257,7 @@ class FunctionDictGetOrDefault final : public IFunction return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; UInt32 decimal_scale; }; @@ -1314,10 +1304,10 @@ class FunctionDictGetNoType final : public IFunction static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetNoType(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) : dictionaries_loader(dictionaries_loader_), context(context_) {} + FunctionDictGetNoType(const Context & context_) : context(context_), helper(context_) {} String getName() const override { return name; } @@ -1330,7 +1320,7 @@ class FunctionDictGetNoType final : public IFunction bool isInjective(const Block & sample_block) override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override @@ -1370,7 +1360,7 @@ class FunctionDictGetNoType final : public IFunction + ", must be convertible to " + TypeName::get() + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; } - auto dict = dictionaries_loader.getDictionary(dict_name); + auto dict = helper.getDictionary(dict_name); const DictionaryStructure & structure = dict->getStructure(); for (const auto idx : ext::range(0, structure.attributes.size())) @@ -1450,8 +1440,8 @@ class FunctionDictGetNoType final : public IFunction } private: - const ExternalDictionariesLoader & dictionaries_loader; const Context & context; + mutable FunctionDictHelper helper; mutable FunctionPtr impl; // underlying function used by dictGet function without explicit type info }; @@ -1463,10 +1453,10 @@ class FunctionDictGetNoTypeOrDefault final : public IFunction static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetNoTypeOrDefault(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) : dictionaries_loader(dictionaries_loader_), context(context_) {} + FunctionDictGetNoTypeOrDefault(const Context & context_) : context(context_), helper(context_) {} String getName() const override { return name; } @@ -1478,7 +1468,7 @@ class FunctionDictGetNoTypeOrDefault final : public IFunction bool isInjective(const Block & sample_block) override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override @@ -1506,7 +1496,7 @@ class FunctionDictGetNoTypeOrDefault final : public IFunction throw Exception{"Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ", must be UInt64 or tuple(...).", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - auto dict = dictionaries_loader.getDictionary(dict_name); + auto dict = helper.getDictionary(dict_name); const DictionaryStructure & structure = dict->getStructure(); for (const auto idx : ext::range(0, structure.attributes.size())) @@ -1592,8 +1582,8 @@ class FunctionDictGetNoTypeOrDefault final : public IFunction } private: - const ExternalDictionariesLoader & dictionaries_loader; const Context & context; + mutable FunctionDictHelper helper; mutable FunctionPtr impl; // underlying function used by dictGet function without explicit type info }; @@ -1606,12 +1596,10 @@ class FunctionDictGetHierarchy final : public IFunction static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetHierarchy(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictGetHierarchy(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -1639,10 +1627,6 @@ class FunctionDictGetHierarchy final : public IFunction void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -1650,21 +1634,19 @@ class FunctionDictGetHierarchy final : public IFunction return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGetHierarchy, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1755,8 +1737,7 @@ class FunctionDictGetHierarchy final : public IFunction return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; }; @@ -1767,12 +1748,11 @@ class FunctionDictIsIn final : public IFunction static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictIsIn(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictIsIn(const Context & context_) + : helper(context_) {} String getName() const override { return name; } @@ -1803,10 +1783,6 @@ class FunctionDictIsIn final : public IFunction void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -1814,21 +1790,19 @@ class FunctionDictIsIn final : public IFunction return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictIsIn, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) - && !executeDispatch(block, arguments, result, dict_ptr) - && !executeDispatch(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + if (!executeDispatch(block, arguments, result, dict) + && !executeDispatch(block, arguments, result, dict) + && !executeDispatch(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1850,7 +1824,7 @@ class FunctionDictIsIn final : public IFunction } template - bool execute(Block & block, const size_t result, const DictionaryType * dictionary, + bool execute(Block & block, const size_t result, const DictionaryType * dict, const ColumnUInt64 * child_id_col, const IColumn * ancestor_id_col_untyped) { if (const auto ancestor_id_col = checkAndGetColumn(ancestor_id_col_untyped)) @@ -1863,7 +1837,7 @@ class FunctionDictIsIn final : public IFunction const auto size = child_id_col->size(); data.resize(size); - dictionary->isInVectorVector(child_ids, ancestor_ids, data); + dict->isInVectorVector(child_ids, ancestor_ids, data); block.getByPosition(result).column = std::move(out); } else if (const auto ancestor_id_col_const = checkAndGetColumnConst>(ancestor_id_col_untyped)) @@ -1876,7 +1850,7 @@ class FunctionDictIsIn final : public IFunction const auto size = child_id_col->size(); data.resize(size); - dictionary->isInVectorConstant(child_ids, ancestor_id, data); + dict->isInVectorConstant(child_ids, ancestor_id, data); block.getByPosition(result).column = std::move(out); } else @@ -1889,7 +1863,7 @@ class FunctionDictIsIn final : public IFunction } template - bool execute(Block & block, const size_t result, const DictionaryType * dictionary, + bool execute(Block & block, const size_t result, const DictionaryType * dict, const ColumnConst * child_id_col, const IColumn * ancestor_id_col_untyped) { if (const auto ancestor_id_col = checkAndGetColumn(ancestor_id_col_untyped)) @@ -1902,7 +1876,7 @@ class FunctionDictIsIn final : public IFunction const auto size = child_id_col->size(); data.resize(size); - dictionary->isInConstantVector(child_id, ancestor_ids, data); + dict->isInConstantVector(child_id, ancestor_ids, data); block.getByPosition(result).column = std::move(out); } else if (const auto ancestor_id_col_const = checkAndGetColumnConst>(ancestor_id_col_untyped)) @@ -1911,7 +1885,7 @@ class FunctionDictIsIn final : public IFunction const auto ancestor_id = ancestor_id_col_const->getValue(); UInt8 res = 0; - dictionary->isInConstantConstant(child_id, ancestor_id, res); + dict->isInConstantConstant(child_id, ancestor_id, res); block.getByPosition(result).column = DataTypeUInt8().createColumnConst(child_id_col->size(), res); } else @@ -1921,8 +1895,7 @@ class FunctionDictIsIn final : public IFunction return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; }; diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index 8b8f03a0c899..43404b394968 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -42,7 +42,7 @@ using UInt8Container = ColumnUInt8::Container; using UInt8ColumnPtrs = std::vector; -MutableColumnPtr convertFromTernaryData(const UInt8Container & ternary_data, const bool make_nullable) +MutableColumnPtr buildColumnFromTernaryData(const UInt8Container & ternary_data, const bool make_nullable) { const size_t rows_count = ternary_data.size(); @@ -63,7 +63,7 @@ MutableColumnPtr convertFromTernaryData(const UInt8Container & ternary_data, con } template -bool tryConvertColumnToUInt8(const IColumn * column, UInt8Container & res) +bool tryConvertColumnToBool(const IColumn * column, UInt8Container & res) { const auto col = checkAndGetColumn>(column); if (!col) @@ -71,22 +71,22 @@ bool tryConvertColumnToUInt8(const IColumn * column, UInt8Container & res) std::transform( col->getData().cbegin(), col->getData().cend(), res.begin(), - [](const auto x) { return x != 0; }); + [](const auto x) { return !!x; }); return true; } -void convertColumnToUInt8(const IColumn * column, UInt8Container & res) +void convertAnyColumnToBool(const IColumn * column, UInt8Container & res) { - if (!tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res)) + if (!tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res)) throw Exception("Unexpected type of column: " + column->getName(), ErrorCodes::ILLEGAL_COLUMN); } @@ -119,7 +119,7 @@ static bool extractConstColumns(ColumnRawPtrs & in, UInt8 & res, Func && func) } template -inline bool extractConstColumns(ColumnRawPtrs & in, UInt8 & res) +inline bool extractConstColumnsAsBool(ColumnRawPtrs & in, UInt8 & res) { return extractConstColumns( in, res, @@ -131,7 +131,7 @@ inline bool extractConstColumns(ColumnRawPtrs & in, UInt8 & res) } template -inline bool extractConstColumnsTernary(ColumnRawPtrs & in, UInt8 & res_3v) +inline bool extractConstColumnsAsTernary(ColumnRawPtrs & in, UInt8 & res_3v) { return extractConstColumns( in, res_3v, @@ -145,6 +145,7 @@ inline bool extractConstColumnsTernary(ColumnRawPtrs & in, UInt8 & res_3v) } +/// N.B. This class calculates result only for non-nullable types template class AssociativeApplierImpl { @@ -158,7 +159,7 @@ class AssociativeApplierImpl /// Returns a combination of values in the i-th row of all columns stored in the constructor. inline ResultValueType apply(const size_t i) const { - const auto & a = vec[i]; + const auto a = !!vec[i]; if constexpr (Op::isSaturable()) return Op::isSaturatedValue(a) ? a : Op::apply(a, next.apply(i)); else @@ -179,7 +180,7 @@ class AssociativeApplierImpl AssociativeApplierImpl(const UInt8ColumnPtrs & in) : vec(in[in.size() - 1]->getData()) {} - inline ResultValueType apply(const size_t i) const { return vec[i]; } + inline ResultValueType apply(const size_t i) const { return !!vec[i]; } private: const UInt8Container & vec; @@ -188,7 +189,7 @@ class AssociativeApplierImpl /// A helper class used by AssociativeGenericApplierImpl /// Allows for on-the-fly conversion of any data type into intermediate ternary representation -using ValueGetter = std::function; +using TernaryValueGetter = std::function; template struct ValueGetterBuilderImpl; @@ -196,7 +197,7 @@ struct ValueGetterBuilderImpl; template struct ValueGetterBuilderImpl { - static ValueGetter build(const IColumn * x) + static TernaryValueGetter build(const IColumn * x) { if (const auto nullable_column = typeid_cast(x)) { @@ -218,7 +219,7 @@ struct ValueGetterBuilderImpl template <> struct ValueGetterBuilderImpl<> { - static ValueGetter build(const IColumn * x) + static TernaryValueGetter build(const IColumn * x) { throw Exception( std::string("Unknown numeric column of type: ") + demangle(typeid(x).name()), @@ -247,13 +248,13 @@ class AssociativeGenericApplierImpl { const auto a = val_getter(i); if constexpr (Op::isSaturable()) - return Op::isSaturatedValue(a) ? a : Op::apply(a, next.apply(i)); + return Op::isSaturatedValueTernary(a) ? a : Op::apply(a, next.apply(i)); else return Op::apply(a, next.apply(i)); } private: - const ValueGetter val_getter; + const TernaryValueGetter val_getter; const AssociativeGenericApplierImpl next; }; @@ -271,7 +272,7 @@ class AssociativeGenericApplierImpl inline ResultValueType apply(const size_t i) const { return val_getter(i); } private: - const ValueGetter val_getter; + const TernaryValueGetter val_getter; }; @@ -332,13 +333,13 @@ static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAn { /// Combine all constant columns into a single constant value. UInt8 const_3v_value = 0; - const bool has_consts = extractConstColumnsTernary(arguments, const_3v_value); + const bool has_consts = extractConstColumnsAsTernary(arguments, const_3v_value); /// If the constant value uniquely determines the result, return it. - if (has_consts && (arguments.empty() || Op::isSaturatedValue(const_3v_value))) + if (has_consts && (arguments.empty() || Op::isSaturatedValueTernary(const_3v_value))) { result_info.column = ColumnConst::create( - convertFromTernaryData(UInt8Container({const_3v_value}), result_info.type->isNullable()), + buildColumnFromTernaryData(UInt8Container({const_3v_value}), result_info.type->isNullable()), input_rows_count ); return; @@ -349,7 +350,7 @@ static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAn OperationApplier::apply(arguments, result_column->getData(), has_consts); - result_info.column = convertFromTernaryData(result_column->getData(), result_info.type->isNullable()); + result_info.column = buildColumnFromTernaryData(result_column->getData(), result_info.type->isNullable()); } @@ -402,12 +403,13 @@ struct TypedExecutorInvoker }; +/// Types of all of the arguments are guaranteed to be non-nullable here template static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & result_info, size_t input_rows_count) { /// Combine all constant columns into a single constant value. UInt8 const_val = 0; - bool has_consts = extractConstColumns(arguments, const_val); + bool has_consts = extractConstColumnsAsBool(arguments, const_val); /// If the constant value uniquely determines the result, return it. if (has_consts && (arguments.empty() || Op::apply(const_val, 0) == Op::apply(const_val, 1))) @@ -447,7 +449,7 @@ static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & re else { auto converted_column = ColumnUInt8::create(input_rows_count); - convertColumnToUInt8(column, converted_column->getData()); + convertAnyColumnToBool(column, converted_column->getData()); uint8_args.push_back(converted_column.get()); converted_columns_holder.emplace_back(std::move(converted_column)); } @@ -496,7 +498,8 @@ DataTypePtr FunctionAnyArityLogical::getReturnTypeImpl(const DataTyp } template -void FunctionAnyArityLogical::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_index, size_t input_rows_count) +void FunctionAnyArityLogical::executeImpl( + Block & block, const ColumnNumbers & arguments, size_t result_index, size_t input_rows_count) { ColumnRawPtrs args_in; for (const auto arg_index : arguments) diff --git a/src/Functions/FunctionsLogical.h b/src/Functions/FunctionsLogical.h index 520a24188906..a6dc9865e6ff 100644 --- a/src/Functions/FunctionsLogical.h +++ b/src/Functions/FunctionsLogical.h @@ -36,9 +36,21 @@ namespace Ternary { using ResultType = UInt8; - static constexpr UInt8 False = 0; - static constexpr UInt8 True = -1; - static constexpr UInt8 Null = 1; + /** These carefully picked values magically work so bitwise "and", "or" on them + * corresponds to the expected results in three-valued logic. + * + * False and True are represented by all-0 and all-1 bits, so all bitwise operations on them work as expected. + * Null is represented as single 1 bit. So, it is something in between False and True. + * And "or" works like maximum and "and" works like minimum: + * "or" keeps True as is and lifts False with Null to Null. + * "and" keeps False as is and downs True with Null to Null. + * + * This logic does not apply for "not" and "xor" - they work with default implementation for NULLs: + * anything with NULL returns NULL, otherwise use conventional two-valued logic. + */ + static constexpr UInt8 False = 0; /// All zero bits. + static constexpr UInt8 True = -1; /// All one bits. + static constexpr UInt8 Null = 1; /// Single one bit. template inline ResultType makeValue(T value) @@ -61,8 +73,16 @@ struct AndImpl using ResultType = UInt8; static inline constexpr bool isSaturable() { return true; } - static inline constexpr bool isSaturatedValue(UInt8 a) { return a == Ternary::False; } + + /// Final value in two-valued logic (no further operations with True, False will change this value) + static inline constexpr bool isSaturatedValue(bool a) { return !a; } + + /// Final value in three-valued logic (no further operations with True, False, Null will change this value) + static inline constexpr bool isSaturatedValueTernary(UInt8 a) { return a == Ternary::False; } + static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return a & b; } + + /// Will use three-valued logic for NULLs (see above) or default implementation (any operation with NULL returns NULL). static inline constexpr bool specialImplementationForNulls() { return true; } }; @@ -71,7 +91,8 @@ struct OrImpl using ResultType = UInt8; static inline constexpr bool isSaturable() { return true; } - static inline constexpr bool isSaturatedValue(UInt8 a) { return a == Ternary::True; } + static inline constexpr bool isSaturatedValue(bool a) { return a; } + static inline constexpr bool isSaturatedValueTernary(UInt8 a) { return a == Ternary::True; } static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return a | b; } static inline constexpr bool specialImplementationForNulls() { return true; } }; @@ -82,7 +103,8 @@ struct XorImpl static inline constexpr bool isSaturable() { return false; } static inline constexpr bool isSaturatedValue(bool) { return false; } - static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return !!a != !!b; } + static inline constexpr bool isSaturatedValueTernary(UInt8) { return false; } + static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return a != b; } static inline constexpr bool specialImplementationForNulls() { return false; } #if USE_EMBEDDED_COMPILER diff --git a/src/Functions/FunctionsMiscellaneous.h b/src/Functions/FunctionsMiscellaneous.h index 7c884cc70405..5703f72ce2ad 100644 --- a/src/Functions/FunctionsMiscellaneous.h +++ b/src/Functions/FunctionsMiscellaneous.h @@ -8,11 +8,13 @@ #include #include + namespace DB { namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; } class ExecutableFunctionExpression : public IExecutableFunctionImpl @@ -115,6 +117,7 @@ class ExecutableFunctionCapture : public IExecutableFunctionImpl String getName() const override { return "FunctionCapture"; } bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { @@ -202,6 +205,11 @@ class FunctionCaptureOverloadResolver : public IFunctionOverloadResolverImpl const String & expression_return_name_) : expression_actions(std::move(expression_actions_)) { + /// Check that expression does not contain unusual actions that will break blocks structure. + for (const auto & action : expression_actions->getActions()) + if (action.type == ExpressionAction::Type::JOIN || action.type == ExpressionAction::Type::ARRAY_JOIN) + throw Exception("Expression with arrayJoin or other unusual action cannot be captured", ErrorCodes::BAD_ARGUMENTS); + std::unordered_map arguments_map; const auto & all_arguments = expression_actions->getRequiredColumnsWithTypes(); @@ -243,6 +251,7 @@ class FunctionCaptureOverloadResolver : public IFunctionOverloadResolverImpl String getName() const override { return name; } bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } DataTypePtr getReturnType(const ColumnsWithTypeAndName &) const override { return return_type; } size_t getNumberOfArguments() const override { return capture->captured_types.size(); } diff --git a/src/Functions/GeoUtils.cpp b/src/Functions/GeoUtils.cpp index 488a102e2089..b3529b3551c6 100644 --- a/src/Functions/GeoUtils.cpp +++ b/src/Functions/GeoUtils.cpp @@ -266,10 +266,10 @@ void geohashDecode(const char * encoded_string, size_t encoded_len, Float64 * lo *latitude = decodeCoordinate(lat_encoded, LAT_MIN, LAT_MAX, singleCoordBitsPrecision(precision, LATITUDE)); } -GeohashesInBoxPreparedArgs geohashesInBoxPrepare(const Float64 longitude_min, - const Float64 latitude_min, - const Float64 longitude_max, - const Float64 latitude_max, +GeohashesInBoxPreparedArgs geohashesInBoxPrepare(Float64 longitude_min, + Float64 latitude_min, + Float64 longitude_max, + Float64 latitude_max, uint8_t precision) { precision = geohashPrecision(precision); @@ -279,6 +279,11 @@ GeohashesInBoxPreparedArgs geohashesInBoxPrepare(const Float64 longitude_min, return {}; } + longitude_min = std::max(longitude_min, LON_MIN); + longitude_max = std::min(longitude_max, LON_MAX); + latitude_min = std::max(latitude_min, LAT_MIN); + latitude_max = std::min(latitude_max, LAT_MAX); + const auto lon_step = getSpan(precision, LONGITUDE); const auto lat_step = getSpan(precision, LATITUDE); diff --git a/src/Functions/GeoUtils.h b/src/Functions/GeoUtils.h index f5e15ae013de..45fbe05df5e2 100644 --- a/src/Functions/GeoUtils.h +++ b/src/Functions/GeoUtils.h @@ -301,6 +301,9 @@ bool PointInPolygonWithGrid::contains(CoordinateType x, Coordina if (has_empty_bound) return false; + if (std::isnan(x) || std::isnan(y)) + return false; + CoordinateType float_row = (y + y_shift) * y_scale; CoordinateType float_col = (x + x_shift) * x_scale; diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index 3887f3f16698..f0418e37f81c 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -130,6 +130,10 @@ class IFunctionBase * But we assume, that it is injective. This could be documented as implementation-specific behaviour. * * sample_block should contain data types of arguments and values of constants, if relevant. + * NOTE: to check is function injective with any arguments, you can pass + * empty block as sample_block (since most of the time function will + * ignore it anyway, and creating arguments just for checking is + * function injective or not is overkill). */ virtual bool isInjective(const Block & /*sample_block*/) { return false; } diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 3313f2a8e6d5..667f8f8c5ce8 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -33,7 +33,8 @@ namespace ErrorCodes * arrayMap(x1,...,xn -> expression, array1,...,arrayn) - apply the expression to each element of the array (or set of parallel arrays). * arrayFilter(x -> predicate, array) - leave in the array only the elements for which the expression is true. * - * For some functions arrayCount, arrayExists, arrayAll, an overload of the form f(array) is available, which works in the same way as f(x -> x, array). + * For some functions arrayCount, arrayExists, arrayAll, an overload of the form f(array) is available, + * which works in the same way as f(x -> x, array). * * See the example of Impl template parameter in arrayMap.cpp */ @@ -72,7 +73,7 @@ class FunctionArrayMapped : public IFunction if (!array_type) throw Exception("Argument " + toString(i + 2) + " of function " + getName() + " must be array. Found " + arguments[i + 1]->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - nested_types[i] = removeLowCardinality(array_type->getNestedType()); + nested_types[i] = recursiveRemoveLowCardinality(array_type->getNestedType()); } const DataTypeFunction * function_type = checkAndGetDataType(arguments[0].get()); @@ -189,9 +190,7 @@ class FunctionArrayMapped : public IFunction const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); if (!column_const_array) throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); - column_array_ptr = column_const_array->convertToFullColumn(); - if (column_array_ptr->lowCardinality()) - column_array_ptr = column_array_ptr->convertToFullColumnIfLowCardinality(); + column_array_ptr = recursiveRemoveLowCardinality(column_const_array->convertToFullColumn()); column_array = checkAndGetColumn(column_array_ptr.get()); } @@ -217,7 +216,7 @@ class FunctionArrayMapped : public IFunction } arrays.emplace_back(ColumnWithTypeAndName(column_array->getDataPtr(), - removeLowCardinality(array_type->getNestedType()), + recursiveRemoveLowCardinality(array_type->getNestedType()), array_with_type_and_name.name)); } diff --git a/src/Functions/array/arrayFill.cpp b/src/Functions/array/arrayFill.cpp index 0a463f908a32..0d2bcfcb185c 100644 --- a/src/Functions/array/arrayFill.cpp +++ b/src/Functions/array/arrayFill.cpp @@ -45,11 +45,11 @@ struct ArrayFillImpl for (auto in_offset : in_offsets) { - array_end = in_offset - 1; + array_end = in_offset; - for (; end <= array_end; ++end) + for (; end < array_end; ++end) { - if (end == array_end || fill[end + 1] != fill[begin]) + if (end + 1 == array_end || fill[end + 1] != fill[begin]) { if (fill[begin]) out_data.insertRangeFrom(in_data, begin, end + 1 - begin); @@ -57,8 +57,8 @@ struct ArrayFillImpl { if constexpr (reverse) { - if (end == array_end) - out_data.insertManyFrom(in_data, array_end, end + 1 - begin); + if (end + 1 == array_end) + out_data.insertManyFrom(in_data, end, end + 1 - begin); else out_data.insertManyFrom(in_data, end + 1, end + 1 - begin); } @@ -75,7 +75,7 @@ struct ArrayFillImpl } } - array_begin = array_end + 1; + array_begin = array_end; } } else @@ -88,8 +88,7 @@ struct ArrayFillImpl if (column_fill_const->getValue()) return ColumnArray::create( array.getDataPtr(), - array.getOffsetsPtr() - ); + array.getOffsetsPtr()); size_t array_begin = 0; size_t array_end = 0; diff --git a/src/Functions/defaultValueOfArgumentType.cpp b/src/Functions/defaultValueOfArgumentType.cpp index 85da76ce6943..b07a92bd6777 100644 --- a/src/Functions/defaultValueOfArgumentType.cpp +++ b/src/Functions/defaultValueOfArgumentType.cpp @@ -22,6 +22,7 @@ class FunctionDefaultValueOfArgumentType : public IFunction } bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } size_t getNumberOfArguments() const override { diff --git a/src/Functions/geohashesInBox.cpp b/src/Functions/geohashesInBox.cpp index ff11747642a3..edf5dfe3ad4f 100644 --- a/src/Functions/geohashesInBox.cpp +++ b/src/Functions/geohashesInBox.cpp @@ -120,7 +120,7 @@ class FunctionGeohashesInBox : public IFunction // Actually write geohashes into preallocated buffer. GeoUtils::geohashesInBox(prepared_args, out); - for (UInt8 i = 1; i <= prepared_args.items_count ; ++i) + for (UInt64 i = 1; i <= prepared_args.items_count ; ++i) { res_strings_offsets.push_back(starting_offset + (prepared_args.precision + 1) * i); } diff --git a/src/Functions/greatCircleDistance.cpp b/src/Functions/greatCircleDistance.cpp index cdaf15890741..c5ca9cb4b5c6 100644 --- a/src/Functions/greatCircleDistance.cpp +++ b/src/Functions/greatCircleDistance.cpp @@ -100,8 +100,6 @@ void geodistInit() inline float geodistDegDiff(float f) { f = fabsf(f); - while (f > 360) - f -= 360; if (f > 180) f = 360 - f; return f; diff --git a/src/Functions/hasColumnInTable.cpp b/src/Functions/hasColumnInTable.cpp index 228052e8d460..1119cc6a33fd 100644 --- a/src/Functions/hasColumnInTable.cpp +++ b/src/Functions/hasColumnInTable.cpp @@ -17,6 +17,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int UNKNOWN_TABLE; } @@ -110,6 +111,9 @@ void FunctionHasColumnInTable::executeImpl(Block & block, const ColumnNumbers & String table_name = get_string_from_block(arguments[arg++]); String column_name = get_string_from_block(arguments[arg++]); + if (table_name.empty()) + throw Exception("Table name is empty", ErrorCodes::UNKNOWN_TABLE); + bool has_column; if (host_name.empty()) { diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index 7ae1042c22c2..6a25ea11d409 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -158,7 +158,7 @@ struct NumIfImpl private: [[noreturn]] static void throw_error() { - throw Exception("Internal logic error: invalid types of arguments 2 and 3 of if", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("Invalid types of arguments 2 and 3 of if", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } public: template static void vector_vector(Args &&...) { throw_error(); } @@ -433,8 +433,7 @@ class FunctionIf : public FunctionIfBase const PaddedPODArray & cond_data = cond_col->getData(); size_t rows = cond_data.size(); - if ((col_then_fixed || col_then_const_fixed) - && (col_else_fixed || col_else_const_fixed)) + if (isFixedString(block.getByPosition(result).type)) { /// The result is FixedString. @@ -449,16 +448,19 @@ class FunctionIf : public FunctionIfBase else if (col_then_const_fixed && col_else_fixed) conditional(ConstSource(*col_then_const_fixed), FixedStringSource(*col_else_fixed), sink, cond_data); else if (col_then_const_fixed && col_else_const_fixed) - conditional(ConstSource(*col_then_const_fixed), ConstSource(*col_else_const_fixed), sink, cond_data); + conditional(ConstSource(*col_then_const_fixed), + ConstSource(*col_else_const_fixed), sink, cond_data); + else + return false; block.getByPosition(result).column = std::move(col_res_untyped); return true; } - if ((col_then || col_then_const || col_then_fixed || col_then_const_fixed) - && (col_else || col_else_const || col_else_fixed || col_else_const_fixed)) + if (isString(block.getByPosition(result).type)) { /// The result is String. + auto col_res = ColumnString::create(); auto sink = StringSink(*col_res, rows); @@ -486,6 +488,17 @@ class FunctionIf : public FunctionIfBase conditional(ConstSource(*col_then_const), ConstSource(*col_else_const_fixed), sink, cond_data); else if (col_then_const_fixed && col_else_const) conditional(ConstSource(*col_then_const_fixed), ConstSource(*col_else_const), sink, cond_data); + else if (col_then_fixed && col_else_fixed) + conditional(FixedStringSource(*col_then_fixed), FixedStringSource(*col_else_fixed), sink, cond_data); + else if (col_then_fixed && col_else_const_fixed) + conditional(FixedStringSource(*col_then_fixed), ConstSource(*col_else_const_fixed), sink, cond_data); + else if (col_then_const_fixed && col_else_fixed) + conditional(ConstSource(*col_then_const_fixed), FixedStringSource(*col_else_fixed), sink, cond_data); + else if (col_then_const_fixed && col_else_const_fixed) + conditional(ConstSource(*col_then_const_fixed), + ConstSource(*col_else_const_fixed), sink, cond_data); + else + return false; block.getByPosition(result).column = std::move(col_res); return true; @@ -591,7 +604,8 @@ class FunctionIf : public FunctionIfBase return true; } - void executeGeneric(const ColumnUInt8 * cond_col, Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) + void executeGeneric( + const ColumnUInt8 * cond_col, Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) { /// Convert both columns to the common type (if needed). @@ -657,30 +671,89 @@ class FunctionIf : public FunctionIfBase block.getByPosition(result).column = std::move(result_column); } - bool executeForNullableCondition(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) + bool executeForConstAndNullableCondition(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) { const ColumnWithTypeAndName & arg_cond = block.getByPosition(arguments[0]); bool cond_is_null = arg_cond.column->onlyNull(); - if (cond_is_null) + ColumnPtr not_const_condition = arg_cond.column; + bool cond_is_const = false; + bool cond_is_true = false; + bool cond_is_false = false; + if (const auto * const_arg = checkAndGetColumn(*arg_cond.column)) { - block.getByPosition(result).column = std::move(block.getByPosition(arguments[2]).column); - return true; + cond_is_const = true; + not_const_condition = const_arg->getDataColumnPtr(); + ColumnPtr data_column = const_arg->getDataColumnPtr(); + if (const auto * const_nullable_arg = checkAndGetColumn(*data_column)) + { + data_column = const_nullable_arg->getNestedColumnPtr(); + if (!data_column->empty()) + cond_is_null = const_nullable_arg->getNullMapData()[0]; + } + + if (!data_column->empty()) + { + cond_is_true = !cond_is_null && checkAndGetColumn(*data_column)->getBool(0); + cond_is_false = !cond_is_null && !cond_is_true; + } + } + + const auto & column1 = block.getByPosition(arguments[1]); + const auto & column2 = block.getByPosition(arguments[2]); + auto & result_column = block.getByPosition(result); + + if (cond_is_true) + { + if (result_column.type->equals(*column1.type)) + { + result_column.column = std::move(column1.column); + return true; + } + } + else if (cond_is_false || cond_is_null) + { + if (result_column.type->equals(*column2.type)) + { + result_column.column = std::move(column2.column); + return true; + } } - if (auto * nullable = checkAndGetColumn(*arg_cond.column)) + if (const auto * nullable = checkAndGetColumn(*not_const_condition)) { + ColumnPtr new_cond_column = nullable->getNestedColumnPtr(); + size_t column_size = arg_cond.column->size(); + + if (cond_is_null || cond_is_true || cond_is_false) /// Nullable(Nothing) or consts + { + UInt8 value = cond_is_true ? 1 : 0; + new_cond_column = ColumnConst::create(ColumnUInt8::create(1, value), column_size); + } + else if (checkAndGetColumn(*new_cond_column)) + { + auto nested_column_copy = new_cond_column->cloneResized(new_cond_column->size()); + typeid_cast(nested_column_copy.get())->applyZeroMap(nullable->getNullMapData()); + new_cond_column = std::move(nested_column_copy); + + if (cond_is_const) + new_cond_column = ColumnConst::create(new_cond_column, column_size); + } + else + throw Exception("Illegal column " + arg_cond.column->getName() + " of " + getName() + " condition", + ErrorCodes::ILLEGAL_COLUMN); + Block temporary_block { - { nullable->getNestedColumnPtr(), removeNullable(arg_cond.type), arg_cond.name }, - block.getByPosition(arguments[1]), - block.getByPosition(arguments[2]), - block.getByPosition(result) + { new_cond_column, removeNullable(arg_cond.type), arg_cond.name }, + column1, + column2, + result_column }; executeImpl(temporary_block, {0, 1, 2}, 3, temporary_block.rows()); - block.getByPosition(result).column = std::move(temporary_block.getByPosition(3).column); + result_column.column = std::move(temporary_block.getByPosition(3).column); return true; } @@ -916,7 +989,7 @@ class FunctionIf : public FunctionIfBase void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - if (executeForNullableCondition(block, arguments, result, input_rows_count) + if (executeForConstAndNullableCondition(block, arguments, result, input_rows_count) || executeForNullThenElse(block, arguments, result, input_rows_count) || executeForNullableThenElse(block, arguments, result, input_rows_count)) return; @@ -964,10 +1037,7 @@ class FunctionIf : public FunctionIfBase using T0 = typename Types::LeftType; using T1 = typename Types::RightType; - if constexpr (IsDecimalNumber == IsDecimalNumber) - return executeTyped(cond_col, block, arguments, result, input_rows_count); - else - throw Exception("Conditional function with Decimal and non Decimal", ErrorCodes::NOT_IMPLEMENTED); + return executeTyped(cond_col, block, arguments, result, input_rows_count); }; TypeIndex left_id = arg_then.type->getTypeId(); diff --git a/src/Functions/intDiv.cpp b/src/Functions/intDiv.cpp index 5bd6abcbff89..ae32750e401d 100644 --- a/src/Functions/intDiv.cpp +++ b/src/Functions/intDiv.cpp @@ -26,12 +26,11 @@ struct DivideIntegralByConstantImpl static NO_INLINE void vector_constant(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size) { - if (unlikely(b == 0)) - throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION); - #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wsign-compare" + /// Division by -1. By the way, we avoid FPE by division of the largest negative number by -1. + /// And signed integer overflow is well defined in C++20. if (unlikely(is_signed_v && b == -1)) { for (size_t i = 0; i < size; ++i) @@ -39,8 +38,20 @@ struct DivideIntegralByConstantImpl return; } + /// Division with too large divisor. + if (unlikely(b > std::numeric_limits::max() + || (std::is_signed_v && std::is_signed_v && b < std::numeric_limits::lowest()))) + { + for (size_t i = 0; i < size; ++i) + c_pos[i] = 0; + return; + } + #pragma GCC diagnostic pop + if (unlikely(static_cast(b) == 0)) + throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION); + libdivide::divider divider(b); const A * a_end = a_pos + size; diff --git a/src/Functions/isZeroOrNull.cpp b/src/Functions/isZeroOrNull.cpp new file mode 100644 index 000000000000..4554d50b0258 --- /dev/null +++ b/src/Functions/isZeroOrNull.cpp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; +} + +/// Returns 1 if argument is zero or NULL. +/// It can be used to negate filter in WHERE condition. +/// "WHERE isZeroOrNull(expr)" will return exactly the same rows that "WHERE expr" will filter out. +class FunctionIsZeroOrNull : public IFunction +{ +public: + static constexpr auto name = "isZeroOrNull"; + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; } + + DataTypePtr getReturnTypeImpl(const DataTypes & types) const override + { + if (!isNumber(removeNullable(types.at(0)))) + throw Exception("The argument of function " + getName() + " must have simple numeric type, possibly Nullable", ErrorCodes::BAD_ARGUMENTS); + + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + const ColumnPtr & input_column = block.getByPosition(arguments[0]).column; + + if (const ColumnNullable * input_column_nullable = checkAndGetColumn(input_column.get())) + { + const NullMap & null_map = input_column_nullable->getNullMapData(); + const IColumn * nested_column = &input_column_nullable->getNestedColumn(); + + if (!castTypeToEither< + ColumnUInt8, ColumnUInt16, ColumnUInt32, ColumnUInt64, + ColumnInt8, ColumnInt16, ColumnInt32, ColumnInt64, + ColumnFloat32, ColumnFloat64>( + nested_column, [&](const auto & column) + { + auto res = ColumnUInt8::create(input_rows_count); + processNullable(column.getData(), null_map, res->getData(), input_rows_count); + block.getByPosition(result).column = std::move(res); + return true; + })) + { + throw Exception("The argument of function " + getName() + " must have simple numeric type, possibly Nullable", ErrorCodes::ILLEGAL_COLUMN); + } + } + else + { + if (!castTypeToEither< + ColumnUInt8, ColumnUInt16, ColumnUInt32, ColumnUInt64, + ColumnInt8, ColumnInt16, ColumnInt32, ColumnInt64, + ColumnFloat32, ColumnFloat64>( + input_column.get(), [&](const auto & column) + { + auto res = ColumnUInt8::create(input_rows_count); + processNotNullable(column.getData(), res->getData(), input_rows_count); + block.getByPosition(result).column = std::move(res); + return true; + })) + { + throw Exception("The argument of function " + getName() + " must have simple numeric type, possibly Nullable", ErrorCodes::ILLEGAL_COLUMN); + } + } + } + +private: + template + void processNotNullable(const InputData & input_data, ColumnUInt8::Container & result_data, size_t input_rows_count) + { + for (size_t i = 0; i < input_rows_count; ++i) + result_data[i] = !input_data[i]; + } + + template + void processNullable(const InputData & input_data, const NullMap & input_null_map, + ColumnUInt8::Container & result_data, size_t input_rows_count) + { + for (size_t i = 0; i < input_rows_count; ++i) + result_data[i] = input_null_map[i] || !input_data[i]; + } +}; + + +void registerFunctionIsZeroOrNull(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/modulo.cpp b/src/Functions/modulo.cpp index 3cfa2fa0c1a3..5b11213f4a94 100644 --- a/src/Functions/modulo.cpp +++ b/src/Functions/modulo.cpp @@ -27,12 +27,10 @@ struct ModuloByConstantImpl static NO_INLINE void vector_constant(const A * __restrict src, B b, ResultType * __restrict dst, size_t size) { - if (unlikely(b == 0)) - throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION); - #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wsign-compare" + /// Modulo with too small divisor. if (unlikely((std::is_signed_v && b == -1) || b == 1)) { for (size_t i = 0; i < size; ++i) @@ -40,8 +38,20 @@ struct ModuloByConstantImpl return; } + /// Modulo with too large divisor. + if (unlikely(b > std::numeric_limits::max() + || (std::is_signed_v && std::is_signed_v && b < std::numeric_limits::lowest()))) + { + for (size_t i = 0; i < size; ++i) + dst[i] = src[i]; + return; + } + #pragma GCC diagnostic pop + if (unlikely(static_cast(b) == 0)) + throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION); + libdivide::divider divider(b); /// Here we failed to make the SSE variant from libdivide give an advantage. diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index 68609af91024..9c7955c04759 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -41,6 +41,7 @@ class FunctionMultiIf final : public FunctionIfBase bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } bool useDefaultImplementationForNulls() const override { return false; } + ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t number_of_arguments) const override { ColumnNumbers args; @@ -72,7 +73,6 @@ class FunctionMultiIf final : public FunctionIfBase throw Exception{"Invalid number of arguments for function " + getName(), ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - for_conditions([&](const DataTypePtr & arg) { const IDataType * nested_type; diff --git a/src/Functions/neighbor.cpp b/src/Functions/neighbor.cpp index 1080507ced5e..0203f9109203 100644 --- a/src/Functions/neighbor.cpp +++ b/src/Functions/neighbor.cpp @@ -12,6 +12,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ARGUMENT_OUT_OF_BOUND; } // Implements function, giving value for column within range of given @@ -113,6 +114,10 @@ class FunctionNeighbor : public IFunction Int64 offset = offset_column->getInt(0); + /// Protection from possible overflow. + if (unlikely(offset > (1 << 30) || offset < -(1 << 30))) + throw Exception("Too large offset: " + toString(offset) + " in function " + getName(), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + auto result_column = result_type->createColumn(); auto insert_range_from = [&](bool is_const, const ColumnPtr & src, Int64 begin, Int64 size) @@ -147,7 +152,9 @@ class FunctionNeighbor : public IFunction if (offset == 0) { /// Degenerate case, just copy source column as is. - block.getByPosition(result).column = source_is_constant ? ColumnConst::create(source_column_casted, input_rows_count) : source_column_casted; + block.getByPosition(result).column = source_is_constant + ? ColumnConst::create(source_column_casted, input_rows_count) + : source_column_casted; } else if (offset > 0) { @@ -168,7 +175,13 @@ class FunctionNeighbor : public IFunction for (size_t row = 0; row < input_rows_count; ++row) { - Int64 src_idx = row + offset_column->getInt(offset_is_constant ? 0 : row); + Int64 offset = offset_column->getInt(offset_is_constant ? 0 : row); + + /// Protection from possible overflow. + if (unlikely(offset > (1 << 30) || offset < -(1 << 30))) + throw Exception("Too large offset: " + toString(offset) + " in function " + getName(), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + Int64 src_idx = row + offset; if (src_idx >= 0 && src_idx < Int64(input_rows_count)) result_column->insertFrom(*source_column_casted, source_is_constant ? 0 : src_idx); diff --git a/src/Functions/registerFunctionsNull.cpp b/src/Functions/registerFunctionsNull.cpp index e8894e199077..238133fbb67f 100644 --- a/src/Functions/registerFunctionsNull.cpp +++ b/src/Functions/registerFunctionsNull.cpp @@ -10,6 +10,7 @@ void registerFunctionIfNull(FunctionFactory & factory); void registerFunctionNullIf(FunctionFactory & factory); void registerFunctionAssumeNotNull(FunctionFactory & factory); void registerFunctionToNullable(FunctionFactory & factory); +void registerFunctionIsZeroOrNull(FunctionFactory & factory); void registerFunctionsNull(FunctionFactory & factory) @@ -21,6 +22,7 @@ void registerFunctionsNull(FunctionFactory & factory) registerFunctionNullIf(factory); registerFunctionAssumeNotNull(factory); registerFunctionToNullable(factory); + registerFunctionIsZeroOrNull(factory); } } diff --git a/src/Functions/visitParamExtractRaw.cpp b/src/Functions/visitParamExtractRaw.cpp index 5eeb36286a54..84d887ab5c7f 100644 --- a/src/Functions/visitParamExtractRaw.cpp +++ b/src/Functions/visitParamExtractRaw.cpp @@ -22,6 +22,14 @@ struct ExtractRaw expects_end.pop_back(); current_expect_end = expects_end.empty() ? 0 : expects_end.back(); } + else if (current_expect_end == '"') + { + /// skip backslash + if (*pos == '\\' && pos + 1 < end && pos[1] == '"') + { + pos++; + } + } else { switch (*pos) @@ -38,11 +46,6 @@ struct ExtractRaw current_expect_end = '"'; expects_end.push_back(current_expect_end); break; - case '\\': - /// skip backslash - if (pos + 1 < end && pos[1] == '"') - pos++; - break; default: if (!current_expect_end && (*pos == ',' || *pos == '}')) { diff --git a/src/IO/ReadBufferAIO.cpp b/src/IO/ReadBufferAIO.cpp index ffe8183f0057..8b01b67c0c05 100644 --- a/src/IO/ReadBufferAIO.cpp +++ b/src/IO/ReadBufferAIO.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -95,11 +96,8 @@ bool ReadBufferAIO::nextImpl() if (profile_callback) watch.emplace(clock_type); - if (!is_aio) - { + if (!is_pending_read) synchronousRead(); - is_aio = true; - } else receive(); @@ -215,7 +213,9 @@ void ReadBufferAIO::synchronousRead() void ReadBufferAIO::receive() { if (!waitForAIOCompletion()) - return; + { + throw Exception("Trying to receive data from AIO, but nothing was queued. It's a bug", ErrorCodes::LOGICAL_ERROR); + } finalize(); } @@ -224,8 +224,6 @@ void ReadBufferAIO::skip() if (!waitForAIOCompletion()) return; - is_aio = false; - /// @todo I presume this assignment is redundant since waitForAIOCompletion() performs a similar one // bytes_read = future_bytes_read.get(); if ((bytes_read < 0) || (static_cast(bytes_read) < region_left_padding)) @@ -274,6 +272,9 @@ void ReadBufferAIO::prepare() region_aligned_size = region_aligned_end - region_aligned_begin; buffer_begin = fill_buffer.internalBuffer().begin(); + + /// Unpoison because msan doesn't instrument linux AIO + __msan_unpoison(buffer_begin, fill_buffer.internalBuffer().size()); } void ReadBufferAIO::finalize() diff --git a/src/IO/ReadBufferAIO.h b/src/IO/ReadBufferAIO.h index 2dd11b440956..d351ce1a4c78 100644 --- a/src/IO/ReadBufferAIO.h +++ b/src/IO/ReadBufferAIO.h @@ -100,8 +100,6 @@ class ReadBufferAIO : public ReadBufferFromFileBase bool is_eof = false; /// At least one read request was sent. bool is_started = false; - /// Is the operation asynchronous? - bool is_aio = false; /// Did the asynchronous operation fail? bool aio_failed = false; diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index e17ddae9566e..32c3c53b5288 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -275,8 +275,11 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf) switch (*buf.position()) { case '+': + { break; + } case '-': + { if constexpr (is_signed_v) negative = true; else @@ -287,6 +290,7 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf) return ReturnType(false); } break; + } case '0': [[fallthrough]]; case '1': [[fallthrough]]; case '2': [[fallthrough]]; @@ -297,20 +301,27 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf) case '7': [[fallthrough]]; case '8': [[fallthrough]]; case '9': + { if constexpr (check_overflow == ReadIntTextCheckOverflow::CHECK_OVERFLOW) { - // perform relativelly slow overflow check only when number of decimal digits so far is close to the max for given type. - if (buf.count() - initial_pos >= std::numeric_limits::max_digits10) + /// Perform relativelly slow overflow check only when + /// number of decimal digits so far is close to the max for given type. + /// Example: 20 * 10 will overflow Int8. + + if (buf.count() - initial_pos + 1 >= std::numeric_limits::max_digits10) { - if (common::mulOverflow(res, static_cast(10), res) - || common::addOverflow(res, static_cast(*buf.position() - '0'), res)) + T signed_res = res; + if (common::mulOverflow(signed_res, 10, signed_res) + || common::addOverflow(signed_res, (*buf.position() - '0'), signed_res)) return ReturnType(false); + res = signed_res; break; } } res *= 10; res += *buf.position() - '0'; break; + } default: goto end; } @@ -318,7 +329,23 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf) } end: - x = negative ? -res : res; + if (!negative) + { + x = res; + } + else + { + if constexpr (check_overflow == ReadIntTextCheckOverflow::CHECK_OVERFLOW) + { + x = res; + if (common::mulOverflow(x, -1, x)) + return ReturnType(false); + } + else + { + x = -res; + } + } return ReturnType(true); } @@ -658,35 +685,34 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re return ReturnType(false); } - DB::DecimalUtils::DecimalComponents c{static_cast(whole), 0}; + DB::DecimalUtils::DecimalComponents components{static_cast(whole), 0}; if (!buf.eof() && *buf.position() == '.') { - buf.ignore(1); // skip separator - const auto pos_before_fractional = buf.count(); - if (!tryReadIntText(c.fractional, buf)) - { - return ReturnType(false); - } - - // Adjust fractional part to the scale, since decimalFromComponents knows nothing - // about convention of ommiting trailing zero on fractional part - // and assumes that fractional part value is less than 10^scale. + ++buf.position(); - // If scale is 3, but we read '12', promote fractional part to '120'. - // And vice versa: if we read '1234', denote it to '123'. - const auto fractional_length = static_cast(buf.count() - pos_before_fractional); - if (const auto adjust_scale = static_cast(scale) - fractional_length; adjust_scale > 0) + /// Read digits, up to 'scale' positions. + for (size_t i = 0; i < scale; ++i) { - c.fractional *= common::exp10_i64(adjust_scale); - } - else if (adjust_scale < 0) - { - c.fractional /= common::exp10_i64(-1 * adjust_scale); + if (!buf.eof() && isNumericASCII(*buf.position())) + { + components.fractional *= 10; + components.fractional += *buf.position() - '0'; + ++buf.position(); + } + else + { + /// Adjust to scale. + components.fractional *= 10; + } } + + /// Ignore digits that are out of precision. + while (!buf.eof() && isNumericASCII(*buf.position())) + ++buf.position(); } - datetime64 = DecimalUtils::decimalFromComponents(c, scale); + datetime64 = DecimalUtils::decimalFromComponents(components, scale); return ReturnType(true); } diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index f5e6f91537d4..0c4e9cdf3df3 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -152,7 +152,8 @@ namespace detail public: using OutStreamCallback = std::function; - explicit ReadWriteBufferFromHTTPBase(UpdatableSessionPtr session_, + explicit ReadWriteBufferFromHTTPBase( + UpdatableSessionPtr session_, Poco::URI uri_, const std::string & method_ = {}, OutStreamCallback out_stream_callback_ = {}, @@ -241,9 +242,9 @@ class ReadWriteBufferFromHTTP : public detail::ReadWriteBufferFromHTTPBase #include +#include +#include +#include #include #include @@ -32,6 +35,7 @@ namespace ErrorCodes extern const int CANNOT_FSYNC; extern const int CANNOT_SEEK_THROUGH_FILE; extern const int CANNOT_TRUNCATE_FILE; + extern const int CANNOT_FSTAT; } @@ -127,4 +131,14 @@ void WriteBufferFromFileDescriptor::truncate(off_t length) throwFromErrnoWithPath("Cannot truncate file " + getFileName(), getFileName(), ErrorCodes::CANNOT_TRUNCATE_FILE); } + +off_t WriteBufferFromFileDescriptor::size() +{ + struct stat buf; + int res = fstat(fd, &buf); + if (-1 == res) + throwFromErrnoWithPath("Cannot execute fstat " + getFileName(), getFileName(), ErrorCodes::CANNOT_FSTAT); + return buf.st_size; +} + } diff --git a/src/IO/WriteBufferFromFileDescriptor.h b/src/IO/WriteBufferFromFileDescriptor.h index d7df04ee9409..18c0ac64f633 100644 --- a/src/IO/WriteBufferFromFileDescriptor.h +++ b/src/IO/WriteBufferFromFileDescriptor.h @@ -44,6 +44,8 @@ class WriteBufferFromFileDescriptor : public WriteBufferFromFileBase off_t seek(off_t offset, int whence); void truncate(off_t length); + + off_t size(); }; } diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 79c660689d79..a96f8837bd0d 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -375,7 +375,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & } SetPtr prepared_set; - if (functionIsInOrGlobalInOperator(node.name)) + if (checkFunctionIsInOrGlobalInOperator(node)) { /// Let's find the type of the first argument (then getActionsImpl will be called again and will not affect anything). visit(node.arguments->children.at(0), data); @@ -451,7 +451,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & /// Select the name in the next cycle. argument_names.emplace_back(); } - else if (functionIsInOrGlobalInOperator(node.name) && arg == 1 && prepared_set) + else if (checkFunctionIsInOrGlobalInOperator(node) && arg == 1 && prepared_set) { ColumnWithTypeAndName column; column.type = std::make_shared(); @@ -510,7 +510,8 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & if (data.only_consts) arguments_present = false; else - throw Exception("Unknown identifier: " + name, ErrorCodes::UNKNOWN_IDENTIFIER); + throw Exception("Unknown identifier: " + name + " there are columns: " + data.getSampleBlock().dumpNames(), + ErrorCodes::UNKNOWN_IDENTIFIER); } } } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 774963e725de..3336181de098 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -29,6 +29,7 @@ #include #include #include +#include namespace ProfileEvents @@ -128,11 +129,11 @@ Block Aggregator::getHeader(bool final) const if (final) { - for (size_t i = 0; i < params.aggregates_size; ++i) + for (const auto & aggregate : params.aggregates) { - auto & elem = res.getByPosition(params.keys_size + i); + auto & elem = res.getByName(aggregate.column_name); - elem.type = params.aggregates[i].function->getReturnType(); + elem.type = aggregate.function->getReturnType(); elem.column = elem.type->createColumn(); } } @@ -964,6 +965,73 @@ void Aggregator::convertToBlockImpl( data.clearAndShrink(); } + +template +inline void Aggregator::insertAggregatesIntoColumns( + Mapped & mapped, + MutableColumns & final_aggregate_columns) const +{ + /** Final values of aggregate functions are inserted to columns. + * Then states of aggregate functions, that are not longer needed, are destroyed. + * + * We mark already destroyed states with "nullptr" in data, + * so they will not be destroyed in destructor of Aggregator + * (other values will be destroyed in destructor in case of exception). + * + * But it becomes tricky, because we have multiple aggregate states pointed by a single pointer in data. + * So, if exception is thrown in the middle of moving states for different aggregate functions, + * we have to catch exceptions and destroy all the states that are no longer needed, + * to keep the data in consistent state. + * + * It is also tricky, because there are aggregate functions with "-State" modifier. + * When we call "insertResultInto" for them, they insert a pointer to the state to ColumnAggregateFunction + * and ColumnAggregateFunction will take ownership of this state. + * So, for aggregate functions with "-State" modifier, the state must not be destroyed + * after it has been transferred to ColumnAggregateFunction. + * But we should mark that the data no longer owns these states. + */ + + size_t insert_i = 0; + std::exception_ptr exception; + + try + { + /// Insert final values of aggregate functions into columns. + for (; insert_i < params.aggregates_size; ++insert_i) + aggregate_functions[insert_i]->insertResultInto( + mapped + offsets_of_aggregate_states[insert_i], + *final_aggregate_columns[insert_i]); + } + catch (...) + { + exception = std::current_exception(); + } + + /** Destroy states that are no longer needed. This loop does not throw. + * + * Don't destroy states for "-State" aggregate functions, + * because the ownership of this state is transferred to ColumnAggregateFunction + * and ColumnAggregateFunction will take care. + * + * But it's only for states that has been transferred to ColumnAggregateFunction + * before exception has been thrown; + */ + for (size_t destroy_i = 0; destroy_i < params.aggregates_size; ++destroy_i) + { + /// If ownership was not transferred to ColumnAggregateFunction. + if (!(destroy_i < insert_i && aggregate_functions[destroy_i]->isState())) + aggregate_functions[destroy_i]->destroy( + mapped + offsets_of_aggregate_states[destroy_i]); + } + + /// Mark the cell as destroyed so it will not be destroyed in destructor. + mapped = nullptr; + + if (exception) + std::rethrow_exception(exception); +} + + template void NO_INLINE Aggregator::convertToBlockImplFinal( Method & method, @@ -976,25 +1044,15 @@ void NO_INLINE Aggregator::convertToBlockImplFinal( if (data.hasNullKeyData()) { key_columns[0]->insertDefault(); - - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->insertResultInto( - data.getNullKeyData() + offsets_of_aggregate_states[i], - *final_aggregate_columns[i]); + insertAggregatesIntoColumns(data.getNullKeyData(), final_aggregate_columns); } } data.forEachValue([&](const auto & key, auto & mapped) { method.insertKeyIntoColumns(key, key_columns, key_sizes); - - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->insertResultInto( - mapped + offsets_of_aggregate_states[i], - *final_aggregate_columns[i]); + insertAggregatesIntoColumns(mapped, final_aggregate_columns); }); - - destroyImpl(data); } template @@ -1012,6 +1070,8 @@ void NO_INLINE Aggregator::convertToBlockImplNotFinal( for (size_t i = 0; i < params.aggregates_size; ++i) aggregate_columns[i]->push_back(data.getNullKeyData() + offsets_of_aggregate_states[i]); + + data.getNullKeyData() = nullptr; } } @@ -1052,7 +1112,8 @@ Block Aggregator::prepareBlockAndFill( { if (!final) { - aggregate_columns[i] = header.safeGetByPosition(i + params.keys_size).type->createColumn(); + const auto & aggregate_column_name = params.aggregates[i].column_name; + aggregate_columns[i] = header.getByName(aggregate_column_name).type->createColumn(); /// The ColumnAggregateFunction column captures the shared ownership of the arena with the aggregate function states. ColumnAggregateFunction & column_aggregate_func = assert_cast(*aggregate_columns[i]); @@ -1071,10 +1132,17 @@ Block Aggregator::prepareBlockAndFill( if (aggregate_functions[i]->isState()) { /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. - ColumnAggregateFunction & column_aggregate_func = assert_cast(*final_aggregate_columns[i]); + if (auto * column_aggregate_func = typeid_cast(final_aggregate_columns[i].get())) + for (auto & pool : data_variants.aggregates_pools) + column_aggregate_func->addArena(pool); - for (auto & pool : data_variants.aggregates_pools) - column_aggregate_func.addArena(pool); + /// Aggregate state can be wrapped into array if aggregate function ends with -Resample combinator. + final_aggregate_columns[i]->forEachSubcolumn([&data_variants](auto & subcolumn) + { + if (auto * column_aggregate_func = typeid_cast(subcolumn.get())) + for (auto & pool : data_variants.aggregates_pools) + column_aggregate_func->addArena(pool); + }); } } } @@ -1088,10 +1156,11 @@ Block Aggregator::prepareBlockAndFill( for (size_t i = 0; i < params.aggregates_size; ++i) { + const auto & aggregate_column_name = params.aggregates[i].column_name; if (final) - res.getByPosition(i + params.keys_size).column = std::move(final_aggregate_columns[i]); + res.getByName(aggregate_column_name).column = std::move(final_aggregate_columns[i]); else - res.getByPosition(i + params.keys_size).column = std::move(aggregate_columns[i]); + res.getByName(aggregate_column_name).column = std::move(aggregate_columns[i]); } /// Change the size of the columns-constants in the block. @@ -1118,16 +1187,16 @@ Block Aggregator::prepareBlockAndFillWithoutKey(AggregatedDataVariants & data_va { AggregatedDataWithoutKey & data = data_variants.without_key; - for (size_t i = 0; i < params.aggregates_size; ++i) + if (!final_) { - if (!final_) + for (size_t i = 0; i < params.aggregates_size; ++i) aggregate_columns[i]->push_back(data + offsets_of_aggregate_states[i]); - else - aggregate_functions[i]->insertResultInto(data + offsets_of_aggregate_states[i], *final_aggregate_columns[i]); - } - - if (!final_) data = nullptr; + } + else + { + insertAggregatesIntoColumns(data, final_aggregate_columns); + } if (params.overflow_row) for (size_t i = 0; i < params.keys_size; ++i) @@ -1810,7 +1879,10 @@ void NO_INLINE Aggregator::mergeStreamsImplCase( key_columns[i] = block.safeGetByPosition(i).column.get(); for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns[i] = &typeid_cast(*block.safeGetByPosition(params.keys_size + i).column).getData(); + { + const auto & aggregate_column_name = params.aggregates[i].column_name; + aggregate_columns[i] = &typeid_cast(*block.getByName(aggregate_column_name).column).getData(); + } typename Method::State state(key_columns, key_sizes, aggregation_state_cache); @@ -1886,7 +1958,10 @@ void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl( /// Remember the columns we will work with for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns[i] = &typeid_cast(*block.safeGetByPosition(params.keys_size + i).column).getData(); + { + const auto & aggregate_column_name = params.aggregates[i].column_name; + aggregate_columns[i] = &typeid_cast(*block.getByName(aggregate_column_name).column).getData(); + } AggregatedDataWithoutKey & res = result.without_key; if (!res) @@ -2308,8 +2383,7 @@ void NO_INLINE Aggregator::destroyImpl(Table & table) const return; for (size_t i = 0; i < params.aggregates_size; ++i) - if (!aggregate_functions[i]->isState()) - aggregate_functions[i]->destroy(data + offsets_of_aggregate_states[i]); + aggregate_functions[i]->destroy(data + offsets_of_aggregate_states[i]); data = nullptr; }); @@ -2323,8 +2397,7 @@ void Aggregator::destroyWithoutKey(AggregatedDataVariants & result) const if (nullptr != res_data) { for (size_t i = 0; i < params.aggregates_size; ++i) - if (!aggregate_functions[i]->isState()) - aggregate_functions[i]->destroy(res_data + offsets_of_aggregate_states[i]); + aggregate_functions[i]->destroy(res_data + offsets_of_aggregate_states[i]); res_data = nullptr; } diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 0fa9bd509bde..9c0dd2536672 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -1140,6 +1140,11 @@ class Aggregator MutableColumns & final_aggregate_columns, bool final) const; + template + void insertAggregatesIntoColumns( + Mapped & mapped, + MutableColumns & final_aggregate_columns) const; + template void convertToBlockImplFinal( Method & method, diff --git a/src/Interpreters/BloomFilterHash.h b/src/Interpreters/BloomFilterHash.h index e74114337812..43f5d7b5e873 100644 --- a/src/Interpreters/BloomFilterHash.h +++ b/src/Interpreters/BloomFilterHash.h @@ -196,18 +196,17 @@ struct BloomFilterHash const ColumnString::Chars & data = index_column->getChars(); const ColumnString::Offsets & offsets = index_column->getOffsets(); - ColumnString::Offset current_offset = pos; for (size_t index = 0, size = vec.size(); index < size; ++index) { + ColumnString::Offset current_offset = offsets[index + pos - 1]; + size_t length = offsets[index + pos] - current_offset - 1 /* terminating zero */; UInt64 city_hash = CityHash_v1_0_2::CityHash64( - reinterpret_cast(&data[current_offset]), offsets[index + pos] - current_offset - 1); + reinterpret_cast(&data[current_offset]), length); if constexpr (is_first) vec[index] = city_hash; else vec[index] = CityHash_v1_0_2::Hash128to64(CityHash_v1_0_2::uint128(vec[index], city_hash)); - - current_offset = offsets[index + pos]; } } else if (const auto * fixed_string_index_column = typeid_cast(column)) diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index f1790249cc02..3231007e96c2 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -139,8 +139,11 @@ String Cluster::Address::toFullString(bool use_compact_format) const { if (use_compact_format) { - return ((shard_index == 0) ? "" : "shard" + std::to_string(shard_index)) - + ((replica_index == 0) ? "" : "_replica" + std::to_string(replica_index)); + if (shard_index == 0 || replica_index == 0) + // shard_num/replica_num like in system.clusters table + throw Exception("shard_num/replica_num cannot be zero", ErrorCodes::LOGICAL_ERROR); + + return "shard" + std::to_string(shard_index) + "_replica" + std::to_string(replica_index); } else { @@ -284,7 +287,7 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, const Setting const auto & prefix = config_prefix + key; const auto weight = config.getInt(prefix + ".weight", default_weight); - addresses.emplace_back(config, prefix); + addresses.emplace_back(config, prefix, current_shard_num, 1); const auto & address = addresses.back(); ShardInfo info; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 4c05c047cf55..68ea6e2d6b33 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -156,6 +156,8 @@ class NamedSessions if (!session.unique()) throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); + session->context.client_info = context.client_info; + return session; } @@ -852,7 +854,8 @@ void Context::calculateUserSettings() setProfile(default_profile_name); /// 3) Apply settings from current user - setProfile(profile); + if (!profile.empty()) + setProfile(profile); /// 4) Recalculate access rights if it's necessary. if ((settings.readonly != old_readonly) || (settings.allow_ddl != old_allow_ddl) || (settings.allow_introspection_functions != old_allow_introspection_functions)) diff --git a/src/Interpreters/CrossToInnerJoinVisitor.cpp b/src/Interpreters/CrossToInnerJoinVisitor.cpp index e5a011e7c14b..08429201ebcc 100644 --- a/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -240,9 +240,13 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tabl size_t num_array_join = 0; size_t num_using = 0; - for (auto & child : tables->children) + // For diagnostic messages. + std::vector tables_with_using; + tables_with_using.reserve(num_tables); + + for (const auto & child : tables->children) { - auto table_element = child->as(); + auto * table_element = child->as(); if (!table_element) throw Exception("Logical error: TablesInSelectQueryElement expected", ErrorCodes::LOGICAL_ERROR); @@ -258,6 +262,7 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tabl if (t.hasUsing()) { ++num_using; + tables_with_using.push_back(table_element); continue; } @@ -276,7 +281,11 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tabl } if (num_using && (num_tables - num_array_join) > 2) - throw Exception("Multiple CROSS/COMMA JOIN do not support USING", ErrorCodes::NOT_IMPLEMENTED); + { + throw Exception("Multiple CROSS/COMMA JOIN do not support USING (while " + "processing '" + IAST::formatForErrorMessage(tables_with_using) + "')", + ErrorCodes::NOT_IMPLEMENTED); + } return !(num_array_join || num_using); } diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 221ead198746..7cc0ca329ea0 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -180,7 +180,7 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTFunction & func, ASTPtr & as /// But if an argument is not subquery, than deeper may be scalar subqueries and we need to descend in them. std::vector out; - if (functionIsInOrGlobalInOperator(func.name)) + if (checkFunctionIsInOrGlobalInOperator(func)) { for (auto & child : ast->children) { diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 67899995c4ac..953e9fae469e 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -338,11 +338,7 @@ void ExpressionAction::execute(Block & block, bool dry_run, ExtraBlockPtr & not_ { ColumnNumbers arguments(argument_names.size()); for (size_t i = 0; i < argument_names.size(); ++i) - { - if (!block.has(argument_names[i])) - throw Exception("Not found column: '" + argument_names[i] + "'", ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK); arguments[i] = block.getPositionByName(argument_names[i]); - } size_t num_columns_without_result = block.columns(); block.insert({ nullptr, result_type, result_name}); diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index d49560866ffd..73acc6da0c33 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -74,6 +74,8 @@ namespace ErrorCodes extern const int UNKNOWN_IDENTIFIER; extern const int ILLEGAL_PREWHERE; extern const int LOGICAL_ERROR; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER; } namespace @@ -110,7 +112,7 @@ bool sanitizeBlock(Block & block) return false; col.column = col.type->createColumn(); } - else if (isColumnConst(*col.column) && !col.column->empty()) + else if (!col.column->empty()) col.column = col.column->cloneEmpty(); } return true; @@ -612,6 +614,11 @@ bool SelectQueryExpressionAnalyzer::appendPrewhere( step.required_output.push_back(prewhere_column_name); step.can_remove_required_output.push_back(true); + auto filter_type = step.actions->getSampleBlock().getByName(prewhere_column_name).type; + if (!filter_type->canBeUsedInBooleanContext()) + throw Exception("Invalid type for filter in PREWHERE: " + filter_type->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER); + { /// Remove unused source_columns from prewhere actions. auto tmp_actions = std::make_shared(sourceColumns(), context); @@ -694,11 +701,17 @@ bool SelectQueryExpressionAnalyzer::appendWhere(ExpressionActionsChain & chain, initChain(chain, sourceColumns()); ExpressionActionsChain::Step & step = chain.steps.back(); - step.required_output.push_back(select_query->where()->getColumnName()); + auto where_column_name = select_query->where()->getColumnName(); + step.required_output.push_back(where_column_name); step.can_remove_required_output = {true}; getRootActions(select_query->where(), only_types, step.actions); + auto filter_type = step.actions->getSampleBlock().getByName(where_column_name).type; + if (!filter_type->canBeUsedInBooleanContext()) + throw Exception("Invalid type for filter in WHERE: " + filter_type->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER); + return true; } diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 7caed9a69e17..6c8e24c8b7c0 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -205,7 +205,9 @@ struct ExpressionAnalysisResult const FilterInfoPtr & filter_info, const Block & source_header); + /// Filter for row-level security. bool hasFilter() const { return filter_info.get(); } + bool hasJoin() const { return before_join.get(); } bool hasPrewhere() const { return prewhere_info.get(); } bool hasWhere() const { return before_where.get(); } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 8593181b28ad..296ce429c2a2 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -118,7 +118,8 @@ namespace ErrorCodes } /// Assumes `storage` is set and the table filter (row-level security) is not empty. -String InterpreterSelectQuery::generateFilterActions(ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns) const +String InterpreterSelectQuery::generateFilterActions( + ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns) const { const auto & db_name = table_id.getDatabaseName(); const auto & table_name = table_id.getTableName(); @@ -515,8 +516,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl(bool try_move_to_prewhere) second_stage, options.only_analyze, filter_info, - source_header - ); + source_header); if (options.to_stage == QueryProcessingStage::Enum::FetchColumns) { @@ -1042,7 +1042,7 @@ void InterpreterSelectQuery::executeFetchColumns( { if (!settings.optimize_trivial_count_query || !syntax_analyzer_result->maybe_optimize_trivial_count || !storage || query.sample_size() || query.sample_offset() || query.final() || query.prewhere() || query.where() - || !query_analyzer->hasAggregation() || processing_stage != QueryProcessingStage::FetchColumns) + || filter_info || !query_analyzer->hasAggregation() || processing_stage != QueryProcessingStage::FetchColumns) return {}; const AggregateDescriptions & aggregates = query_analyzer->aggregates(); @@ -1272,6 +1272,9 @@ void InterpreterSelectQuery::executeFetchColumns( + ", maximum: " + settings.max_columns_to_read.toString(), ErrorCodes::TOO_MANY_COLUMNS); + /// General limit for the number of threads. + pipeline.setMaxThreads(settings.max_threads); + /** With distributed query processing, almost no computations are done in the threads, * but wait and receive data from remote servers. * If we have 20 remote servers, and max_threads = 8, then it would not be very good @@ -1459,16 +1462,19 @@ void InterpreterSelectQuery::executeFetchColumns( * But limits on data size to read and maximum execution time are reasonable to check both on initiator and * additionally on each remote server, because these limits are checked per block of data processed, * and remote servers may process way more blocks of data than are received by initiator. + * + * The limits to throttle maximum execution speed is also checked on all servers. */ if (options.to_stage == QueryProcessingStage::Complete) { limits.speed_limits.min_execution_rps = settings.min_execution_speed; - limits.speed_limits.max_execution_rps = settings.max_execution_speed; limits.speed_limits.min_execution_bps = settings.min_execution_speed_bytes; - limits.speed_limits.max_execution_bps = settings.max_execution_speed_bytes; - limits.speed_limits.timeout_before_checking_execution_speed = settings.timeout_before_checking_execution_speed; } + limits.speed_limits.max_execution_rps = settings.max_execution_speed; + limits.speed_limits.max_execution_bps = settings.max_execution_speed_bytes; + limits.speed_limits.timeout_before_checking_execution_speed = settings.timeout_before_checking_execution_speed; + auto quota = context->getQuota(); for (auto & stream : streams) @@ -1493,8 +1499,8 @@ void InterpreterSelectQuery::executeFetchColumns( if constexpr (pipeline_with_processors) { - if (streams.size() == 1 || pipes.size() == 1) - pipeline.setMaxThreads(streams.size()); + if (!storage->isView() && (streams.size() == 1 || pipes.size() == 1)) + pipeline.setMaxThreads(1); /// Unify streams. They must have same headers. if (streams.size() > 1) diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index f30d5efe1cc3..4136ae44d768 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -209,7 +209,8 @@ class InterpreterSelectQuery : public IInterpreter void executeSubqueriesInSetsAndJoins(QueryPipeline & pipeline, const std::unordered_map & subqueries_for_sets); void executeMergeSorted(QueryPipeline & pipeline, const SortDescription & sort_description, UInt64 limit); - String generateFilterActions(ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns = {}) const; + String generateFilterActions( + ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns = {}) const; /// Add ConvertingBlockInputStream to specified header. void unifyStreams(Pipeline & pipeline, Block header); @@ -241,6 +242,7 @@ class InterpreterSelectQuery : public IInterpreter /// Is calculated in getSampleBlock. Is used later in readImpl. ExpressionAnalysisResult analysis_result; + /// For row-level security. FilterInfoPtr filter_info; QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index feec18a1af31..5134bc98deb0 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -269,7 +269,7 @@ QueryPipeline InterpreterSelectWithUnionQuery::executeWithProcessors() if (!pipelines.empty()) { auto common_header = getCommonHeaderForUnion(headers); - main_pipeline.unitePipelines(std::move(pipelines), common_header, *context); + main_pipeline.unitePipelines(std::move(pipelines), common_header, *context, context->getSettingsRef().max_threads); } main_pipeline.addInterpreterContext(context); diff --git a/src/Interpreters/Join.cpp b/src/Interpreters/Join.cpp index e60f532d5175..aab05c872e3e 100644 --- a/src/Interpreters/Join.cpp +++ b/src/Interpreters/Join.cpp @@ -31,6 +31,8 @@ namespace ErrorCodes { extern const int BAD_TYPE_OF_FIELD; extern const int NOT_IMPLEMENTED; + extern const int NO_SUCH_COLUMN_IN_TABLE; + extern const int INCOMPATIBLE_TYPE_OF_JOIN; extern const int UNSUPPORTED_JOIN_KEYS; extern const int LOGICAL_ERROR; extern const int SET_SIZE_LIMIT_EXCEEDED; @@ -91,7 +93,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, { if (nullable) { - JoinCommon::convertColumnToNullable(column); + JoinCommon::convertColumnToNullable(column, true); if (column.type->isNullable() && !negative_null_map.empty()) { MutableColumnPtr mutable_column = (*std::move(column.column)).mutate(); @@ -1125,7 +1127,7 @@ DataTypePtr Join::joinGetReturnType(const String & column_name) const std::shared_lock lock(data->rwlock); if (!sample_block_with_columns_to_add.has(column_name)) - throw Exception("StorageJoin doesn't contain column " + column_name, ErrorCodes::LOGICAL_ERROR); + throw Exception("StorageJoin doesn't contain column " + column_name, ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); return sample_block_with_columns_to_add.getByName(column_name).type; } @@ -1146,7 +1148,7 @@ void Join::joinGet(Block & block, const String & column_name) const std::shared_lock lock(data->rwlock); if (key_names_right.size() != 1) - throw Exception("joinGet only supports StorageJoin containing exactly one key", ErrorCodes::LOGICAL_ERROR); + throw Exception("joinGet only supports StorageJoin containing exactly one key", ErrorCodes::UNSUPPORTED_JOIN_KEYS); checkTypeOfKey(block, right_table_keys); @@ -1156,7 +1158,7 @@ void Join::joinGet(Block & block, const String & column_name) const joinGetImpl(block, column_name, std::get(data->maps)); } else - throw Exception("joinGet only supports StorageJoin of type Left Any", ErrorCodes::LOGICAL_ERROR); + throw Exception("joinGet only supports StorageJoin of type Left Any", ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN); } diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 58699dbe48af..a46b0092ecd1 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -246,7 +246,7 @@ struct ColumnAliasesMatcher if (!last_table) { IdentifierSemantic::coverName(node, alias); - node.setAlias(""); + node.setAlias({}); } } else if (node.compound()) diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index 0cf85486e224..d607b0bb0ea0 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -136,7 +136,7 @@ StoragePtr JoinedTables::getLeftTableStorage() return {}; if (isLeftTableFunction()) - return context.executeTableFunction(left_table_expression); + return context.getQueryContext().executeTableFunction(left_table_expression); if (left_db_and_table) { @@ -171,7 +171,22 @@ StoragePtr JoinedTables::getLeftTableStorage() bool JoinedTables::resolveTables() { tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context); - checkTablesWithColumns(tables_with_columns, context); + assert(tables_with_columns.size() == table_expressions.size()); + + const auto & settings = context.getSettingsRef(); + if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1) + { + for (size_t i = 0; i < tables_with_columns.size(); ++i) + { + const auto & t = tables_with_columns[i]; + if (t.table.table.empty() && t.table.alias.empty()) + { + throw Exception("No alias for subquery or table function in JOIN (set joined_subquery_requires_alias=0 to disable restriction). While processing '" + + table_expressions[i]->formatForErrorMessage() + "'", + ErrorCodes::ALIAS_REQUIRED); + } + } + } return !tables_with_columns.empty(); } diff --git a/src/Interpreters/MarkTableIdentifiersVisitor.cpp b/src/Interpreters/MarkTableIdentifiersVisitor.cpp index c68a7373680b..5b118d494e86 100644 --- a/src/Interpreters/MarkTableIdentifiersVisitor.cpp +++ b/src/Interpreters/MarkTableIdentifiersVisitor.cpp @@ -35,7 +35,7 @@ void MarkTableIdentifiersMatcher::visit(ASTTableExpression & table, ASTPtr &, Da void MarkTableIdentifiersMatcher::visit(const ASTFunction & func, ASTPtr &, Data & data) { /// `IN t` can be specified, where t is a table, which is equivalent to `IN (SELECT * FROM t)`. - if (functionIsInOrGlobalInOperator(func.name)) + if (checkFunctionIsInOrGlobalInOperator(func)) { auto & ast = func.arguments->children.at(1); if (auto opt_name = tryGetIdentifierName(ast)) diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 14afcdeb16a0..d353bd1ae474 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -343,7 +343,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) if (stages.empty() || !stages.back().column_to_updated.empty()) stages.emplace_back(context); - auto negated_predicate = makeASTFunction("not", command.predicate->clone()); + auto negated_predicate = makeASTFunction("isZeroOrNull", command.predicate->clone()); stages.back().filters.push_back(negated_predicate); } else if (command.type == MutationCommand::UPDATE) diff --git a/src/Interpreters/PredicateExpressionsOptimizer.cpp b/src/Interpreters/PredicateExpressionsOptimizer.cpp index 8791008ac09e..3ec94aa718e8 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -49,13 +49,6 @@ static ASTs splitConjunctionPredicate(const std::initializer_list { std::vector res; - auto remove_expression_at_index = [&res] (const size_t index) - { - if (index < res.size() - 1) - std::swap(res[index], res.back()); - res.pop_back(); - }; - for (const auto & predicate : predicates) { if (!predicate) @@ -65,14 +58,15 @@ static ASTs splitConjunctionPredicate(const std::initializer_list for (size_t idx = 0; idx < res.size();) { - const auto & expression = res.at(idx); + ASTPtr expression = res.at(idx); if (const auto * function = expression->as(); function && function->name == "and") { + res.erase(res.begin() + idx); + for (auto & child : function->arguments->children) res.emplace_back(child); - remove_expression_at_index(idx); continue; } ++idx; diff --git a/src/Interpreters/PredicateRewriteVisitor.cpp b/src/Interpreters/PredicateRewriteVisitor.cpp index a834e68172b9..1556db0f9031 100644 --- a/src/Interpreters/PredicateRewriteVisitor.cpp +++ b/src/Interpreters/PredicateRewriteVisitor.cpp @@ -76,7 +76,7 @@ static void cleanAliasAndCollectIdentifiers(ASTPtr & predicate, std::vectortryGetAlias(); !alias.empty()) - predicate->setAlias(""); + predicate->setAlias({}); if (ASTIdentifier * identifier = predicate->as()) identifiers.emplace_back(identifier); diff --git a/src/Interpreters/QueryNormalizer.cpp b/src/Interpreters/QueryNormalizer.cpp index cb084ee2cbcb..b514f7a023b8 100644 --- a/src/Interpreters/QueryNormalizer.cpp +++ b/src/Interpreters/QueryNormalizer.cpp @@ -76,20 +76,7 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data) if (it_alias != data.aliases.end() && current_alias != node.name) { if (!IdentifierSemantic::canBeAlias(node)) - { - /// This means that column had qualified name, which was translated (so, canBeAlias() returns false). - /// But there is an alias with the same name. So, let's use original name for that column. - /// If alias wasn't set, use original column name as alias. - /// That helps to avoid result set with columns which have same names but different values. - if (node.alias.empty()) - { - node.name.swap(node.alias); - node.restoreCompoundName(); - node.name.swap(node.alias); - } - return; - } /// We are alias for other column (node.name), but we are alias by /// ourselves to some other column diff --git a/src/Interpreters/ReplaceQueryParameterVisitor.cpp b/src/Interpreters/ReplaceQueryParameterVisitor.cpp index 5c29c722f88e..ec824ed0cce1 100644 --- a/src/Interpreters/ReplaceQueryParameterVisitor.cpp +++ b/src/Interpreters/ReplaceQueryParameterVisitor.cpp @@ -25,14 +25,18 @@ namespace ErrorCodes void ReplaceQueryParameterVisitor::visit(ASTPtr & ast) +{ + if (ast->as()) + visitQueryParameter(ast); + else + visitChildren(ast); +} + + +void ReplaceQueryParameterVisitor::visitChildren(ASTPtr & ast) { for (auto & child : ast->children) - { - if (child->as()) - visitQueryParameter(child); - else - visit(child); - } + visit(child); } const String & ReplaceQueryParameterVisitor::getParamValue(const String & name) @@ -49,6 +53,7 @@ void ReplaceQueryParameterVisitor::visitQueryParameter(ASTPtr & ast) const auto & ast_param = ast->as(); const String & value = getParamValue(ast_param.name); const String & type_name = ast_param.type; + String alias = ast_param.alias; const auto data_type = DataTypeFactory::instance().get(type_name); auto temp_column_ptr = data_type->createColumn(); @@ -63,6 +68,7 @@ void ReplaceQueryParameterVisitor::visitQueryParameter(ASTPtr & ast) + value.substr(0, read_buffer.count()), ErrorCodes::BAD_QUERY_PARAMETER); ast = addTypeConversionToAST(std::make_shared(temp_column[0]), type_name); + ast->setAlias(alias); } } diff --git a/src/Interpreters/ReplaceQueryParameterVisitor.h b/src/Interpreters/ReplaceQueryParameterVisitor.h index 1931d4c0ba8c..3a84cd22acd9 100644 --- a/src/Interpreters/ReplaceQueryParameterVisitor.h +++ b/src/Interpreters/ReplaceQueryParameterVisitor.h @@ -22,6 +22,7 @@ class ReplaceQueryParameterVisitor const NameToNameMap & query_parameters; const String & getParamValue(const String & name); void visitQueryParameter(ASTPtr & ast); + void visitChildren(ASTPtr & ast); }; } diff --git a/src/Interpreters/SyntaxAnalyzer.cpp b/src/Interpreters/SyntaxAnalyzer.cpp index e461319e576e..ef980df0d6b5 100644 --- a/src/Interpreters/SyntaxAnalyzer.cpp +++ b/src/Interpreters/SyntaxAnalyzer.cpp @@ -224,6 +224,7 @@ const std::unordered_set injective_function_names const std::unordered_set possibly_injective_function_names { + "dictGet", "dictGetString", "dictGetUInt8", "dictGetUInt16", @@ -301,10 +302,18 @@ void optimizeGroupBy(ASTSelectQuery * select_query, const NameSet & source_colum continue; } - const auto & dict_name = function->arguments->children[0]->as().value.safeGet(); - const auto & dict_ptr = context.getExternalDictionariesLoader().getDictionary(dict_name); - const auto & attr_name = function->arguments->children[1]->as().value.safeGet(); + const auto * dict_name_ast = function->arguments->children[0]->as(); + const auto * attr_name_ast = function->arguments->children[1]->as(); + if (!dict_name_ast || !attr_name_ast) + { + ++i; + continue; + } + + const auto & dict_name = dict_name_ast->value.safeGet(); + const auto & attr_name = attr_name_ast->value.safeGet(); + const auto & dict_ptr = context.getExternalDictionariesLoader().getDictionary(dict_name); if (!dict_ptr->isInjective(attr_name)) { ++i; diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 845627045320..44ad3ca37ac0 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -59,7 +60,6 @@ namespace ErrorCodes #define DBMS_SYSTEM_LOG_QUEUE_SIZE 1048576 -class Context; class QueryLog; class QueryThreadLog; class PartLog; @@ -342,7 +342,11 @@ void SystemLog::flushImpl(const std::vector & to_flush, insert->table = table_name; ASTPtr query_ptr(insert.release()); - InterpreterInsertQuery interpreter(query_ptr, context); + // we need query context to do inserts to target table with MV containing subqueries or joins + auto insert_context = Context(context); + insert_context.makeQueryContext(); + + InterpreterInsertQuery interpreter(query_ptr, insert_context); BlockIO io = interpreter.execute(); io.out->writePrefix(); diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 9d38c759fedf..2e7c63b93661 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -26,6 +26,7 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_IDENTIFIER; + extern const int UNSUPPORTED_JOIN_KEYS; extern const int LOGICAL_ERROR; } @@ -207,13 +208,11 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt if (tables_with_columns.empty()) throw Exception("An asterisk cannot be replaced with empty columns.", ErrorCodes::LOGICAL_ERROR); has_asterisk = true; - break; } else if (const auto * qa = child->as()) { visit(*qa, child, data); /// check if it's OK before rewrite has_asterisk = true; - break; } } @@ -296,20 +295,52 @@ void TranslateQualifiedNamesMatcher::extractJoinUsingColumns(const ASTPtr ast, D { String alias = key->tryGetAlias(); if (alias.empty()) - throw Exception("Logical error: expected identifier or alias, got: " + key->getID(), ErrorCodes::LOGICAL_ERROR); + throw Exception("Wrong key in USING. Expected identifier or alias, got: " + key->getID(), + ErrorCodes::UNSUPPORTED_JOIN_KEYS); data.join_using_columns.insert(alias); } } } -void RestoreQualifiedNamesData::visit(ASTIdentifier & identifier, ASTPtr & ast) + +void RestoreQualifiedNamesMatcher::Data::changeTable(ASTIdentifier & identifier) const +{ + auto match = IdentifierSemantic::canReferColumnToTable(identifier, distributed_table); + switch (match) + { + case IdentifierSemantic::ColumnMatch::AliasedTableName: + case IdentifierSemantic::ColumnMatch::TableName: + case IdentifierSemantic::ColumnMatch::DbAndTable: + IdentifierSemantic::setColumnLongName(identifier, remote_table); + break; + default: + break; + } +} + +bool RestoreQualifiedNamesMatcher::needChildVisit(ASTPtr &, const ASTPtr & child) +{ + /// Do not go into subqueries + if (child->as()) + return false; // NOLINT + return true; +} + +void RestoreQualifiedNamesMatcher::visit(ASTPtr & ast, Data & data) +{ + if (auto * t = ast->as()) + visit(*t, ast, data); +} + +void RestoreQualifiedNamesMatcher::visit(ASTIdentifier & identifier, ASTPtr &, Data & data) { if (IdentifierSemantic::getColumnName(identifier)) { if (IdentifierSemantic::getMembership(identifier)) { - ast = identifier.clone(); - ast->as()->restoreCompoundName(); + identifier.restoreCompoundName(); + if (data.rename) + data.changeTable(identifier); } } } diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.h b/src/Interpreters/TranslateQualifiedNamesVisitor.h index bcd032938aa9..b78a2688382a 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.h +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.h @@ -59,15 +59,25 @@ class TranslateQualifiedNamesMatcher /// It finds columns and translate their names to the normal form. Expand asterisks and qualified asterisks with column names. using TranslateQualifiedNamesVisitor = TranslateQualifiedNamesMatcher::Visitor; -/// Restore ASTIdentifiers to long form -struct RestoreQualifiedNamesData + + +/// Restore ASTIdentifiers to long form, change table name in case of distributed. +struct RestoreQualifiedNamesMatcher { - using TypeToVisit = ASTIdentifier; + struct Data + { + DatabaseAndTableWithAlias distributed_table; + DatabaseAndTableWithAlias remote_table; + bool rename = false; - void visit(ASTIdentifier & identifier, ASTPtr & ast); + void changeTable(ASTIdentifier & identifier) const; + }; + + static bool needChildVisit(ASTPtr & node, const ASTPtr & child); + static void visit(ASTPtr & ast, Data & data); + static void visit(ASTIdentifier & identifier, ASTPtr & ast, Data & data); }; -using RestoreQualifiedNamesMatcher = OneTypeMatcher; using RestoreQualifiedNamesVisitor = InDepthNodeVisitor; } diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index e5d468c88a67..5c667f2c14db 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -366,5 +366,16 @@ Field convertFieldToType(const Field & from_value, const IDataType & to_type, co return convertFieldToTypeImpl(from_value, to_type, from_type_hint); } +Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint) +{ + bool is_null = from_value.isNull(); + if (is_null && !to_type.isNullable()) + throw Exception(ErrorCodes::TYPE_MISMATCH, "Cannot convert NULL to {}", to_type.getName()); + Field converted = convertFieldToType(from_value, to_type, from_type_hint); + if (!is_null && converted.isNull()) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Cannot convert value{}: it cannot be represented as {}", + from_type_hint ? " from " + from_type_hint->getName() : "", to_type.getName()); + return converted; +} } diff --git a/src/Interpreters/convertFieldToType.h b/src/Interpreters/convertFieldToType.h index 801beddd876e..91b631d0e128 100644 --- a/src/Interpreters/convertFieldToType.h +++ b/src/Interpreters/convertFieldToType.h @@ -17,4 +17,7 @@ class IDataType; */ Field convertFieldToType(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint = nullptr); +/// Does the same, but throws ARGUMENT_OUT_OF_BOUND if value does not fall into the range. +Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint = nullptr); + } diff --git a/src/Interpreters/createBlockSelector.cpp b/src/Interpreters/createBlockSelector.cpp index 2b08ca0845c0..cacdf96ebe99 100644 --- a/src/Interpreters/createBlockSelector.cpp +++ b/src/Interpreters/createBlockSelector.cpp @@ -21,6 +21,8 @@ IColumn::Selector createBlockSelector( const std::vector & slots) { const auto total_weight = slots.size(); + assert(total_weight != 0); + size_t num_rows = column.size(); IColumn::Selector selector(num_rows); diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 7ef8d66d752d..395b4d2df02e 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -16,8 +16,14 @@ namespace ErrorCodes namespace JoinCommon { -void convertColumnToNullable(ColumnWithTypeAndName & column) +void convertColumnToNullable(ColumnWithTypeAndName & column, bool low_card_nullability) { + if (low_card_nullability && column.type->lowCardinality()) + { + column.column = recursiveRemoveLowCardinality(column.column); + column.type = recursiveRemoveLowCardinality(column.type); + } + if (column.type->isNullable() || !column.type->canBeInsideNullable()) return; diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index b69a0a4a993d..47fa082e7009 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -13,7 +13,7 @@ using ColumnRawPtrs = std::vector; namespace JoinCommon { -void convertColumnToNullable(ColumnWithTypeAndName & column); +void convertColumnToNullable(ColumnWithTypeAndName & column, bool low_card_nullability = false); void convertColumnsToNullable(Block & block, size_t starting_pos = 0); void removeColumnNullability(ColumnWithTypeAndName & column); Columns materializeColumns(const Block & block, const Names & names); diff --git a/src/Interpreters/misc.h b/src/Interpreters/misc.h index e2f34375dc02..3e9912ad0edf 100644 --- a/src/Interpreters/misc.h +++ b/src/Interpreters/misc.h @@ -1,8 +1,15 @@ #pragma once +#include +#include namespace DB { +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + inline bool functionIsInOperator(const std::string & name) { return name == "in" || name == "notIn"; @@ -18,4 +25,19 @@ inline bool functionIsLikeOperator(const std::string & name) return name == "like" || name == "notLike"; } +inline bool checkFunctionIsInOrGlobalInOperator(const ASTFunction & func) +{ + if (functionIsInOrGlobalInOperator(func.name)) + { + size_t num_arguments = func.arguments->children.size(); + if (num_arguments != 2) + throw Exception("Wrong number of arguments passed to function in. Expected: 2, passed: " + std::to_string(num_arguments), + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return true; + } + + return false; +} + } diff --git a/src/Interpreters/tests/expression_analyzer.cpp b/src/Interpreters/tests/expression_analyzer.cpp index 4bf1e9ff522d..2de9815a05a8 100644 --- a/src/Interpreters/tests/expression_analyzer.cpp +++ b/src/Interpreters/tests/expression_analyzer.cpp @@ -71,7 +71,7 @@ struct TestEntry { ParserSelectQuery parser; std::string message; - auto text = query.data(); + const auto * text = query.data(); if (ASTPtr ast = tryParseQuery(parser, text, text + query.size(), message, false, "", false, 0)) return ast; throw Exception(message, ErrorCodes::SYNTAX_ERROR); diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index eb9f1462666c..8ee4154541b1 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -87,6 +87,12 @@ size_t IAST::checkDepthImpl(size_t max_depth, size_t level) const return res; } +std::string IAST::formatForErrorMessage() const +{ + std::stringstream ss; + format(FormatSettings(ss, true /* one line */)); + return ss.str(); +} void IAST::cloneChildren() { diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 246b88f8c245..88dedc54d3f4 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -9,6 +9,7 @@ #include #include #include +#include class SipHash; @@ -215,6 +216,11 @@ class IAST : public std::enable_shared_from_this, public TypePromotion + static std::string formatForErrorMessage(const AstArray & array); + void cloneChildren(); public: @@ -231,4 +237,19 @@ class IAST : public std::enable_shared_from_this, public TypePromotion +std::string IAST::formatForErrorMessage(const AstArray & array) +{ + std::stringstream ss; + for (size_t i = 0; i < array.size(); ++i) + { + if (i > 0) + { + ss << ", "; + } + array[i]->format(IAST::FormatSettings(ss, true /* one line */)); + } + return ss.str(); +} + } diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index a6eed90d57a9..f0885a23468e 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -570,7 +570,7 @@ class AvroConfluentRowInputFormat::SchemaRegistry } catch (const Poco::Exception & e) { - throw Exception(Exception::CreateFromPoco, e); + throw Exception(Exception::CreateFromPocoTag{}, e); } catch (const avro::Exception & e) { diff --git a/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp b/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp index e5ad8b007c1b..03619b92a4a1 100644 --- a/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp @@ -19,7 +19,7 @@ CSVRowOutputFormat::CSVRowOutputFormat(WriteBuffer & out_, const Block & header_ } -void CSVRowOutputFormat::writePrefix() +void CSVRowOutputFormat::doWritePrefix() { auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); diff --git a/src/Processors/Formats/Impl/CSVRowOutputFormat.h b/src/Processors/Formats/Impl/CSVRowOutputFormat.h index 28bd8edf31ad..05b097f75149 100644 --- a/src/Processors/Formats/Impl/CSVRowOutputFormat.h +++ b/src/Processors/Formats/Impl/CSVRowOutputFormat.h @@ -27,10 +27,11 @@ class CSVRowOutputFormat : public IRowOutputFormat void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; void writeFieldDelimiter() override; void writeRowEndDelimiter() override; - void writePrefix() override; void writeBeforeTotals() override; void writeBeforeExtremes() override; + void doWritePrefix() override; + /// https://www.iana.org/assignments/media-types/text/csv String getContentType() const override { diff --git a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp index 219aef870ffd..924c2435f20a 100644 --- a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp +++ b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp @@ -73,7 +73,7 @@ static void fillLiteralInfo(DataTypes & nested_types, LiteralInfo & info) size_t elements_num = nested_types.size(); info.special_parser.nested_types.reserve(elements_num); - for (auto nested_type : nested_types) + for (auto & nested_type : nested_types) { /// It can be Array(Nullable(nested_type)) or Tuple(..., Nullable(nested_type), ...) bool is_nullable = false; @@ -200,7 +200,10 @@ class ReplaceLiteralsVisitor static void setDataType(LiteralInfo & info) { - /// Type (Field::Types:Which) of literal in AST can be: String, UInt64, Int64, Float64, Null or Array of simple literals (not of Arrays). + /// Type (Field::Types:Which) of literal in AST can be: + /// 1. simple literal type: String, UInt64, Int64, Float64, Null + /// 2. complex literal type: Array or Tuple of simple literals + /// 3. Array or Tuple of complex literals /// Null and empty Array literals are considered as tokens, because template with Nullable(Nothing) or Array(Nothing) is useless. Field::Types::Which field_type = info.literal->value.getType(); diff --git a/src/Processors/Pipe.cpp b/src/Processors/Pipe.cpp index a03fab64c11d..739a50a223e2 100644 --- a/src/Processors/Pipe.cpp +++ b/src/Processors/Pipe.cpp @@ -90,12 +90,20 @@ Pipe::Pipe(Pipes && pipes, ProcessorPtr transform) max_parallel_streams += pipe.max_parallel_streams; processors.insert(processors.end(), pipe.processors.begin(), pipe.processors.end()); + + std::move(pipe.table_locks.begin(), pipe.table_locks.end(), std::back_inserter(table_locks)); + std::move(pipe.interpreter_context.begin(), pipe.interpreter_context.end(), std::back_inserter(interpreter_context)); + std::move(pipe.storage_holders.begin(), pipe.storage_holders.end(), std::back_inserter(storage_holders)); } output_port = &transform->getOutputs().front(); processors.emplace_back(std::move(transform)); } +Pipe::Pipe(OutputPort * port) : output_port(port) +{ +} + void Pipe::addSimpleTransform(ProcessorPtr transform) { checkSimpleTransform(*transform); diff --git a/src/Processors/Pipe.h b/src/Processors/Pipe.h index da4f8bda814d..8eb265759716 100644 --- a/src/Processors/Pipe.h +++ b/src/Processors/Pipe.h @@ -22,6 +22,8 @@ class Pipe /// Transform must have the number of inputs equals to the number of pipes. And single output. /// Will connect pipes outputs with transform inputs automatically. Pipe(Pipes && pipes, ProcessorPtr transform); + /// Create pipe from output port. If pipe was created that way, it possibly will not have tree shape. + explicit Pipe(OutputPort * port); Pipe(const Pipe & other) = delete; Pipe(Pipe && other) = default; diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index d18d88785b54..ffd605af9a1a 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -511,10 +511,16 @@ void QueryPipeline::setOutput(ProcessorPtr output) } void QueryPipeline::unitePipelines( - std::vector && pipelines, const Block & common_header, const Context & context) + std::vector && pipelines, const Block & common_header, const Context & context, size_t max_threads_limit) { checkInitialized(); + /// Should we limit the number of threads for united pipeline. True if all pipelines have max_threads != 0. + /// If true, result max_threads will be sum(max_threads). + /// Note: it may be > than settings.max_threads, so we should apply this limit again. + bool will_limit_max_threads = !initialized() || max_threads != 0; + + addSimpleTransform([&](const Block & header) { return std::make_shared( @@ -568,9 +574,20 @@ void QueryPipeline::unitePipelines( interpreter_context.insert(interpreter_context.end(), pipeline.interpreter_context.begin(), pipeline.interpreter_context.end()); storage_holders.insert(storage_holders.end(), pipeline.storage_holders.begin(), pipeline.storage_holders.end()); - max_threads = std::max(max_threads, pipeline.max_threads); + max_threads += pipeline.max_threads; + will_limit_max_threads = will_limit_max_threads && pipeline.max_threads != 0; + + /// If one of pipelines uses more threads then current limit, will keep it. + /// It may happen if max_distributed_connections > max_threads + if (pipeline.max_threads > max_threads_limit) + max_threads_limit = pipeline.max_threads; } + if (!will_limit_max_threads) + max_threads = 0; + else + limitMaxThreads(max_threads_limit); + if (!extremes.empty()) { if (extremes.size() == 1) @@ -703,6 +720,11 @@ void QueryPipeline::initRowsBeforeLimit() Pipe QueryPipeline::getPipe() && { resize(1); + return std::move(std::move(*this).getPipes()[0]); +} + +Pipes QueryPipeline::getPipes() && +{ Pipe pipe(std::move(processors), streams.at(0), totals_having_port, extremes_port); pipe.max_parallel_streams = streams.maxParallelStreams(); @@ -721,7 +743,13 @@ Pipe QueryPipeline::getPipe() && if (extremes_port) pipe.setExtremesPort(extremes_port); - return pipe; + Pipes pipes; + pipes.emplace_back(std::move(pipe)); + + for (size_t i = 1; i < streams.size(); ++i) + pipes.emplace_back(Pipe(streams[i])); + + return pipes; } PipelineExecutorPtr QueryPipeline::execute() diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 876a61cdb125..09ea433dbce2 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -122,7 +122,7 @@ class QueryPipeline void enableQuotaForCurrentStreams(); - void unitePipelines(std::vector && pipelines, const Block & common_header, const Context & context); + void unitePipelines(std::vector && pipelines, const Block & common_header, const Context & context, size_t max_threads_limit = 0); PipelineExecutorPtr execute(); @@ -155,8 +155,16 @@ class QueryPipeline /// Set upper limit for the recommend number of threads void setMaxThreads(size_t max_threads_) { max_threads = max_threads_; } - /// Convert query pipeline to single pipe. + /// Update upper limit for the recommend number of threads + void limitMaxThreads(size_t max_threads_) + { + if (max_threads == 0 || max_threads_ < max_threads) + max_threads = max_threads_; + } + + /// Convert query pipeline to single or several pipes. Pipe getPipe() &&; + Pipes getPipes() &&; private: /// Destruction order: processors, header, locks, temporary storages, local contexts diff --git a/src/Processors/ResizeProcessor.cpp b/src/Processors/ResizeProcessor.cpp index 25184b1ac5c8..d620a850b06d 100644 --- a/src/Processors/ResizeProcessor.cpp +++ b/src/Processors/ResizeProcessor.cpp @@ -347,11 +347,16 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in auto & waiting_output = output_ports[input_with_data.waiting_output]; - if (waiting_output.status != OutputStatus::NeedData) - throw Exception("Invalid status for associated output.", ErrorCodes::LOGICAL_ERROR); + if (waiting_output.status == OutputStatus::NotActive) + throw Exception("Invalid status NotActive for associated output.", ErrorCodes::LOGICAL_ERROR); - waiting_output.port->pushData(input_with_data.port->pullData(/* set_not_deeded = */ true)); - waiting_output.status = OutputStatus::NotActive; + if (waiting_output.status != OutputStatus::Finished) + { + waiting_output.port->pushData(input_with_data.port->pullData(/* set_not_needed = */ true)); + waiting_output.status = OutputStatus::NotActive; + } + else + abandoned_chunks.emplace_back(input_with_data.port->pullData(/* set_not_needed = */ true)); if (input_with_data.port->isFinished()) { @@ -370,6 +375,18 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in return Status::Finished; } + /// Process abandoned chunks if any. + while (!abandoned_chunks.empty() && !waiting_outputs.empty()) + { + auto & waiting_output = output_ports[waiting_outputs.front()]; + waiting_outputs.pop(); + + waiting_output.port->pushData(std::move(abandoned_chunks.back())); + abandoned_chunks.pop_back(); + + waiting_output.status = OutputStatus::NotActive; + } + /// Enable more inputs if needed. while (!disabled_input_ports.empty() && !waiting_outputs.empty()) { @@ -383,6 +400,7 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in waiting_outputs.pop(); } + /// Close all other waiting for data outputs (there is no corresponding input for them). while (!waiting_outputs.empty()) { auto & output = output_ports[waiting_outputs.front()]; diff --git a/src/Processors/ResizeProcessor.h b/src/Processors/ResizeProcessor.h index 1f364ffcf155..5c9660af1132 100644 --- a/src/Processors/ResizeProcessor.h +++ b/src/Processors/ResizeProcessor.h @@ -128,6 +128,9 @@ class StrictResizeProcessor : public IProcessor std::vector input_ports; std::vector output_ports; + /// This field contained chunks which were read for output which had became finished while reading was happening. + /// They will be pushed to any next waiting output. + std::vector abandoned_chunks; }; } diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp index 50fac1218196..292b86a1446f 100644 --- a/src/Processors/Transforms/FillingTransform.cpp +++ b/src/Processors/Transforms/FillingTransform.cpp @@ -18,11 +18,7 @@ FillingTransform::FillingTransform( , filling_row(sort_description_) , next_row(sort_description_) { - std::vector is_fill_column(header_.columns()); - for (const auto & elem : sort_description) - is_fill_column[header_.getPositionByName(elem.column_name)] = true; - - auto try_convert_fields = [](FillColumnDescription & descr, const DataTypePtr & type) + auto try_convert_fields = [](auto & descr, const auto & type) { auto max_type = Field::Types::Null; WhichDataType which(type); @@ -49,30 +45,32 @@ FillingTransform::FillingTransform( return true; }; - for (size_t i = 0; i < header_.columns(); ++i) + std::vector is_fill_column(header_.columns()); + for (size_t i = 0; i < sort_description.size(); ++i) { - if (is_fill_column[i]) - { - size_t pos = fill_column_positions.size(); - auto & descr = filling_row.getFillDescription(pos); - auto type = header_.getByPosition(i).type; - if (!try_convert_fields(descr, type)) - throw Exception("Incompatible types of WITH FILL expression values with column type " - + type->getName(), ErrorCodes::INVALID_WITH_FILL_EXPRESSION); - - if (type->isValueRepresentedByUnsignedInteger() && - ((!descr.fill_from.isNull() && less(descr.fill_from, Field{0}, 1)) || - (!descr.fill_to.isNull() && less(descr.fill_to, Field{0}, 1)))) - { - throw Exception("WITH FILL bound values cannot be negative for unsigned type " - + type->getName(), ErrorCodes::INVALID_WITH_FILL_EXPRESSION); - } + size_t block_position = header_.getPositionByName(sort_description[i].column_name); + is_fill_column[block_position] = true; + fill_column_positions.push_back(block_position); - fill_column_positions.push_back(i); + auto & descr = filling_row.getFillDescription(i); + const auto & type = header_.getByPosition(block_position).type; + + if (!try_convert_fields(descr, type)) + throw Exception("Incompatible types of WITH FILL expression values with column type " + + type->getName(), ErrorCodes::INVALID_WITH_FILL_EXPRESSION); + + if (type->isValueRepresentedByUnsignedInteger() && + ((!descr.fill_from.isNull() && less(descr.fill_from, Field{0}, 1)) || + (!descr.fill_to.isNull() && less(descr.fill_to, Field{0}, 1)))) + { + throw Exception("WITH FILL bound values cannot be negative for unsigned type " + + type->getName(), ErrorCodes::INVALID_WITH_FILL_EXPRESSION); } - else - other_column_positions.push_back(i); } + + for (size_t i = 0; i < header_.columns(); ++i) + if (!is_fill_column[i]) + other_column_positions.push_back(i); } IProcessor::Status FillingTransform::prepare() @@ -107,8 +105,9 @@ void FillingTransform::transform(Chunk & chunk) { for (size_t pos : positions) { - new_columns.push_back(old_columns[pos]); - new_mutable_columns.push_back(old_columns[pos]->cloneEmpty()->assumeMutable()); + auto old_column = old_columns[pos]->convertToFullColumnIfConst(); + new_columns.push_back(old_column); + new_mutable_columns.push_back(old_column->cloneEmpty()->assumeMutable()); } }; diff --git a/src/Processors/Transforms/SortingTransform.cpp b/src/Processors/Transforms/SortingTransform.cpp index 6f4d36e125ba..1e3ebda4608a 100644 --- a/src/Processors/Transforms/SortingTransform.cpp +++ b/src/Processors/Transforms/SortingTransform.cpp @@ -285,7 +285,9 @@ IProcessor::Status SortingTransform::prepareGenerate() if (output.isFinished()) { - inputs.front().close(); + for (auto & input : inputs) + input.close(); + return Status::Finished; } diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 01bf0798a638..b64703570ee7 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -269,7 +269,6 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa { LOG_TRACE(log, "Started processing `" << file_path << '`'); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.global_context.getSettingsRef()); - auto connection = pool->get(timeouts); try { @@ -282,6 +281,8 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa ClientInfo client_info; readHeader(in, insert_settings, insert_query, client_info, log); + auto connection = pool->get(timeouts, &insert_settings); + RemoteBlockOutputStream remote{*connection, timeouts, insert_query, &insert_settings, &client_info}; remote.writePrefix(); diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 80b7d4c019e8..6f0558556396 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -15,8 +15,8 @@ #include #include #include -#include #include +#include #include #include diff --git a/src/Storages/Kafka/KafkaBlockInputStream.cpp b/src/Storages/Kafka/KafkaBlockInputStream.cpp index c1252086b1de..3c47da59e9e9 100644 --- a/src/Storages/Kafka/KafkaBlockInputStream.cpp +++ b/src/Storages/Kafka/KafkaBlockInputStream.cpp @@ -33,7 +33,7 @@ KafkaBlockInputStream::KafkaBlockInputStream( KafkaBlockInputStream::~KafkaBlockInputStream() { - if (!claimed) + if (!buffer) return; if (broken) @@ -51,7 +51,6 @@ void KafkaBlockInputStream::readPrefixImpl() { auto timeout = std::chrono::milliseconds(context.getSettingsRef().kafka_max_wait_ms.totalMilliseconds()); buffer = storage.popReadBuffer(timeout); - claimed = !!buffer; if (!buffer) return; @@ -173,7 +172,7 @@ Block KafkaBlockInputStream::readImpl() } } - if (buffer->rebalanceHappened() || total_rows == 0) + if (buffer->polledDataUnusable() || total_rows == 0) return Block(); /// MATERIALIZED columns can be added here, but I think diff --git a/src/Storages/Kafka/KafkaBlockInputStream.h b/src/Storages/Kafka/KafkaBlockInputStream.h index 1c853a4d486b..e30521228940 100644 --- a/src/Storages/Kafka/KafkaBlockInputStream.h +++ b/src/Storages/Kafka/KafkaBlockInputStream.h @@ -25,6 +25,7 @@ class KafkaBlockInputStream : public IBlockInputStream void readSuffixImpl() override; void commit(); + bool isStalled() const { return buffer->isStalled(); } private: StorageKafka & storage; @@ -33,9 +34,12 @@ class KafkaBlockInputStream : public IBlockInputStream UInt64 max_block_size; ConsumerBufferPtr buffer; - bool broken = true, finished = false, claimed = false, commit_in_suffix; + bool broken = true; + bool finished = false; + bool commit_in_suffix; - const Block non_virtual_header, virtual_header; + const Block non_virtual_header; + const Block virtual_header; }; } diff --git a/src/Storages/Kafka/KafkaSettings.h b/src/Storages/Kafka/KafkaSettings.h index fc823848ab45..93983fb60808 100644 --- a/src/Storages/Kafka/KafkaSettings.h +++ b/src/Storages/Kafka/KafkaSettings.h @@ -23,7 +23,7 @@ struct KafkaSettings : public SettingsCollection M(SettingChar, kafka_row_delimiter, '\0', "The character to be considered as a delimiter in Kafka message.", 0) \ M(SettingString, kafka_schema, "", "Schema identifier (used by schema-based formats) for Kafka engine", 0) \ M(SettingUInt64, kafka_num_consumers, 1, "The number of consumers per table for Kafka engine.", 0) \ - M(SettingUInt64, kafka_max_block_size, 0, "The maximum block size per table for Kafka engine.", 0) \ + M(SettingUInt64, kafka_max_block_size, 0, "The maximum batch size for poll.", 0) \ M(SettingUInt64, kafka_skip_broken_messages, 0, "Skip at least this number of broken messages from Kafka topic per block", 0) \ M(SettingUInt64, kafka_commit_every_batch, 0, "Commit every consumed and handled batch instead of a single commit after writing a whole block", 0) diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 608d8ba22134..275a1a60d754 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -8,8 +8,14 @@ namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_COMMIT_OFFSET; +} + using namespace std::chrono_literals; const auto MAX_TIME_TO_WAIT_FOR_ASSIGNMENT_MS = 15000; +const auto DRAIN_TIMEOUT_MS = 5000ms; ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( @@ -31,14 +37,14 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( , topics(_topics) { // called (synchroniously, during poll) when we enter the consumer group - consumer->set_assignment_callback([this](const cppkafka::TopicPartitionList& topic_partitions) + consumer->set_assignment_callback([this](const cppkafka::TopicPartitionList & topic_partitions) { LOG_TRACE(log, "Topics/partitions assigned: " << topic_partitions); assignment = topic_partitions; }); // called (synchroniously, during poll) when we leave the consumer group - consumer->set_revocation_callback([this](const cppkafka::TopicPartitionList& topic_partitions) + consumer->set_revocation_callback([this](const cppkafka::TopicPartitionList & topic_partitions) { // Rebalance is happening now, and now we have a chance to finish the work // with topics/partitions we were working with before rebalance @@ -77,21 +83,71 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( ReadBufferFromKafkaConsumer::~ReadBufferFromKafkaConsumer() { - /// NOTE: see https://github.com/edenhill/librdkafka/issues/2077 try { if (!consumer->get_subscription().empty()) - consumer->unsubscribe(); - if (!assignment.empty()) - consumer->unassign(); - while (consumer->get_consumer_queue().next_event(100ms)); + { + try + { + consumer->unsubscribe(); + } + catch (const cppkafka::HandleException & e) + { + LOG_ERROR(log, "Error during unsubscribe: " << e.what()); + } + drain(); + } } catch (const cppkafka::HandleException & e) { - LOG_ERROR(log, "Exception from ReadBufferFromKafkaConsumer destructor: " << e.what()); + LOG_ERROR(log, "Error while destructing consumer: " << e.what()); } + } +// Needed to drain rest of the messages / queued callback calls from the consumer +// after unsubscribe, otherwise consumer will hang on destruction +// see https://github.com/edenhill/librdkafka/issues/2077 +// https://github.com/confluentinc/confluent-kafka-go/issues/189 etc. +void ReadBufferFromKafkaConsumer::drain() +{ + auto start_time = std::chrono::steady_clock::now(); + cppkafka::Error last_error(RD_KAFKA_RESP_ERR_NO_ERROR); + + while (true) + { + auto msg = consumer->poll(100ms); + if (!msg) + break; + + auto error = msg.get_error(); + + if (error) + { + if (msg.is_eof() || error == last_error) + { + break; + } + else + { + LOG_ERROR(log, "Error during draining: " << error); + } + } + + // i don't stop draining on first error, + // only if it repeats once again sequentially + last_error = error; + + auto ts = std::chrono::steady_clock::now(); + if (std::chrono::duration_cast(ts-start_time) > DRAIN_TIMEOUT_MS) + { + LOG_ERROR(log, "Timeout during draining."); + break; + } + } +} + + void ReadBufferFromKafkaConsumer::commit() { auto PrintOffsets = [this] (const char * prefix, const cppkafka::TopicPartitionList & offsets) @@ -140,16 +196,41 @@ void ReadBufferFromKafkaConsumer::commit() // we may need to repeat commit in sync mode in revocation callback, // but it seems like existing API doesn't allow us to to that // in a controlled manner (i.e. we don't know the offsets to commit then) - consumer->commit(); + + size_t max_retries = 5; + bool commited = false; + + while (!commited && max_retries > 0) + { + try + { + // See https://github.com/edenhill/librdkafka/issues/1470 + // broker may reject commit if during offsets.commit.timeout.ms (5000 by default), + // there were not enough replicas available for the __consumer_offsets topic. + // also some other temporary issues like client-server connectivity problems are possible + consumer->commit(); + commited = true; + PrintOffsets("Committed offset", consumer->get_offsets_committed(consumer->get_assignment())); + } + catch (const cppkafka::HandleException & e) + { + LOG_ERROR(log, "Exception during commit attempt: " << e.what()); + } + --max_retries; + } + + if (!commited) + { + // TODO: insert atomicity / transactions is needed here (possibility to rollback, ot 2 phase commits) + throw Exception("All commit attempts failed. Last block was already written to target table(s), but was not commited to Kafka.", ErrorCodes::CANNOT_COMMIT_OFFSET); + } } else { - LOG_TRACE(log,"Nothing to commit."); + LOG_TRACE(log, "Nothing to commit."); } - PrintOffsets("Committed offset", consumer->get_offsets_committed(consumer->get_assignment())); offsets_stored = 0; - stalled = false; } @@ -196,8 +277,13 @@ void ReadBufferFromKafkaConsumer::unsubscribe() // it should not raise exception as used in destructor try { - if (!consumer->get_subscription().empty()) - consumer->unsubscribe(); + // From docs: Any previous subscription will be unassigned and unsubscribed first. + consumer->subscribe(topics); + + // I wanted to avoid explicit unsubscribe as it requires draining the messages + // to close the consumer safely after unsubscribe + // see https://github.com/edenhill/librdkafka/issues/2077 + // https://github.com/confluentinc/confluent-kafka-go/issues/189 etc. } catch (const cppkafka::HandleException & e) { @@ -222,19 +308,30 @@ void ReadBufferFromKafkaConsumer::resetToLastCommitted(const char * msg) } auto committed_offset = consumer->get_offsets_committed(consumer->get_assignment()); consumer->assign(committed_offset); - LOG_TRACE(log, msg << "Returned to committed position: " << committed_offset); - + LOG_TRACE(log, msg << " Returned to committed position: " << committed_offset); } /// Do commit messages implicitly after we processed the previous batch. bool ReadBufferFromKafkaConsumer::nextImpl() { + /// NOTE: ReadBuffer was implemented with an immutable underlying contents in mind. /// If we failed to poll any message once - don't try again. /// Otherwise, the |poll_timeout| expectations get flawn. - if (stalled || stopped || !allowed || rebalance_happened) + + // we can react on stop only during fetching data + // after block is formed (i.e. during copying data to MV / commiting) we ignore stop attempts + if (stopped) + { + was_stopped = true; + offsets_stored = 0; + return false; + } + + if (stalled || was_stopped || !allowed || rebalance_happened) return false; + if (current == messages.end()) { if (intermediate_commit) @@ -246,7 +343,13 @@ bool ReadBufferFromKafkaConsumer::nextImpl() /// Don't drop old messages immediately, since we may need them for virtual columns. auto new_messages = consumer->poll_batch(batch_size, std::chrono::milliseconds(poll_timeout)); - if (rebalance_happened) + if (stopped) + { + was_stopped = true; + offsets_stored = 0; + return false; + } + else if (rebalance_happened) { if (!new_messages.empty()) { @@ -317,7 +420,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() void ReadBufferFromKafkaConsumer::storeLastReadMessageOffset() { - if (!stalled && !rebalance_happened) + if (!stalled && !was_stopped && !rebalance_happened) { consumer->store_offset(*(current - 1)); ++offsets_stored; diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h index 700a69cf49bc..e90e3b488816 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h @@ -29,7 +29,6 @@ class ReadBufferFromKafkaConsumer : public ReadBuffer const Names & _topics ); ~ReadBufferFromKafkaConsumer() override; - void allowNext() { allowed = true; } // Allow to read next message. void commit(); // Commit all processed messages. void subscribe(); // Subscribe internal consumer to topics. @@ -38,7 +37,8 @@ class ReadBufferFromKafkaConsumer : public ReadBuffer auto pollTimeout() const { return poll_timeout; } bool hasMorePolledMessages() const; - auto rebalanceHappened() const { return rebalance_happened; } + bool polledDataUnusable() const { return (was_stopped || rebalance_happened); } + bool isStalled() const { return stalled; } void storeLastReadMessageOffset(); void resetToLastCommitted(const char * msg); @@ -64,13 +64,20 @@ class ReadBufferFromKafkaConsumer : public ReadBuffer const std::atomic & stopped; + // order is important, need to be destructed before consumer Messages messages; Messages::const_iterator current; bool rebalance_happened = false; + + bool was_stopped = false; + + // order is important, need to be destructed before consumer cppkafka::TopicPartitionList assignment; const Names topics; + void drain(); + bool nextImpl() override; }; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 6aca827208d0..34e480419e29 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -49,6 +49,7 @@ namespace { const auto RESCHEDULE_MS = 500; const auto CLEANUP_TIMEOUT_MS = 3000; + const auto MAX_THREAD_WORK_DURATION_MS = 60000; // once per minute leave do reschedule (we can't lock threads in pool forever) /// Configuration prefix const String CONFIG_PREFIX = "kafka"; @@ -90,7 +91,6 @@ StorageKafka::StorageKafka( {"_partition", std::make_shared()}, {"_timestamp", std::make_shared(std::make_shared())}}, true)) , global_context(context_.getGlobalContext()) - , kafka_context(Context(global_context)) , topics(global_context.getMacros()->expand(topics_)) , brokers(global_context.getMacros()->expand(brokers_)) , group(global_context.getMacros()->expand(group_)) @@ -104,8 +104,6 @@ StorageKafka::StorageKafka( , skip_broken(skip_broken_) , intermediate_commit(intermediate_commit_) { - kafka_context.makeQueryContext(); - setColumns(columns_); task = global_context.getSchedulePool().createTask(log->name(), [this]{ threadFunc(); }); task->deactivate(); @@ -175,17 +173,14 @@ void StorageKafka::shutdown() // Interrupt streaming thread stream_cancelled = true; + LOG_TRACE(log, "Waiting for cleanup"); + task->deactivate(); + // Close all consumers for (size_t i = 0; i < num_created_consumers; ++i) - { auto buffer = popReadBuffer(); - // FIXME: not sure if we really close consumers here, and if we really need to close them here. - } - LOG_TRACE(log, "Waiting for cleanup"); rd_kafka_wait_destroyed(CLEANUP_TIMEOUT_MS); - - task->deactivate(); } void StorageKafka::updateDependencies() @@ -248,17 +243,23 @@ ProducerBufferPtr StorageKafka::createWriteBuffer(const Block & header) ConsumerBufferPtr StorageKafka::createReadBuffer() { cppkafka::Configuration conf; + conf.set("metadata.broker.list", brokers); conf.set("group.id", group); conf.set("client.id", VERSION_FULL); + conf.set("auto.offset.reset", "smallest"); // If no offset stored for this group, read all messages from the start + + updateConfiguration(conf); + + // those settings should not be changed by users. conf.set("enable.auto.commit", "false"); // We manually commit offsets after a stream successfully finished conf.set("enable.auto.offset.store", "false"); // Update offset automatically - to commit them all at once. conf.set("enable.partition.eof", "false"); // Ignore EOF messages - updateConfiguration(conf); // Create a consumer and subscribe to topics auto consumer = std::make_shared(conf); + consumer->set_destroy_flags(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); // Limit the number of batched messages to allow early cancellations const Settings & settings = global_context.getSettingsRef(); @@ -323,6 +324,8 @@ void StorageKafka::threadFunc() // Check if at least one direct dependency is attached auto dependencies = global_context.getDependencies(table_id); + auto start_time = std::chrono::steady_clock::now(); + // Keep streaming as long as there are attached views and streaming is not cancelled while (!stream_cancelled && num_created_consumers > 0 && !dependencies.empty()) { @@ -331,9 +334,21 @@ void StorageKafka::threadFunc() LOG_DEBUG(log, "Started streaming to " << dependencies.size() << " attached views"); - // Reschedule if not limited - if (!streamToViews()) + // Exit the loop & reschedule if some stream stalled + auto some_stream_is_stalled = streamToViews(); + if (some_stream_is_stalled) + { + LOG_TRACE(log, "Stream(s) stalled. Reschedule."); + break; + } + + auto ts = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(ts-start_time); + if (duration.count() > MAX_THREAD_WORK_DURATION_MS) + { + LOG_TRACE(log, "Thread work duration limit exceeded. Reschedule."); break; + } } } catch (...) @@ -364,6 +379,9 @@ bool StorageKafka::streamToViews() if (block_size == 0) block_size = settings.max_block_size; + auto kafka_context = Context(global_context); + kafka_context.makeQueryContext(); + // Create a stream for each consumer and join them in a union stream // Only insert into dependent views and expect that input blocks contain virtual columns InterpreterInsertQuery interpreter(insert, kafka_context, false, true, true); @@ -392,17 +410,19 @@ bool StorageKafka::streamToViews() else in = streams[0]; + // We can't cancel during copyData, as it's not aware of commits and other kafka-related stuff. + // It will be cancelled on underlying layer (kafka buffer) std::atomic stub = {false}; copyData(*in, *block_io.out, &stub); + + bool some_stream_is_stalled = false; for (auto & stream : streams) + { + some_stream_is_stalled = some_stream_is_stalled || stream->as()->isStalled(); stream->as()->commit(); + } - // Check whether the limits were applied during query execution - bool limits_applied = false; - const BlockStreamProfileInfo & info = in->getProfileInfo(); - limits_applied = info.hasAppliedLimit(); - - return limits_applied; + return some_stream_is_stalled; } void registerStorageKafka(StorageFactory & factory) diff --git a/src/Storages/Kafka/StorageKafka.h b/src/Storages/Kafka/StorageKafka.h index ecd58deceb71..6bab6088b0ec 100644 --- a/src/Storages/Kafka/StorageKafka.h +++ b/src/Storages/Kafka/StorageKafka.h @@ -80,7 +80,6 @@ class StorageKafka : public ext::shared_ptr_helper, public IStorag private: // Configuration and state Context global_context; - Context kafka_context; Names topics; const String brokers; const String group; diff --git a/src/Storages/MergeTree/ActiveDataPartSet.h b/src/Storages/MergeTree/ActiveDataPartSet.h index beb538e2ca59..e17f4903a26f 100644 --- a/src/Storages/MergeTree/ActiveDataPartSet.h +++ b/src/Storages/MergeTree/ActiveDataPartSet.h @@ -67,6 +67,18 @@ class ActiveDataPartSet return result; } + /// Remove only covered parts from active set + bool removePartsCoveredBy(const String & part_name) + { + Strings parts_covered_by = getPartsCoveredBy(MergeTreePartInfo::fromPartName(part_name, format_version)); + bool result = true; + for (const auto & part : parts_covered_by) + if (part != part_name) + result &= remove(part); + + return result; + } + /// If not found, return an empty string. String getContainingPart(const MergeTreePartInfo & part_info) const; String getContainingPart(const String & name) const; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index b2d0b4fe8f23..0da6dc86c4d7 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -85,8 +85,6 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo try { - auto storage_lock = data.lockStructureForShare(false, RWLockImpl::NO_QUERY); - MergeTreeData::DataPartPtr part = findPart(part_name); std::shared_lock part_lock(part->columns_lock); diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 7d9e60695a6c..2e826be93ba9 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -181,8 +181,11 @@ const KeyCondition::AtomMap KeyCondition::atom_map }, { "empty", - [] (RPNElement & out, const Field &) + [] (RPNElement & out, const Field & value) { + if (value.getType() != Field::Types::String) + return false; + out.function = RPNElement::FUNCTION_IN_RANGE; out.range = Range(""); return true; @@ -190,8 +193,11 @@ const KeyCondition::AtomMap KeyCondition::atom_map }, { "notEmpty", - [] (RPNElement & out, const Field &) + [] (RPNElement & out, const Field & value) { + if (value.getType() != Field::Types::String) + return false; + out.function = RPNElement::FUNCTION_NOT_IN_RANGE; out.range = Range(""); return true; diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index f1d4f9164733..4dc0f656ad89 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -315,12 +316,17 @@ void MergeTreeBaseSelectProcessor::executePrewhereActions(Block & block, const P prewhere_info->alias_actions->execute(block); prewhere_info->prewhere_actions->execute(block); + auto & prewhere_column = block.getByName(prewhere_info->prewhere_column_name); + + if (!prewhere_column.type->canBeUsedInBooleanContext()) + throw Exception("Invalid type for filter in PREWHERE: " + prewhere_column.type->getName(), + ErrorCodes::LOGICAL_ERROR); + if (prewhere_info->remove_prewhere_column) block.erase(prewhere_info->prewhere_column_name); else { auto & ctn = block.getByName(prewhere_info->prewhere_column_name); - ctn.type = std::make_shared(); ctn.column = ctn.type->createColumnConst(block.rows(), 1u)->convertToFullColumnIfConst(); } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index b5a8645597e2..6dfbbc9da964 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -606,6 +606,11 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( .save_marks_in_cache = true }; + /// Projection, that needed to drop columns, which have appeared by execution + /// of some extra expressions, and to allow execute the same expressions later. + /// NOTE: It may lead to double computation of expressions. + ExpressionActionsPtr result_projection; + if (select.final()) { /// Add columns needed to calculate the sorting expression and the sign. @@ -628,7 +633,8 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( query_info, virt_column_names, settings, - reader_settings); + reader_settings, + result_projection); } else if (settings.optimize_read_in_order && query_info.input_sorting_info) { @@ -649,7 +655,8 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( sorting_key_prefix_expr, virt_column_names, settings, - reader_settings); + reader_settings, + result_projection); } else { @@ -672,6 +679,13 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( pipe.getHeader(), filter_expression, filter_function->getColumnName(), false)); } + if (result_projection) + { + for (auto & pipe : res) + pipe.addSimpleTransform(std::make_shared( + pipe.getHeader(), result_projection)); + } + /// By the way, if a distributed query or query to a Merge table is made, then the `_sample_factor` column can have different values. if (sample_factor_column_queried) { @@ -818,6 +832,14 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( return res; } +static ExpressionActionsPtr createProjection(const Pipe & pipe, const MergeTreeData & data) +{ + const auto & header = pipe.getHeader(); + auto projection = std::make_shared(header.getNamesAndTypesList(), data.global_context); + projection->add(ExpressionAction::project(header.getNames())); + return projection; +} + Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts, size_t num_streams, @@ -828,7 +850,8 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( const ExpressionActionsPtr & sorting_key_prefix_expr, const Names & virt_columns, const Settings & settings, - const MergeTreeReaderSettings & reader_settings) const + const MergeTreeReaderSettings & reader_settings, + ExpressionActionsPtr & out_projection) const { size_t sum_marks = 0; const InputSortingInfoPtr & input_sorting_info = query_info.input_sorting_info; @@ -1004,6 +1027,8 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( sort_description.emplace_back(data.sorting_key_columns[j], input_sorting_info->direction, 1); + /// Drop temporary columns, added by 'sorting_key_prefix_expr' + out_projection = createProjection(pipes.back(), data); for (auto & pipe : pipes) pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), sorting_key_prefix_expr)); @@ -1028,7 +1053,8 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( const SelectQueryInfo & query_info, const Names & virt_columns, const Settings & settings, - const MergeTreeReaderSettings & reader_settings) const + const MergeTreeReaderSettings & reader_settings, + ExpressionActionsPtr & out_projection) const { const auto data_settings = data.getSettings(); size_t sum_marks = 0; @@ -1066,6 +1092,10 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( virt_columns, part.part_index_in_query); Pipe pipe(std::move(source_processor)); + /// Drop temporary columns, added by 'sorting_key_expr' + if (!out_projection) + out_projection = createProjection(pipe, data); + pipe.addSimpleTransform(std::make_shared(pipe.getHeader(), data.sorting_key_expr)); pipes.emplace_back(std::move(pipe)); } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index e6eb26da7e3a..09b7958c59fb 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -57,6 +57,7 @@ class MergeTreeDataSelectExecutor const Settings & settings, const MergeTreeReaderSettings & reader_settings) const; + /// out_projection - save projection only with columns, requested to read Pipes spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts, size_t num_streams, @@ -67,7 +68,8 @@ class MergeTreeDataSelectExecutor const ExpressionActionsPtr & sorting_key_prefix_expr, const Names & virt_columns, const Settings & settings, - const MergeTreeReaderSettings & reader_settings) const; + const MergeTreeReaderSettings & reader_settings, + ExpressionActionsPtr & out_projection) const; Pipes spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, @@ -77,7 +79,8 @@ class MergeTreeDataSelectExecutor const SelectQueryInfo & query_info, const Names & virt_columns, const Settings & settings, - const MergeTreeReaderSettings & reader_settings) const; + const MergeTreeReaderSettings & reader_settings, + ExpressionActionsPtr & out_projection) const; /// Get the approximate value (bottom estimate - only by full marks) of the number of rows falling under the index. size_t getApproximateTotalRowsToRead( diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 7718ed642d35..7578abf5e56a 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -902,13 +902,17 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r if (prewhere->remove_prewhere_column) result.columns.erase(result.columns.begin() + prewhere_column_pos); else - result.columns[prewhere_column_pos] = DataTypeUInt8().createColumnConst(result.num_rows, 1u)->convertToFullColumnIfConst(); + result.columns[prewhere_column_pos] = + getSampleBlock().getByName(prewhere->prewhere_column_name).type-> + createColumnConst(result.num_rows, 1u)->convertToFullColumnIfConst(); } } /// Filter in WHERE instead else { result.columns[prewhere_column_pos] = result.getFilterHolder()->convertToFullColumnIfConst(); + if (getSampleBlock().getByName(prewhere->prewhere_column_name).type->isNullable()) + result.columns[prewhere_column_pos] = makeNullable(std::move(result.columns[prewhere_column_pos])); result.clearFilter(); // Acting as a flag to not filter in PREWHERE } } diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index b733bb8dfb84..eb31eaec69fc 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -36,8 +36,11 @@ MergedBlockOutputStream::MergedBlockOutputStream( : IMergedBlockOutputStream(data_part) , columns_list(columns_list_) { - MergeTreeWriterSettings writer_settings(data_part->storage.global_context.getSettings(), - data_part->storage.canUseAdaptiveGranularity(), aio_threshold, blocks_are_granules_size); + MergeTreeWriterSettings writer_settings( + storage.global_context.getSettings(), + data_part->index_granularity_info.is_adaptive, + aio_threshold, + blocks_are_granules_size); if (aio_threshold > 0 && !merged_column_to_size.empty()) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index fda0a8eb5a82..db925c0ae4e1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -198,7 +198,8 @@ void ReplicatedMergeTreeBlockOutputStream::writeExistingPart(MergeTreeData::Muta } -void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zookeeper, MergeTreeData::MutableDataPartPtr & part, const String & block_id) +void ReplicatedMergeTreeBlockOutputStream::commitPart( + zkutil::ZooKeeperPtr & zookeeper, MergeTreeData::MutableDataPartPtr & part, const String & block_id) { storage.check(part->getColumns()); assertSessionIsNotExpired(zookeeper); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 00eca0e9dbae..5262bc483b5d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -203,21 +203,25 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( if (is_successful) { - if (!entry->actual_new_part_name.empty()) { /// We don't add bigger fetched part to current_parts because we /// have an invariant `virtual_parts` = `current_parts` + `queue`. - /// But we can remove it from mutations, because we actually have it. - removePartFromMutations(entry->actual_new_part_name); + /// + /// But we remove covered parts from mutations, because we actually + /// have replacing part. + /// + /// NOTE actual_new_part_name is very confusing and error-prone. This approach must be fixed. + removeCoveredPartsFromMutations(entry->actual_new_part_name, /*remove_part = */ false, /*remove_covered_parts = */ true); } for (const String & virtual_part_name : entry->getVirtualPartNames()) { current_parts.add(virtual_part_name); - /// Each processed part may be already mutated, so we try to remove - /// all current parts from mutations. - removePartFromMutations(virtual_part_name); + + /// These parts are already covered by newer part, we don't have to + /// mutate it. + removeCoveredPartsFromMutations(virtual_part_name, /*remove_part = */ false, /*remove_covered_parts = */ true); } String drop_range_part_name; @@ -242,16 +246,16 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( { for (const String & virtual_part_name : entry->getVirtualPartNames()) { - /// Because execution of the entry is unsuccessful, `virtual_part_name` will never appear - /// so we won't need to mutate it. - removePartFromMutations(virtual_part_name); + /// Because execution of the entry is unsuccessful, + /// `virtual_part_name` will never appear so we won't need to mutate + /// it. + removeCoveredPartsFromMutations(virtual_part_name, /*remove_part = */ true, /*remove_covered_parts = */ false); } - } } -void ReplicatedMergeTreeQueue::removePartFromMutations(const String & part_name) +void ReplicatedMergeTreeQueue::removeCoveredPartsFromMutations(const String & part_name, bool remove_part, bool remove_covered_parts) { auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); auto in_partition = mutations_by_partition.find(part_info.partition_id); @@ -265,7 +269,15 @@ void ReplicatedMergeTreeQueue::removePartFromMutations(const String & part_name) { MutationStatus & status = *it->second; - status.parts_to_do.removePartAndCoveredParts(part_name); + if (remove_part && remove_covered_parts) + status.parts_to_do.removePartAndCoveredParts(part_name); + else if (remove_covered_parts) + status.parts_to_do.removePartsCoveredBy(part_name); + else if (remove_part) + status.parts_to_do.remove(part_name); + else + throw Exception("Called remove part from mutations, but nothing removed", ErrorCodes::LOGICAL_ERROR); + if (status.parts_to_do.size() == 0) some_mutations_are_probably_done = true; @@ -678,6 +690,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C const String & partition_id = pair.first; Int64 block_num = pair.second; mutations_by_partition[partition_id].emplace(block_num, &mutation); + LOG_TRACE(log, "Adding mutation " << entry->znode_name << " for partition " << partition_id << " for all block numbers less than " << block_num); } /// Initialize `mutation.parts_to_do`. First we need to mutate all parts in `current_parts`. @@ -1949,6 +1962,7 @@ void ReplicatedMergeTreeQueue::removeCurrentPartsFromMutations() { std::lock_guard state_lock(state_mutex); for (const auto & part_name : current_parts.getParts()) - removePartFromMutations(part_name); + removeCoveredPartsFromMutations(part_name, /*remove_part = */ true, /*remove_covered_parts = */ true); } + } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index ab1e268deedb..412163cc5a4d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -107,8 +107,13 @@ class ReplicatedMergeTreeQueue ReplicatedMergeTreeMutationEntryPtr entry; - /// Parts we have to mutate to complete mutation. We use ActiveDataPartSet structure - /// to be able to manage covering and covered parts. + /// Current parts we have to mutate to complete mutation. + /// + /// current_part_name =mutation> result_part_name + /// ^~~parts_to_do~~^ ^~virtual_parts~^ + /// + /// We use ActiveDataPartSet structure to be able to manage covering and + /// covered parts. ActiveDataPartSet parts_to_do; /// Note that is_done is not equivalent to parts_to_do.size() == 0 @@ -204,11 +209,16 @@ class ReplicatedMergeTreeQueue /// Add part for mutations with block_number > part.getDataVersion() void addPartToMutations(const String & part_name); - /// Remove part from mutations which were assigned to mutate it - /// with block_number > part.getDataVersion() - /// and block_number == part.getDataVersion() - /// ^ (this may happen if we downloaded mutated part from other replica) - void removePartFromMutations(const String & part_name); + /// Remove covered parts from mutations (parts_to_do) which were assigned + /// for mutation. If remove_covered_parts = true, than remove parts covered + /// by first argument. If remove_part == true, than also remove part itself. + /// Both negative flags will throw exception. + /// + /// Part removed from mutations which satisfy contitions: + /// block_number > part.getDataVersion() + /// or block_number == part.getDataVersion() + /// ^ (this may happen if we downloaded mutated part from other replica) + void removeCoveredPartsFromMutations(const String & part_name, bool remove_part, bool remove_covered_parts); /// Update the insertion times in ZooKeeper. void updateTimesInZooKeeper(zkutil::ZooKeeperPtr zookeeper, diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index e0bbb3e4fb4e..e1c532e64fea 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -245,6 +245,18 @@ Pipes StorageBuffer::read( for (auto & buf : buffers) pipes_from_buffers.emplace_back(std::make_shared(column_names, buf, *this)); + /// Convert pipes from table to structure from buffer. + if (!pipes_from_buffers.empty() && !pipes_from_dst.empty() + && !blocksHaveEqualStructure(pipes_from_buffers.front().getHeader(), pipes_from_dst.front().getHeader())) + { + for (auto & pipe : pipes_from_dst) + pipe.addSimpleTransform(std::make_shared( + pipe.getHeader(), + pipes_from_buffers.front().getHeader(), + ConvertingTransform::MatchColumnsMode::Name, + context)); + } + /** If the sources from the table were processed before some non-initial stage of query execution, * then sources from the buffers must also be wrapped in the processing pipeline before the same stage. */ @@ -696,7 +708,10 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl for (const auto & column : block_to_write) list_of_columns->children.push_back(std::make_shared(column.name)); - InterpreterInsertQuery interpreter{insert, global_context, allow_materialized}; + auto insert_context = Context(global_context); + insert_context.makeQueryContext(); + + InterpreterInsertQuery interpreter{insert, insert_context, allow_materialized}; auto block_io = interpreter.execute(); block_io.out->writePrefix(); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index cfe87be002bf..6a9947a3f831 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include @@ -89,18 +90,23 @@ ASTPtr rewriteSelectQuery(const ASTPtr & query, const std::string & database, co auto modified_query_ast = query->clone(); ASTSelectQuery & select_query = modified_query_ast->as(); + if (table_function_ptr) + select_query.addTableFunction(table_function_ptr); + else + select_query.replaceDatabaseAndTable(database, table); - /// restore long column names in JOIN ON expressions - if (auto tables = select_query.tables()) + /// Restore long column names (cause our short names are ambiguous). + /// TODO: aliased table functions & CREATE TABLE AS table function cases + if (!table_function_ptr) { RestoreQualifiedNamesVisitor::Data data; - RestoreQualifiedNamesVisitor(data).visit(tables); + data.distributed_table = DatabaseAndTableWithAlias(*getTableExpression(query->as(), 0)); + data.remote_table.database = database; + data.remote_table.table = table; + data.rename = true; + RestoreQualifiedNamesVisitor(data).visit(modified_query_ast); } - if (table_function_ptr) - select_query.addTableFunction(table_function_ptr); - else - select_query.replaceDatabaseAndTable(database, table); return modified_query_ast; } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 92d9c3ccfe22..d01cad39a73a 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -431,6 +431,7 @@ class StorageFileBlockOutputStream : public IBlockOutputStream const Context & context) : storage(storage_), lock(storage.rwlock) { + std::unique_ptr naked_buffer = nullptr; if (storage.use_table_fd) { /** NOTE: Using real file binded to FD may be misleading: @@ -438,17 +439,21 @@ class StorageFileBlockOutputStream : public IBlockOutputStream * INSERT data; SELECT *; last SELECT returns only insert_data */ storage.table_fd_was_used = true; - write_buf = wrapWriteBufferWithCompressionMethod(std::make_unique(storage.table_fd), compression_method, 3); + naked_buffer = std::make_unique(storage.table_fd); } else { if (storage.paths.size() != 1) throw Exception("Table '" + storage.getStorageID().getNameForLogs() + "' is in readonly mode because of globs in filepath", ErrorCodes::DATABASE_ACCESS_DENIED); - write_buf = wrapWriteBufferWithCompressionMethod( - std::make_unique(storage.paths[0], DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT), - compression_method, 3); + naked_buffer = std::make_unique(storage.paths[0], DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT); } + /// In case of CSVWithNames we have already written prefix. + if (naked_buffer->size()) + prefix_written = true; + + write_buf = wrapWriteBufferWithCompressionMethod(std::move(naked_buffer), compression_method, 3); + writer = FormatFactory::instance().getOutput(storage.format_name, *write_buf, storage.getSampleBlock(), context); } @@ -461,7 +466,9 @@ class StorageFileBlockOutputStream : public IBlockOutputStream void writePrefix() override { - writer->writePrefix(); + if (!prefix_written) + writer->writePrefix(); + prefix_written = true; } void writeSuffix() override @@ -479,6 +486,7 @@ class StorageFileBlockOutputStream : public IBlockOutputStream std::unique_lock lock; std::unique_ptr write_buf; BlockOutputStreamPtr writer; + bool prefix_written{false}; }; BlockOutputStreamPtr StorageFile::write( diff --git a/src/Storages/StorageGenerateRandom.cpp b/src/Storages/StorageGenerateRandom.cpp index fd2f56f961b7..3faa0b86500f 100644 --- a/src/Storages/StorageGenerateRandom.cpp +++ b/src/Storages/StorageGenerateRandom.cpp @@ -38,6 +38,8 @@ namespace ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int TOO_LARGE_ARRAY_SIZE; + extern const int TOO_LARGE_STRING_SIZE; } @@ -387,6 +389,14 @@ StorageGenerateRandom::StorageGenerateRandom(const StorageID & table_id_, const UInt64 max_array_length_, UInt64 max_string_length_, std::optional random_seed_) : IStorage(table_id_), max_array_length(max_array_length_), max_string_length(max_string_length_) { + static constexpr size_t MAX_ARRAY_SIZE = 1 << 30; + static constexpr size_t MAX_STRING_SIZE = 1 << 30; + + if (max_array_length > MAX_ARRAY_SIZE) + throw Exception("Too large array size in GenerateRandom: " + toString(max_array_length) + ", maximum: " + toString(MAX_ARRAY_SIZE), ErrorCodes::TOO_LARGE_ARRAY_SIZE); + if (max_string_length > MAX_STRING_SIZE) + throw Exception("Too large string size in GenerateRandom: " + toString(max_string_length) + ", maximum: " + toString(MAX_STRING_SIZE), ErrorCodes::TOO_LARGE_STRING_SIZE); + random_seed = random_seed_ ? sipHash64(*random_seed_) : randomSeed(); setColumns(columns_); } @@ -420,7 +430,6 @@ void registerStorageGenerateRandom(StorageFactory & factory) if (engine_args.size() == 3) max_array_length = engine_args[2]->as().value.safeGet(); - return StorageGenerateRandom::create(args.table_id, args.columns, max_array_length, max_string_length, random_seed); }); } diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index a42f5e6801eb..a3569b56e0b0 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -251,22 +251,26 @@ class JoinSource : public SourceWithProgress , max_block_size(max_block_size_) , sample_block(std::move(sample_block_)) { - columns.resize(sample_block.columns()); column_indices.resize(sample_block.columns()); - column_with_null.resize(sample_block.columns()); + + auto & saved_block = parent.getJoinedData()->sample_block; + for (size_t i = 0; i < sample_block.columns(); ++i) { auto & [_, type, name] = sample_block.getByPosition(i); if (parent.right_table_keys.has(name)) { key_pos = i; - column_with_null[i] = parent.right_table_keys.getByName(name).type->isNullable(); + const auto & column = parent.right_table_keys.getByName(name); + restored_block.insert(column); } else { - auto pos = parent.sample_block_with_columns_to_add.getPositionByName(name); + size_t pos = saved_block.getPositionByName(name); column_indices[i] = pos; - column_with_null[i] = !parent.sample_block_with_columns_to_add.getByPosition(pos).type->equals(*type); + + const auto & column = saved_block.getByPosition(pos); + restored_block.insert(column); } } } @@ -291,11 +295,10 @@ class JoinSource : public SourceWithProgress std::shared_lock lock; UInt64 max_block_size; Block sample_block; + Block restored_block; /// sample_block with parent column types ColumnNumbers column_indices; - std::vector column_with_null; std::optional key_pos; - MutableColumns columns; std::unique_ptr> position; /// type erasure @@ -303,23 +306,7 @@ class JoinSource : public SourceWithProgress template Chunk createChunk(const Maps & maps) { - for (size_t i = 0; i < sample_block.columns(); ++i) - { - const auto & src_col = sample_block.safeGetByPosition(i); - columns[i] = src_col.type->createColumn(); - if (column_with_null[i]) - { - if (key_pos == i) - { - // unwrap null key column - auto & nullable_col = assert_cast(*columns[i]); - columns[i] = nullable_col.getNestedColumnPtr()->assumeMutable(); - } - else - // wrap non key column with null - columns[i] = makeNullable(std::move(columns[i]))->assumeMutable(); - } - } + MutableColumns columns = restored_block.cloneEmpty().mutateColumns(); size_t rows_added = 0; @@ -327,7 +314,7 @@ class JoinSource : public SourceWithProgress { #define M(TYPE) \ case Join::Type::TYPE: \ - rows_added = fillColumns(*maps.TYPE); \ + rows_added = fillColumns(*maps.TYPE, columns); \ break; APPLY_FOR_JOIN_VARIANTS_LIMITED(M) #undef M @@ -340,29 +327,27 @@ class JoinSource : public SourceWithProgress if (!rows_added) return {}; - Columns res_columns; - res_columns.reserve(columns.size()); - + /// Correct nullability for (size_t i = 0; i < columns.size(); ++i) - if (column_with_null[i]) + { + bool src_nullable = restored_block.getByPosition(i).type->isNullable(); + bool dst_nullable = sample_block.getByPosition(i).type->isNullable(); + + if (src_nullable && !dst_nullable) { - if (key_pos == i) - res_columns.emplace_back(makeNullable(std::move(columns[i]))); - else - { - const auto & nullable_col = assert_cast(*columns[i]); - res_columns.emplace_back(makeNullable(nullable_col.getNestedColumnPtr())); - } + auto & nullable_column = assert_cast(*columns[i]); + columns[i] = nullable_column.getNestedColumnPtr()->assumeMutable(); } - else - res_columns.emplace_back(std::move(columns[i])); + else if (!src_nullable && dst_nullable) + columns[i] = makeNullable(std::move(columns[i]))->assumeMutable(); + } - UInt64 num_rows = res_columns.at(0)->size(); - return Chunk(std::move(res_columns), num_rows); + UInt64 num_rows = columns.at(0)->size(); + return Chunk(std::move(columns), num_rows); } template - size_t fillColumns(const Map & map) + size_t fillColumns(const Map & map, MutableColumns & columns) { size_t rows_added = 0; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 54953554ed10..e38a2463f979 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3322,7 +3322,9 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer zookeeper->multi(requests); { - /// TODO (relax this lock) + /// TODO (relax this lock and remove this action locks) + auto merges_block = getActionLock(ActionLocks::PartsMerge); + auto fetchers_block = getActionLock(ActionLocks::PartsFetch); auto table_lock = lockExclusively(RWLockImpl::NO_QUERY); LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 962150ebc298..ac0c84d21a9a 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -24,7 +24,7 @@ #include #include -#include +#include #include #include @@ -230,18 +230,18 @@ Strings listFilesWithRegexpMatching(Aws::S3::S3Client & client, const S3::URI & return {globbed_uri.key}; } - Aws::S3::Model::ListObjectsRequest request; + Aws::S3::Model::ListObjectsV2Request request; request.SetBucket(globbed_uri.bucket); request.SetPrefix(key_prefix); re2::RE2 matcher(makeRegexpPatternFromGlobs(globbed_uri.key)); Strings result; - Aws::S3::Model::ListObjectsOutcome outcome; + Aws::S3::Model::ListObjectsV2Outcome outcome; int page = 0; do { ++page; - outcome = client.ListObjects(request); + outcome = client.ListObjectsV2(request); if (!outcome.IsSuccess()) { throw Exception("Could not list objects in bucket " + quoteString(request.GetBucket()) @@ -256,7 +256,7 @@ Strings listFilesWithRegexpMatching(Aws::S3::S3Client & client, const S3::URI & result.emplace_back(std::move(key)); } - request.SetMarker(outcome.GetResult().GetNextMarker()); + request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); } while (outcome.GetResult().GetIsTruncated()); diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index cdb11393e58c..7dfab3491ae2 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -18,6 +18,8 @@ #include #include #include +#include +#include namespace DB @@ -61,29 +63,42 @@ Pipes StorageView::read( if (context.getSettings().enable_optimize_predicate_expression) current_inner_query = getRuntimeViewQuery(*query_info.query->as(), context); - QueryPipeline pipeline; InterpreterSelectWithUnionQuery interpreter(current_inner_query, context, {}, column_names); /// FIXME res may implicitly use some objects owned be pipeline, but them will be destructed after return if (query_info.force_tree_shaped_pipeline) { + QueryPipeline pipeline; BlockInputStreams streams = interpreter.executeWithMultipleStreams(pipeline); + + for (auto & stream : streams) + { + stream = std::make_shared(stream); + stream = std::make_shared(context, stream, getSampleBlockForColumns(column_names), + ConvertingBlockInputStream::MatchColumnsMode::Name); + } + for (auto & stream : streams) pipes.emplace_back(std::make_shared(std::move(stream))); } else - /// TODO: support multiple streams here. Need more general interface than pipes. - pipes.emplace_back(interpreter.executeWithProcessors().getPipe()); - - /// It's expected that the columns read from storage are not constant. - /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery. - for (auto & pipe : pipes) { - pipe.addSimpleTransform(std::make_shared(pipe.getHeader())); + auto pipeline = interpreter.executeWithProcessors(); + + /// It's expected that the columns read from storage are not constant. + /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery. + pipeline.addSimpleTransform([](const Block & header) + { + return std::make_shared(header); + }); /// And also convert to expected structure. - pipe.addSimpleTransform(std::make_shared( - pipe.getHeader(), getSampleBlockForColumns(column_names), - ConvertingTransform::MatchColumnsMode::Name, context)); + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, getSampleBlockForColumns(column_names), + ConvertingTransform::MatchColumnsMode::Name, context); + }); + + pipes = std::move(pipeline).getPipes(); } return pipes; diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index 42c503b8efa3..e729a87a1ce6 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -464,6 +464,7 @@ const char * auto_contributors[] { "ggerogery", "glockbender", "hcz", + "hexiaoting", "hotid", "igor", "igor.lapko", diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index 2e1ce33951a5..f0718a014b80 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -127,12 +128,21 @@ void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & c /// Filter the block. String filter_column_name = expression_ast->getColumnName(); ColumnPtr filter_column = block_with_filter.getByName(filter_column_name).column->convertToFullColumnIfConst(); - const IColumn::Filter & filter = typeid_cast(*filter_column).getData(); + + ConstantFilterDescription constant_filter(*filter_column); + + if (constant_filter.always_true) + return; + + if (constant_filter.always_false) + block = block.cloneEmpty(); + + FilterDescription filter(*filter_column); for (size_t i = 0; i < block.columns(); ++i) { ColumnPtr & column = block.safeGetByPosition(i).column; - column = column->filter(filter, -1); + column = column->filter(*filter.data, -1); } } diff --git a/src/Storages/getStructureOfRemoteTable.cpp b/src/Storages/getStructureOfRemoteTable.cpp index 731cbb5fee78..b9f8f6af54fe 100644 --- a/src/Storages/getStructureOfRemoteTable.cpp +++ b/src/Storages/getStructureOfRemoteTable.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 509897443892..c40fe0f3dbcb 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,8 @@ struct State {"apply_type", std::make_shared()}, {"apply_status", std::make_shared()}, {"create_time", std::make_shared()}, + {"field", std::make_shared()}, + {"value", std::make_shared()}, }; State() @@ -103,3 +106,11 @@ TEST(TransformQueryForExternalDatabase, Issue7245) R"(SELECT "apply_id", "apply_type", "apply_status", "create_time" FROM "test"."table" WHERE ("apply_type" = 2) AND ("create_time" > '2018-12-25 01:02:03') AND ("apply_status" IN (3, 4)))", state().context, state().columns); } + +TEST(TransformQueryForExternalDatabase, Aliases) +{ + + check("SELECT field AS value, field AS display WHERE field NOT IN ('') AND display LIKE '%test%'", + R"(SELECT "field" FROM "test"."table" WHERE ("field" NOT IN ('')) AND ("field" LIKE '%test%'))", + state().context, state().columns); +} diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index 77efedc684da..b62341ac06ac 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -70,6 +70,24 @@ class ReplacingConstantExpressionsMatcher } }; +class DropAliasesMatcher +{ +public: + struct Data {}; + Data data; + + static bool needChildVisit(ASTPtr &, const ASTPtr &) + { + return true; + } + + static void visit(ASTPtr & node, Data) + { + if (!node->tryGetAlias().empty()) + node->setAlias({}); + } +}; + void replaceConstantExpressions(ASTPtr & node, const Context & context, const NamesAndTypesList & all_columns) { auto syntax_result = SyntaxAnalyzer(context).analyze(node, all_columns); @@ -79,6 +97,13 @@ void replaceConstantExpressions(ASTPtr & node, const Context & context, const Na visitor.visit(node); } +void dropAliases(ASTPtr & node) +{ + DropAliasesMatcher::Data data; + InDepthNodeVisitor visitor(data); + visitor.visit(node); +} + bool isCompatible(const IAST & node) { @@ -191,6 +216,9 @@ String transformQueryForExternalDatabase( } } + ASTPtr select_ptr = select; + dropAliases(select_ptr); + std::stringstream out; IAST::FormatSettings settings(out, true); settings.identifier_quoting_style = identifier_quoting_style; diff --git a/src/TableFunctions/ITableFunctionXDBC.cpp b/src/TableFunctions/ITableFunctionXDBC.cpp index 50236b654458..1a2079a7ce6f 100644 --- a/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/src/TableFunctions/ITableFunctionXDBC.cpp @@ -77,7 +77,7 @@ StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Co columns_info_uri.addQueryParameter("external_table_functions_use_nulls", Poco::NumberFormatter::format(use_nulls)); - ReadWriteBufferFromHTTP buf(columns_info_uri, Poco::Net::HTTPRequest::HTTP_POST, nullptr); + ReadWriteBufferFromHTTP buf(columns_info_uri, Poco::Net::HTTPRequest::HTTP_POST, {}, ConnectionTimeouts::getHTTPTimeouts(context)); std::string columns_info; readStringBinary(columns_info, buf); diff --git a/src/TableFunctions/TableFunctionValues.cpp b/src/TableFunctions/TableFunctionValues.cpp index 6f568fbea60a..71c6f5a379e1 100644 --- a/src/TableFunctions/TableFunctionValues.cpp +++ b/src/TableFunctions/TableFunctionValues.cpp @@ -38,7 +38,7 @@ static void parseAndInsertValues(MutableColumns & res_columns, const ASTs & args { const auto & [value_field, value_type_ptr] = evaluateConstantExpression(args[i], context); - Field value = convertFieldToType(value_field, *sample_block.getByPosition(0).type, value_type_ptr.get()); + Field value = convertFieldToTypeOrThrow(value_field, *sample_block.getByPosition(0).type, value_type_ptr.get()); res_columns[0]->insert(value); } } @@ -51,11 +51,11 @@ static void parseAndInsertValues(MutableColumns & res_columns, const ASTs & args const Tuple & value_tuple = value_field.safeGet(); if (value_tuple.size() != sample_block.columns()) - throw Exception("Values size should match with number of columns", ErrorCodes::LOGICAL_ERROR); + throw Exception("Values size should match with number of columns", ErrorCodes::BAD_ARGUMENTS); for (size_t j = 0; j < value_tuple.size(); ++j) { - Field value = convertFieldToType(value_tuple[j], *sample_block.getByPosition(j).type, value_types_tuple[j].get()); + Field value = convertFieldToTypeOrThrow(value_tuple[j], *sample_block.getByPosition(j).type, value_types_tuple[j].get()); res_columns[j]->insert(value); } } diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index b796c39151c6..cfdeac9e1c6d 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -473,7 +473,7 @@ def start(self, destroy_dirs=True): instance.client = Client(instance.ip_address, command=self.client_bin_path) self.is_up = True - + except BaseException, e: print "Failed to start cluster: " print str(e) @@ -505,6 +505,13 @@ def shutdown(self, kill=True): if sanitizer_assert_instance is not None: raise Exception("Sanitizer assert found in {} for instance {}".format(self.docker_logs_path, sanitizer_assert_instance)) + def pause_container(self, instance_name): + subprocess_check_call(self.base_cmd + ['pause', instance_name]) + # subprocess_check_call(self.base_cmd + ['kill', '-s SIGSTOP', instance_name]) + + def unpause_container(self, instance_name): + subprocess_check_call(self.base_cmd + ['unpause', instance_name]) + # subprocess_check_call(self.base_cmd + ['kill', '-s SIGCONT', instance_name]) def open_bash_shell(self, instance_name): os.system(' '.join(self.base_cmd + ['exec', instance_name, '/bin/bash'])) diff --git a/tests/integration/test_adaptive_granularity_different_settings/test.py b/tests/integration/test_adaptive_granularity_different_settings/test.py index b066c437e06a..d1ff785cfba7 100644 --- a/tests/integration/test_adaptive_granularity_different_settings/test.py +++ b/tests/integration/test_adaptive_granularity_different_settings/test.py @@ -6,6 +6,9 @@ node1 = cluster.add_instance('node1', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True) +# no adaptive granularity by default +node3 = cluster.add_instance('node3', image='yandex/clickhouse-server:19.9.5.36', with_installed_binary=True, stay_alive=True) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -47,3 +50,29 @@ def test_attach_detach(start_cluster): assert node1.query("SELECT COUNT() FROM test") == "4\n" assert node2.query("SELECT COUNT() FROM test") == "4\n" + + +def test_mutate_with_mixed_granularity(start_cluster): + node3.query(""" + CREATE TABLE test (date Date, key UInt64, value1 String, value2 String) + ENGINE = MergeTree + ORDER BY key PARTITION BY date""") + + node3.query("INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500)") + + assert node3.query("SELECT COUNT() FROM test") == "500\n" + + node3.restart_with_latest_version() + + assert node3.query("SELECT COUNT() FROM test") == "500\n" + + node3.query("ALTER TABLE test MODIFY SETTING enable_mixed_granularity_parts = 1") + + node3.query("INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500, 500)") + + assert node3.query("SELECT COUNT() FROM test") == "1000\n" + assert node3.query("SELECT COUNT() FROM test WHERE key % 100 == 0") == "10\n" + + node3.query("ALTER TABLE test DELETE WHERE key % 100 == 0", settings={"mutations_sync": "2"}) + + assert node3.query("SELECT COUNT() FROM test WHERE key % 100 == 0") == "0\n" diff --git a/tests/integration/test_authentication/test.py b/tests/integration/test_authentication/test.py index 11ca967fbeee..8dcfd1f2dbd5 100644 --- a/tests/integration/test_authentication/test.py +++ b/tests/integration/test_authentication/test.py @@ -23,6 +23,10 @@ def test_authentication_pass(): assert instance.query("SELECT currentUser()", user='sasha') == 'sasha\n' assert instance.query("SELECT currentUser()", user='masha', password='qwerty') == 'masha\n' + # 'no_password' authentication type allows to login with any password. + assert instance.query("SELECT currentUser()", user='sasha', password='something') == 'sasha\n' + assert instance.query("SELECT currentUser()", user='sasha', password='something2') == 'sasha\n' + def test_authentication_fail(): # User doesn't exist. diff --git a/tests/integration/test_dictionaries_access/__init__.py b/tests/integration/test_dictionaries_access/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_dictionaries_access/test.py b/tests/integration/test_dictionaries_access/test.py new file mode 100644 index 000000000000..3d8f090c9e6c --- /dev/null +++ b/tests/integration/test_dictionaries_access/test.py @@ -0,0 +1,93 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance') + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + instance.query("CREATE USER mira") + instance.query("CREATE TABLE test_table(x Int32, y Int32) ENGINE=Log") + instance.query("INSERT INTO test_table VALUES (5,6)") + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def clear_after_test(): + try: + yield + finally: + instance.query("CREATE USER OR REPLACE mira") + instance.query("DROP DICTIONARY IF EXISTS test_dict") + + +create_query = """ + CREATE DICTIONARY test_dict(x Int32, y Int32) PRIMARY KEY x + LAYOUT(FLAT()) + SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test_table' DB 'default')) + LIFETIME(0) + """ + +drop_query = "DROP DICTIONARY test_dict" + + +def test_create(): + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + + instance.query("GRANT CREATE DICTIONARY ON *.* TO mira") + instance.query(create_query, user="mira") + instance.query(drop_query) + + instance.query("REVOKE CREATE DICTIONARY ON *.* FROM mira") + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + + instance.query("GRANT CREATE DICTIONARY ON default.* TO mira") + instance.query(create_query, user="mira") + instance.query(drop_query) + + instance.query("REVOKE CREATE DICTIONARY ON default.* FROM mira") + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + + instance.query("GRANT CREATE DICTIONARY ON default.test_dict TO mira") + instance.query(create_query, user="mira") + + +def test_drop(): + instance.query(create_query) + + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(drop_query, user="mira") + + instance.query("GRANT DROP DICTIONARY ON *.* TO mira") + instance.query(drop_query, user="mira") + instance.query(create_query) + + +@pytest.mark.skip(reason="GRANT dictGet doesn't work well in 20.3") +def test_dictget(): + instance.query(create_query) + + dictget_query = "SELECT dictGet('default.test_dict', 'y', toUInt64(5))" + instance.query(dictget_query) == "6\n" + assert "Not enough privileges" in instance.query_and_get_error(dictget_query, user='mira') + + instance.query("GRANT dictGet ON default.test_dict TO mira") + instance.query(dictget_query, user='mira') == "6\n" + + dictget_query = "SELECT dictGet('default.test_dict', 'y', toUInt64(1))" + instance.query(dictget_query) == "0\n" + instance.query(dictget_query, user='mira') == "0\n" + + instance.query("REVOKE dictGet ON *.* FROM mira") + assert "Not enough privileges" in instance.query_and_get_error(dictget_query, user='mira') diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/configs/dictionaries/cache_strings_default_settings.xml b/tests/integration/test_dictionary_allow_read_expired_keys/configs/dictionaries/cache_strings_default_settings.xml new file mode 100644 index 000000000000..11807bc1ad69 --- /dev/null +++ b/tests/integration/test_dictionary_allow_read_expired_keys/configs/dictionaries/cache_strings_default_settings.xml @@ -0,0 +1,35 @@ + + + default_string + + + dictionary_node + 9000 + default + + test + strings
+ + + + 2 + 1 + + + + 1000 + 10000 + + + + + key + + + value + String + > + + + + \ No newline at end of file diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py new file mode 100644 index 000000000000..7d762db2a6dd --- /dev/null +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py @@ -0,0 +1,61 @@ +from __future__ import print_function +import pytest +import os +import random +import string +import time + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs')) + +dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) +main_node = cluster.add_instance('main_node', main_configs=['configs/dictionaries/cache_strings_default_settings.xml']) + + +def get_random_string(string_length=8): + alphabet = string.ascii_letters + string.digits + return ''.join((random.choice(alphabet) for _ in range(string_length))) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + dictionary_node.query("CREATE DATABASE IF NOT EXISTS test;") + dictionary_node.query("DROP TABLE IF EXISTS test.strings;") + dictionary_node.query(""" + CREATE TABLE test.strings + (key UInt64, value String) + ENGINE = Memory; + """) + + values_to_insert = ", ".join(["({}, '{}')".format(1000000 + number, get_random_string()) for number in range(100)]) + dictionary_node.query("INSERT INTO test.strings VALUES {}".format(values_to_insert)) + + yield cluster + finally: + cluster.shutdown() + +# @pytest.mark.skip(reason="debugging") +def test_return_real_values(started_cluster): + assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" + + first_batch = """ + SELECT count(*) + FROM + ( + SELECT + arrayJoin(arrayMap(x -> (x + 1000000), range(100))) AS id, + dictGetString('default_string', 'value', toUInt64(id)) AS value + ) + WHERE value = ''; + """ + + assert TSV("0") == TSV(main_node.query(first_batch)) + + # Waiting for cache to become expired + time.sleep(5) + + assert TSV("0") == TSV(main_node.query(first_batch)) diff --git a/tests/integration/test_distributed_format/configs/remote_servers.xml b/tests/integration/test_distributed_format/configs/remote_servers.xml index 7d8d64bb78ba..5c86713bd785 100644 --- a/tests/integration/test_distributed_format/configs/remote_servers.xml +++ b/tests/integration/test_distributed_format/configs/remote_servers.xml @@ -8,5 +8,12 @@ + + + + not_existing + 9000 + + diff --git a/tests/integration/test_distributed_format/test.py b/tests/integration/test_distributed_format/test.py index 3f139c8a6eb8..6a9f27aa83c0 100644 --- a/tests/integration/test_distributed_format/test.py +++ b/tests/integration/test_distributed_format/test.py @@ -11,67 +11,75 @@ cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', config_dir="configs", main_configs=['configs/remote_servers.xml']) +cluster_param = pytest.mark.parametrize("cluster", [ + ('test_cluster'), + ('test_cluster_2'), +]) + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() + node.query("create database test engine=Ordinary") yield cluster finally: cluster.shutdown() +@cluster_param +def test_single_file_new(started_cluster, cluster): + node.query("create table test.distr_1 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) + node.query("insert into test.distr_1 values (1, 'a'), (2, 'bb'), (3, 'ccc')", settings={"use_compact_format_in_distributed_parts_names": "1"}) -def test_single_file_new(started_cluster): - node.query("create table distr_1 (x UInt64, s String) engine = Distributed('test_cluster', database, table)") - node.query("insert into distr_1 values (1, 'a'), (2, 'bb'), (3, 'ccc')", settings={"use_compact_format_in_distributed_parts_names": "1"}) - - query = "select * from file('/var/lib/clickhouse/data/default/distr_1/shard1_replica1/1.bin', 'Distributed')" + query = "select * from file('/var/lib/clickhouse/data/test/distr_1/shard1_replica1/1.bin', 'Distributed')" out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) assert out == '1\ta\n2\tbb\n3\tccc\n' - query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/default/distr_1/shard1_replica1/1.bin');" \ + query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/test/distr_1/shard1_replica1/1.bin');" \ "select * from t" out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) assert out == '1\ta\n2\tbb\n3\tccc\n' - node.query("drop table distr_1") + node.query("drop table test.distr_1") -def test_two_files(started_cluster): - node.query("create table distr_2 (x UInt64, s String) engine = Distributed('test_cluster', database, table)") - node.query("insert into distr_2 values (0, '_'), (1, 'a')", settings={"use_compact_format_in_distributed_parts_names": "1"}) - node.query("insert into distr_2 values (2, 'bb'), (3, 'ccc')", settings={"use_compact_format_in_distributed_parts_names": "1"}) +@cluster_param +def test_two_files(started_cluster, cluster): + node.query("create table test.distr_2 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) + node.query("insert into test.distr_2 values (0, '_'), (1, 'a')", settings={"use_compact_format_in_distributed_parts_names": "1"}) + node.query("insert into test.distr_2 values (2, 'bb'), (3, 'ccc')", settings={"use_compact_format_in_distributed_parts_names": "1"}) - query = "select * from file('/var/lib/clickhouse/data/default/distr_2/shard1_replica1/{1,2,3,4}.bin', 'Distributed') order by x" + query = "select * from file('/var/lib/clickhouse/data/test/distr_2/shard1_replica1/{1,2,3,4}.bin', 'Distributed') order by x" out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) assert out == '0\t_\n1\ta\n2\tbb\n3\tccc\n' - query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/default/distr_2/shard1_replica1/{1,2,3,4}.bin');" \ + query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/test/distr_2/shard1_replica1/{1,2,3,4}.bin');" \ "select * from t order by x" out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) assert out == '0\t_\n1\ta\n2\tbb\n3\tccc\n' - node.query("drop table distr_2") + node.query("drop table test.distr_2") -def test_single_file_old(started_cluster): - node.query("create table distr_3 (x UInt64, s String) engine = Distributed('test_cluster', database, table)") - node.query("insert into distr_3 values (1, 'a'), (2, 'bb'), (3, 'ccc')") +@cluster_param +def test_single_file_old(started_cluster, cluster): + node.query("create table test.distr_3 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) + node.query("insert into test.distr_3 values (1, 'a'), (2, 'bb'), (3, 'ccc')") - query = "select * from file('/var/lib/clickhouse/data/default/distr_3/default@not_existing:9000/1.bin', 'Distributed')" + query = "select * from file('/var/lib/clickhouse/data/test/distr_3/default@not_existing:9000/1.bin', 'Distributed')" out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) assert out == '1\ta\n2\tbb\n3\tccc\n' - query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/default/distr_3/default@not_existing:9000/1.bin');" \ + query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/test/distr_3/default@not_existing:9000/1.bin');" \ "select * from t" out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) assert out == '1\ta\n2\tbb\n3\tccc\n' - node.query("drop table distr_3") + node.query("drop table test.distr_3") diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index f4af04b5fbc7..16f4b3667dd3 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -91,8 +91,8 @@ def test_mysql_simple_select_works(started_cluster): with conn.cursor() as cursor: cursor.execute("INSERT INTO clickhouse.{} VALUES(50, 'null-guy', 127, 255, NULL), (100, 'non-null-guy', 127, 255, 511);".format(table_name)) conn.commit() - assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}') SETTINGS external_table_functions_use_nulls=1".format(mysql_setup["DSN"], table_name)) == '\\N\n511\n' - assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}') SETTINGS external_table_functions_use_nulls=0".format(mysql_setup["DSN"], table_name)) == '0\n511\n' + assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}')".format(mysql_setup["DSN"], table_name), settings={"external_table_functions_use_nulls": "1"}) == '\\N\n511\n' + assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}')".format(mysql_setup["DSN"], table_name), settings={"external_table_functions_use_nulls": "0"}) == '0\n511\n' node1.query(''' CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, column_x Nullable(UInt32)) ENGINE = MySQL('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse'); diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index 6db24f5799e5..9f067bdb821c 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -8,6 +8,7 @@ cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance1', config_dir="configs", with_zookeeper=True) instance2 = cluster.add_instance('instance2', config_dir="configs", with_zookeeper=True) +instances = [instance, instance2] def copy_policy_xml(local_file_name, reload_immediately = True): @@ -23,42 +24,25 @@ def started_cluster(): try: cluster.start() - instance.query(''' - CREATE DATABASE mydb; - - CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.table (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.table values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.filtered_table2 (a UInt8, b UInt8, c UInt8, d UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table2 values (0, 0, 0, 0), (1, 2, 3, 4), (4, 3, 2, 1), (0, 0, 6, 0); - - CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table3 values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1); - ''') - instance2.query(''' - CREATE DATABASE mydb; - - CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.table (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.table values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.filtered_table2 (a UInt8, b UInt8, c UInt8, d UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table2 values (0, 0, 0, 0), (1, 2, 3, 4), (4, 3, 2, 1), (0, 0, 6, 0); - - CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table3 values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1); - ''') + for current_instance in instances: + current_instance.query(''' + CREATE DATABASE mydb ENGINE=Ordinary; + CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1); + CREATE TABLE mydb.table (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.table values (0, 0), (0, 1), (1, 0), (1, 1); + CREATE TABLE mydb.filtered_table2 (a UInt8, b UInt8, c UInt8, d UInt8) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.filtered_table2 values (0, 0, 0, 0), (1, 2, 3, 4), (4, 3, 2, 1), (0, 0, 6, 0); + CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.filtered_table3 values (0, 0), (0, 1), (1, 0), (1, 1); + CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1); + + CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; + ''') + + instance.query("INSERT INTO mydb.local values (2, 0), (2, 1), (1, 0), (1, 1)") + instance2.query("INSERT INTO mydb.local values (3, 0), (3, 1), (1, 0), (1, 1)") yield cluster @@ -134,6 +118,17 @@ def test_single_table_name(): assert instance.query("SELECT a + b = 1 FROM mydb.filtered_table3") == "1\n1\n" +def test_policy_from_users_xml_affects_only_user_assigned(): + assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" + assert instance.query("SELECT * FROM mydb.filtered_table1", user="another") == "0\t0\n0\t1\n1\t0\n1\t1\n" + + assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" + assert instance.query("SELECT * FROM mydb.filtered_table2", user="another") == "0\t0\t0\t0\n0\t0\t6\t0\n1\t2\t3\t4\n4\t3\t2\t1\n" + + assert instance.query("SELECT * FROM mydb.local") == "1\t0\n1\t1\n2\t0\n2\t1\n" + assert instance.query("SELECT * FROM mydb.local", user="another") == "1\t0\n1\t1\n" + + def test_custom_table_name(): copy_policy_xml('multiple_tags_with_table_names.xml') assert instance.query("SELECT * FROM mydb.table") == "1\t0\n1\t1\n" @@ -303,9 +298,5 @@ def test_miscellaneous_engines(): # DistributedMergeTree instance.query("DROP TABLE IF EXISTS mydb.not_filtered_table") instance.query("CREATE TABLE mydb.not_filtered_table (a UInt8, b UInt8) ENGINE Distributed('test_local_cluster', mydb, local)") - instance.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") - instance2.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") - instance.query("INSERT INTO mydb.local values (2, 0), (2, 1), (1, 0), (1, 1)") - instance2.query("INSERT INTO mydb.local values (3, 0), (3, 1), (1, 0), (1, 1)") assert instance.query("SELECT * FROM mydb.not_filtered_table", user="another") == "1\t0\n1\t1\n1\t0\n1\t1\n" assert instance.query("SELECT sum(a), b FROM mydb.not_filtered_table GROUP BY b ORDER BY b", user="another") == "2\t0\n2\t1\n" diff --git a/tests/integration/test_server_initialization/test.py b/tests/integration/test_server_initialization/test.py index 6278bdc8d34c..0ba8f688566c 100644 --- a/tests/integration/test_server_initialization/test.py +++ b/tests/integration/test_server_initialization/test.py @@ -30,9 +30,9 @@ def test_sophisticated_default(started_cluster): def test_partially_dropped_tables(started_cluster): instance = started_cluster.instances['dummy'] - assert instance.exec_in_container(['bash', '-c', 'cd / && find -name *.sql* | sort'], privileged=True, user='root') \ - == "./var/lib/clickhouse/metadata/default/should_be_restored.sql\n" \ - "./var/lib/clickhouse/metadata/default/sophisticated_default.sql\n" + assert instance.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse -name *.sql* | sort'], privileged=True, user='root') \ + == "/var/lib/clickhouse/metadata/default/should_be_restored.sql\n" \ + "/var/lib/clickhouse/metadata/default/sophisticated_default.sql\n" assert instance.query("SELECT n FROM should_be_restored") == "1\n2\n3\n" assert instance.query("SELECT count() FROM system.tables WHERE name='should_be_dropped'") == "0\n" diff --git a/tests/integration/test_settings_constraints_distributed/test.py b/tests/integration/test_settings_constraints_distributed/test.py index b23b130b2700..e34d83e49704 100644 --- a/tests/integration/test_settings_constraints_distributed/test.py +++ b/tests/integration/test_settings_constraints_distributed/test.py @@ -35,6 +35,7 @@ def started_cluster(): cluster.shutdown() +@pytest.mark.skip(reason="CREATE USER statement doesn't support SETTINGS clause in 20.3") def test_select_clamps_settings(started_cluster): distributed.query("CREATE USER normal DEFAULT ROLE admin SETTINGS max_memory_usage = 80000000") distributed.query("CREATE USER wasteful DEFAULT ROLE admin SETTINGS max_memory_usage = 2000000000") @@ -89,6 +90,7 @@ def test_select_clamps_settings(started_cluster): 'node2\tmax_memory_usage\t10000000000\n'\ 'node2\treadonly\t1\n' +@pytest.mark.skip(reason="ALTER USER statement doesn't support SETTINGS clause in 20.3") def test_insert_clamps_settings(started_cluster): node1.query("ALTER USER shard SETTINGS max_memory_usage = 50000000 MIN 11111111 MAX 99999999") node2.query("ALTER USER shard SETTINGS max_memory_usage = 50000000 MIN 11111111 MAX 99999999") diff --git a/tests/integration/test_storage_kafka/configs/kafka.xml b/tests/integration/test_storage_kafka/configs/kafka.xml index e5c07881e065..ce1a2f502ad7 100644 --- a/tests/integration/test_storage_kafka/configs/kafka.xml +++ b/tests/integration/test_storage_kafka/configs/kafka.xml @@ -1,5 +1,20 @@ earliest + + cgrp,consumer,topic,protocol + + + + 300 + + 6000 + diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 3242b59bca3f..13577864870f 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -28,7 +28,6 @@ # TODO: add test for run-time offset update in CH, if we manually update it on Kafka side. # TODO: add test for SELECT LIMIT is working. -# TODO: modify tests to respect `skip_broken_messages` setting. cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', @@ -199,6 +198,98 @@ def test_kafka_settings_new_syntax(kafka_cluster): kafka_check_result(result, True) +@pytest.mark.timeout(180) +def test_kafka_consumer_hang(kafka_cluster): + + instance.query(''' + DROP TABLE IF EXISTS test.kafka; + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.consumer; + + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'consumer_hang', + kafka_group_name = 'consumer_hang', + kafka_format = 'JSONEachRow', + kafka_num_consumers = 8, + kafka_row_delimiter = '\\n'; + CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = Memory(); + CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; + ''') + + time.sleep(10) + instance.query('SELECT * FROM test.view') + + # This should trigger heartbeat fail, + # which will trigger REBALANCE_IN_PROGRESS, + # and which can lead to consumer hang. + kafka_cluster.pause_container('kafka1') + time.sleep(0.5) + kafka_cluster.unpause_container('kafka1') + + # print("Attempt to drop") + instance.query('DROP TABLE test.kafka') + + #kafka_cluster.open_bash_shell('instance') + + instance.query(''' + DROP TABLE test.consumer; + DROP TABLE test.view; + ''') + + # original problem appearance was a sequence of the following messages in librdkafka logs: + # BROKERFAIL -> |ASSIGN| -> REBALANCE_IN_PROGRESS -> "waiting for rebalance_cb" (repeated forever) + # so it was waiting forever while the application will execute queued rebalance callback + + # from a user perspective: we expect no hanging 'drop' queries + # 'dr'||'op' to avoid self matching + assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 + +@pytest.mark.timeout(180) +def test_kafka_consumer_hang2(kafka_cluster): + + instance.query(''' + DROP TABLE IF EXISTS test.kafka; + + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'consumer_hang2', + kafka_group_name = 'consumer_hang2', + kafka_format = 'JSONEachRow'; + + CREATE TABLE test.kafka2 (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'consumer_hang2', + kafka_group_name = 'consumer_hang2', + kafka_format = 'JSONEachRow'; + ''') + + # first consumer subscribe the topic, try to poll some data, and go to rest + instance.query('SELECT * FROM test.kafka') + + # second consumer do the same leading to rebalance in the first + # consumer, try to poll some data + instance.query('SELECT * FROM test.kafka2') + +#echo 'SELECT * FROM test.kafka; SELECT * FROM test.kafka2; DROP TABLE test.kafka;' | clickhouse client -mn & +# kafka_cluster.open_bash_shell('instance') + + # first consumer has pending rebalance callback unprocessed (no poll after select) + # one of those queries was failing because of + # https://github.com/edenhill/librdkafka/issues/2077 + # https://github.com/edenhill/librdkafka/issues/2898 + instance.query('DROP TABLE test.kafka') + instance.query('DROP TABLE test.kafka2') + + + # from a user perspective: we expect no hanging 'drop' queries + # 'dr'||'op' to avoid self matching + assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 + + @pytest.mark.timeout(180) def test_kafka_csv_with_delimiter(kafka_cluster): instance.query(''' @@ -926,7 +1017,10 @@ def test_kafka_flush_by_block_size(kafka_cluster): time.sleep(1) - result = instance.query('SELECT count() FROM test.view') + # TODO: due to https://github.com/ClickHouse/ClickHouse/issues/11216 + # second flush happens earlier than expected, so we have 2 parts here instead of one + # flush by block size works correctly, so the feature checked by the test is working correctly + result = instance.query("SELECT count() FROM test.view WHERE _part='all_1_1_0'") # print(result) # kafka_cluster.open_bash_shell('instance') @@ -1083,6 +1177,7 @@ def produce(): print(instance.query('SELECT count(), uniqExact(key), max(key) + 1 FROM test.destination')) + # Some queries to debug... # SELECT * FROM test.destination where key in (SELECT key FROM test.destination group by key having count() <> 1) # select number + 1 as key from numbers(4141) left join test.destination using (key) where test.destination.key = 0; # SELECT * FROM test.destination WHERE key between 2360 and 2370 order by key; @@ -1090,6 +1185,18 @@ def produce(): # select toUInt64(0) as _partition, number + 1 as _offset from numbers(400) left join test.destination using (_partition,_offset) where test.destination.key = 0 order by _offset; # SELECT * FROM test.destination WHERE _partition = 0 and _offset between 220 and 240 order by _offset; + # CREATE TABLE test.reference (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', + # kafka_topic_list = 'topic_with_multiple_partitions', + # kafka_group_name = 'rebalance_test_group_reference', + # kafka_format = 'JSONEachRow', + # kafka_max_block_size = 100000; + # + # CREATE MATERIALIZED VIEW test.reference_mv Engine=Log AS + # SELECT key, value, _topic,_key,_offset, _partition, _timestamp, 'reference' as _consumed_by + # FROM test.reference; + # + # select * from test.reference_mv left join test.destination using (key,_topic,_offset,_partition) where test.destination._consumed_by = ''; + result = int(instance.query('SELECT count() == uniqExact(key) FROM test.destination')) for consumer_index in range(NUMBER_OF_CONSURRENT_CONSUMERS): @@ -1190,10 +1297,197 @@ def test_exception_from_destructor(kafka_cluster): DROP TABLE test.kafka; ''') - kafka_cluster.open_bash_shell('instance') + #kafka_cluster.open_bash_shell('instance') assert TSV(instance.query('SELECT 1')) == TSV('1') +@pytest.mark.timeout(120) +def test_commits_of_unprocessed_messages_on_drop(kafka_cluster): + messages = [json.dumps({'key': j+1, 'value': j+1}) for j in range(1)] + kafka_produce('commits_of_unprocessed_messages_on_drop', messages) + + instance.query(''' + DROP TABLE IF EXISTS test.destination; + CREATE TABLE test.destination ( + key UInt64, + value UInt64, + _topic String, + _key String, + _offset UInt64, + _partition UInt64, + _timestamp Nullable(DateTime), + _consumed_by LowCardinality(String) + ) + ENGINE = MergeTree() + ORDER BY key; + + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'commits_of_unprocessed_messages_on_drop', + kafka_group_name = 'commits_of_unprocessed_messages_on_drop_test_group', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 1000; + + CREATE MATERIALIZED VIEW test.kafka_consumer TO test.destination AS + SELECT + key, + value, + _topic, + _key, + _offset, + _partition, + _timestamp + FROM test.kafka; + ''') + + while int(instance.query("SELECT count() FROM test.destination")) == 0: + print("Waiting for test.kafka_consumer to start consume") + time.sleep(1) + + cancel = threading.Event() + + i = [2] + def produce(): + while not cancel.is_set(): + messages = [] + for _ in range(113): + messages.append(json.dumps({'key': i[0], 'value': i[0]})) + i[0] += 1 + kafka_produce('commits_of_unprocessed_messages_on_drop', messages) + time.sleep(1) + + kafka_thread = threading.Thread(target=produce) + kafka_thread.start() + time.sleep(12) + + instance.query(''' + DROP TABLE test.kafka; + ''') + + instance.query(''' + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'commits_of_unprocessed_messages_on_drop', + kafka_group_name = 'commits_of_unprocessed_messages_on_drop_test_group', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 10000; + ''') + + cancel.set() + time.sleep(15) + + #kafka_cluster.open_bash_shell('instance') + # SELECT key, _timestamp, _offset FROM test.destination where runningDifference(key) <> 1 ORDER BY key; + + result = instance.query('SELECT count(), uniqExact(key), max(key) FROM test.destination') + print(result) + + instance.query(''' + DROP TABLE test.kafka_consumer; + DROP TABLE test.destination; + ''') + + kafka_thread.join() + assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(i[0]-1)), 'Missing data!' + + + +@pytest.mark.timeout(120) +def test_bad_reschedule(kafka_cluster): + messages = [json.dumps({'key': j+1, 'value': j+1}) for j in range(20000)] + kafka_produce('test_bad_reschedule', messages) + + instance.query(''' + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'test_bad_reschedule', + kafka_group_name = 'test_bad_reschedule', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 1000; + + CREATE MATERIALIZED VIEW test.destination Engine=Log AS + SELECT + key, + now() as consume_ts, + value, + _topic, + _key, + _offset, + _partition, + _timestamp + FROM test.kafka; + ''') + + while int(instance.query("SELECT count() FROM test.destination")) < 20000: + print("Waiting for consume") + time.sleep(1) + + assert int(instance.query("SELECT max(consume_ts) - min(consume_ts) FROM test.destination")) < 8 + + +@pytest.mark.timeout(1200) +def test_kafka_duplicates_when_commit_failed(kafka_cluster): + messages = [json.dumps({'key': j+1, 'value': 'x' * 300}) for j in range(22)] + kafka_produce('duplicates_when_commit_failed', messages) + + instance.query(''' + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.consumer; + + CREATE TABLE test.kafka (key UInt64, value String) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'duplicates_when_commit_failed', + kafka_group_name = 'duplicates_when_commit_failed', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 20; + + CREATE TABLE test.view (key UInt64, value String) + ENGINE = MergeTree() + ORDER BY key; + + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.kafka + WHERE NOT sleepEachRow(0.5); + ''') + + #print time.strftime("%m/%d/%Y %H:%M:%S") + time.sleep(12) # 5-6 sec to connect to kafka, do subscription, and fetch 20 rows, another 10 sec for MV, after that commit should happen + + #print time.strftime("%m/%d/%Y %H:%M:%S") + kafka_cluster.pause_container('kafka1') + # that timeout it VERY important, and picked after lot of experiments + # when too low (<30sec) librdkafka will not report any timeout (alternative is to decrease the default session timeouts for librdkafka) + # when too high (>50sec) broker will decide to remove us from the consumer group, and will start answering "Broker: Unknown member" + time.sleep(40) + + #print time.strftime("%m/%d/%Y %H:%M:%S") + kafka_cluster.unpause_container('kafka1') + + #kafka_cluster.open_bash_shell('instance') + + # connection restored and it will take a while until next block will be flushed + # it takes years on CI :\ + time.sleep(30) + + # as it's a bit tricky to hit the proper moment - let's check in logs if we did it correctly + assert instance.contains_in_log("Local: Waiting for coordinator") + + result = instance.query('SELECT count(), uniqExact(key), max(key) FROM test.view') + print(result) + + instance.query(''' + DROP TABLE test.consumer; + DROP TABLE test.view; + ''') + + assert TSV(result) == TSV('22\t22\t22') + + + if __name__ == '__main__': cluster.start() raw_input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index b92fbba058cf..648f9640084b 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -1,6 +1,7 @@ import json import logging import random +import threading import pytest @@ -284,3 +285,31 @@ def test_wrong_s3_syntax(cluster, s3_storage_args): query = "create table test_table_s3_syntax (id UInt32) ENGINE = S3({})".format(s3_storage_args) assert expected_err_msg in instance.query_and_get_error(query) + + +# https://en.wikipedia.org/wiki/One_Thousand_and_One_Nights +def test_s3_glob_scheherazade(cluster): + bucket = cluster.minio_bucket + instance = cluster.instances["dummy"] # type: ClickHouseInstance + table_format = "column1 UInt32, column2 UInt32, column3 UInt32" + max_path = "" + values = "(1, 1, 1)" + nights_per_job = 1001 // 30 + jobs = [] + for night in range(0, 1001, nights_per_job): + def add_tales(start, end): + for i in range(start, end): + path = "night_{}/tale.csv".format(i) + query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( + cluster.minio_host, cluster.minio_port, bucket, path, table_format, values) + run_query(instance, query) + + jobs.append(threading.Thread(target=add_tales, args=(night, min(night+nights_per_job, 1001)))) + jobs[-1].start() + + for job in jobs: + job.join() + + query = "select count(), sum(column1), sum(column2), sum(column3) from s3('http://{}:{}/{}/night_*/tale.csv', 'CSV', '{}')".format( + cluster.minio_redirect_host, cluster.minio_redirect_port, bucket, table_format) + assert run_query(instance, query).splitlines() == ["1001\t1001\t1001\t1001"] diff --git a/tests/performance/parallel_mv.xml b/tests/performance/parallel_mv.xml new file mode 100644 index 000000000000..ef50d5067085 --- /dev/null +++ b/tests/performance/parallel_mv.xml @@ -0,0 +1,24 @@ + + + 1 + + + create table main_table (number UInt64) engine = MergeTree order by tuple(); + create materialized view mv_1 engine = MergeTree order by tuple() as + select number, toString(number) from main_table where number % 13 != 0; + create materialized view mv_2 engine = MergeTree order by tuple() as + select number, toString(number) from main_table where number % 13 != 1; + create materialized view mv_3 engine = MergeTree order by tuple() as + select number, toString(number) from main_table where number % 13 != 3; + create materialized view mv_4 engine = MergeTree order by tuple() as + select number, toString(number) from main_table where number % 13 != 4; + + + insert into main_table select number from numbers(1000000) + + drop table if exists main_table; + drop table if exists mv_1; + drop table if exists mv_2; + drop table if exists mv_3; + drop table if exists mv_4; + diff --git a/tests/queries/0_stateless/001283_strict_resize_bug.reference b/tests/queries/0_stateless/001283_strict_resize_bug.reference new file mode 100644 index 000000000000..e1fe7af425d7 --- /dev/null +++ b/tests/queries/0_stateless/001283_strict_resize_bug.reference @@ -0,0 +1 @@ +49999995000000 diff --git a/tests/queries/0_stateless/001283_strict_resize_bug.sql b/tests/queries/0_stateless/001283_strict_resize_bug.sql new file mode 100644 index 000000000000..f462f50c61f3 --- /dev/null +++ b/tests/queries/0_stateless/001283_strict_resize_bug.sql @@ -0,0 +1,7 @@ +drop table if exists num_10m; +create table num_10m (number UInt64) engine = MergeTree order by tuple(); +insert into num_10m select * from numbers(10000000); + +select * from (select sum(number) from num_10m union all select sum(number) from num_10m) limit 1 settings max_block_size = 1024; + +drop table if exists num_1m; diff --git a/tests/queries/0_stateless/00386_has_column_in_table.reference b/tests/queries/0_stateless/00386_has_column_in_table.reference index feb61b437517..5b80b696edab 100644 --- a/tests/queries/0_stateless/00386_has_column_in_table.reference +++ b/tests/queries/0_stateless/00386_has_column_in_table.reference @@ -12,3 +12,4 @@ 0 0 0 +0 diff --git a/tests/queries/0_stateless/00386_has_column_in_table.sql b/tests/queries/0_stateless/00386_has_column_in_table.sql index 42643dd73e6d..d543bb42ca7d 100644 --- a/tests/queries/0_stateless/00386_has_column_in_table.sql +++ b/tests/queries/0_stateless/00386_has_column_in_table.sql @@ -18,5 +18,14 @@ SELECT hasColumnInTable(currentDatabase(), 'has_column_in_table', 'nest.not_exis SELECT hasColumnInTable('localhost', currentDatabase(), 'has_column_in_table', 'nest.not_existing'); SELECT hasColumnInTable(currentDatabase(), 'has_column_in_table', 'not_existing'); SELECT hasColumnInTable('localhost', currentDatabase(), 'has_column_in_table', 'not_existing'); +SELECT hasColumnInTable('system', 'one', ''); + +/* bad queries */ +SELECT hasColumnInTable('', '', ''); -- { serverError 60; } +SELECT hasColumnInTable('', 't', 'c'); -- { serverError 81; } +SELECT hasColumnInTable(currentDatabase(), '', 'c'); -- { serverError 60; } +SELECT hasColumnInTable('d', 't', 's'); -- { serverError 81; } +SELECT hasColumnInTable(currentDatabase(), 't', 's'); -- { serverError 60; } + DROP TABLE has_column_in_table; diff --git a/tests/queries/0_stateless/00500_point_in_polygon_nan.reference b/tests/queries/0_stateless/00500_point_in_polygon_nan.reference new file mode 100644 index 000000000000..573541ac9702 --- /dev/null +++ b/tests/queries/0_stateless/00500_point_in_polygon_nan.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/00500_point_in_polygon_nan.sql b/tests/queries/0_stateless/00500_point_in_polygon_nan.sql new file mode 100644 index 000000000000..37ed8dbeded8 --- /dev/null +++ b/tests/queries/0_stateless/00500_point_in_polygon_nan.sql @@ -0,0 +1 @@ +SELECT pointInPolygon((nan, 10.000100135803223), [(39.83154, 21.41527), (2., 1000.0001220703125), (39.90033, 21.37195), (1.000100016593933, 10.000100135803223), (39.83051, 21.42553), (39.82898, 21.41382), (39.83043, 21.41432), (39.83154, 21.41527)]); diff --git a/tests/queries/0_stateless/00539_functions_for_working_with_json.reference b/tests/queries/0_stateless/00539_functions_for_working_with_json.reference index ee7fb68b7c27..c0399f8ab2e0 100644 --- a/tests/queries/0_stateless/00539_functions_for_working_with_json.reference +++ b/tests/queries/0_stateless/00539_functions_for_working_with_json.reference @@ -9,5 +9,7 @@ test"string "test_string" "test\\"string" "test\\"string" + "{" + "[" ["]", "2", "3"] {"nested" : [1,2,3]} diff --git a/tests/queries/0_stateless/00539_functions_for_working_with_json.sql b/tests/queries/0_stateless/00539_functions_for_working_with_json.sql index 8a4d1794293a..514b5f2e5ead 100644 --- a/tests/queries/0_stateless/00539_functions_for_working_with_json.sql +++ b/tests/queries/0_stateless/00539_functions_for_working_with_json.sql @@ -11,5 +11,7 @@ SELECT visitParamExtractRaw('{"myparam":"test_string"}', 'myparam'); SELECT visitParamExtractRaw('{"myparam": "test_string"}', 'myparam'); SELECT visitParamExtractRaw('{"myparam": "test\\"string"}', 'myparam'); SELECT visitParamExtractRaw('{"myparam": "test\\"string", "other":123}', 'myparam'); +SELECT visitParamExtractRaw('{"myparam": "{"}', 'myparam'); +SELECT visitParamExtractRaw('{"myparam": "["}', 'myparam'); SELECT visitParamExtractRaw('{"myparam": ["]", "2", "3"], "other":123}', 'myparam'); SELECT visitParamExtractRaw('{"myparam": {"nested" : [1,2,3]}, "other":123}', 'myparam'); diff --git a/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference new file mode 100644 index 000000000000..2a1151dc692f --- /dev/null +++ b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference @@ -0,0 +1,9 @@ +1 1 1 1 1 1 +0 0 0 0 0 +0 0 1 0 1 +0 2 0 0 1 +0 2 1 0 1 +4 0 0 0 1 +4 0 1 0 1 +4 2 0 0 1 +4 2 1 1 1 diff --git a/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql new file mode 100644 index 000000000000..feee33add1c0 --- /dev/null +++ b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql @@ -0,0 +1,20 @@ + +-- Test that UInt8 type is processed correctly as bool + +SELECT + 1 AND 2, + 2 AND 4, + 1 AND 2 AND 4, + 1 OR 2, + 2 OR 4, + 1 OR 2 OR 4 +; + +SELECT + toUInt8(bitAnd(number, 4)) AS a, + toUInt8(bitAnd(number, 2)) AS b, + toUInt8(bitAnd(number, 1)) AS c, + a AND b AND c AS AND, + a OR b OR c AS OR +FROM numbers(8) +; diff --git a/tests/queries/0_stateless/00735_conditional.reference b/tests/queries/0_stateless/00735_conditional.reference index a82aefaeadd5..e0db75a10446 100644 --- a/tests/queries/0_stateless/00735_conditional.reference +++ b/tests/queries/0_stateless/00735_conditional.reference @@ -8,6 +8,9 @@ value vs value 0 1 1 Int8 UInt32 Int64 0 1 1 Int8 Float32 Float32 0 1 1 Int8 Float64 Float64 +0 1 1 Int8 Decimal(9, 0) Decimal(9, 0) +0 1 1 Int8 Decimal(18, 0) Decimal(18, 0) +0 1 1 Int8 Decimal(38, 0) Decimal(38, 0) 0 1 1 Int16 Int8 Int16 0 1 1 Int16 Int16 Int16 0 1 1 Int16 Int32 Int32 @@ -17,6 +20,9 @@ value vs value 0 1 1 Int16 UInt32 Int64 0 1 1 Int16 Float32 Float32 0 1 1 Int16 Float64 Float64 +0 1 1 Int16 Decimal(9, 0) Decimal(9, 0) +0 1 1 Int16 Decimal(18, 0) Decimal(18, 0) +0 1 1 Int16 Decimal(38, 0) Decimal(38, 0) 0 1 1 Int32 Int8 Int32 0 1 1 Int32 Int16 Int32 0 1 1 Int32 Int32 Int32 @@ -26,6 +32,9 @@ value vs value 0 1 1 Int32 UInt32 Int64 0 1 1 Int32 Float32 Float64 0 1 1 Int32 Float64 Float64 +0 1 1 Int32 Decimal(9, 0) Decimal(9, 0) +0 1 1 Int32 Decimal(18, 0) Decimal(18, 0) +0 1 1 Int32 Decimal(38, 0) Decimal(38, 0) 0 1 1 Int64 Int8 Int64 0 1 1 Int64 Int16 Int64 0 1 1 Int64 Int32 Int64 @@ -33,6 +42,8 @@ value vs value 0 1 1 Int64 UInt8 Int64 0 1 1 Int64 UInt16 Int64 0 1 1 Int64 UInt32 Int64 +0 1 1 Int64 Decimal(18, 0) Decimal(18, 0) +0 1 1 Int64 Decimal(38, 0) Decimal(38, 0) 0 1 1 UInt8 Int8 Int16 0 1 1 UInt8 Int16 Int16 0 1 1 UInt8 Int32 Int32 @@ -43,6 +54,9 @@ value vs value 0 1 1 UInt8 UInt64 UInt64 0 1 1 UInt8 Float32 Float32 0 1 1 UInt8 Float64 Float64 +0 1 1 UInt8 Decimal(9, 0) Decimal(9, 0) +0 1 1 UInt8 Decimal(18, 0) Decimal(18, 0) +0 1 1 UInt8 Decimal(38, 0) Decimal(38, 0) 0 1 1 UInt16 Int8 Int32 0 1 1 UInt16 Int16 Int32 0 1 1 UInt16 Int32 Int32 @@ -53,6 +67,9 @@ value vs value 0 1 1 UInt16 UInt64 UInt64 0 1 1 UInt16 Float32 Float32 0 1 1 UInt16 Float64 Float64 +0 1 1 UInt16 Decimal(9, 0) Decimal(9, 0) +0 1 1 UInt16 Decimal(18, 0) Decimal(18, 0) +0 1 1 UInt16 Decimal(38, 0) Decimal(38, 0) 0 1 1 UInt32 Int8 Int64 0 1 1 UInt32 Int16 Int64 0 1 1 UInt32 Int32 Int64 @@ -63,10 +80,13 @@ value vs value 0 1 1 UInt32 UInt64 UInt64 0 1 1 UInt32 Float32 Float64 0 1 1 UInt32 Float64 Float64 +0 1 1 UInt32 Decimal(18, 0) Decimal(18, 0) +0 1 1 UInt32 Decimal(38, 0) Decimal(38, 0) 0 1 1 UInt64 UInt8 UInt64 0 1 1 UInt64 UInt16 UInt64 0 1 1 UInt64 UInt32 UInt64 0 1 1 UInt64 UInt64 UInt64 +0 1 1 UInt64 Decimal(38, 0) Decimal(38, 0) 0000-00-00 1970-01-02 1970-01-02 Date Date Date 2000-01-01 2000-01-01 00:00:01 2000-01-01 00:00:01 Date DateTime(\'Europe/Moscow\') DateTime 2000-01-01 00:00:00 2000-01-02 2000-01-02 00:00:00 DateTime(\'Europe/Moscow\') Date DateTime diff --git a/tests/queries/0_stateless/00735_conditional.sql b/tests/queries/0_stateless/00735_conditional.sql index ce49c26ca3d1..5e99a0f501f3 100644 --- a/tests/queries/0_stateless/00735_conditional.sql +++ b/tests/queries/0_stateless/00735_conditional.sql @@ -15,9 +15,9 @@ SELECT toInt8(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), SELECT toInt8(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toInt8(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt8(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toInt8(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toInt8(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toInt8(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT toInt8(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toInt8(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toInt8(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt16(0) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt16(0) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -32,9 +32,9 @@ SELECT toInt16(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x) SELECT toInt16(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toInt16(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt16(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toInt16(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toInt16(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toInt16(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT toInt16(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toInt16(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toInt16(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt32(0) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt32(0) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -49,9 +49,9 @@ SELECT toInt32(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x) SELECT toInt32(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toInt32(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt32(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toInt32(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toInt32(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toInt32(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT toInt32(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toInt32(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toInt32(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt64(0) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt64(0) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -66,9 +66,9 @@ SELECT toInt64(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x) SELECT toInt64(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toInt64(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt64(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toInt64(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toInt64(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toInt64(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT toInt64(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT toInt64(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toInt64(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt8(0) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt8(0) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -83,9 +83,9 @@ SELECT toUInt8(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x) SELECT toUInt8(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toUInt8(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt8(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUInt8(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toUInt8(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toUInt8(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT toUInt8(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toUInt8(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toUInt8(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt16(0) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt16(0) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -100,9 +100,9 @@ SELECT toUInt16(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x SELECT toUInt16(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toUInt16(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt16(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUInt16(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toUInt16(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toUInt16(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT toUInt16(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toUInt16(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toUInt16(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt32(0) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt32(0) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -117,9 +117,9 @@ SELECT toUInt32(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x SELECT toUInt32(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toUInt32(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt32(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUInt32(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toUInt32(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toUInt32(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT toUInt32(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT toUInt32(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toUInt32(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt64(0) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt64(0) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } @@ -134,9 +134,9 @@ SELECT toUInt64(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x SELECT toUInt64(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toUInt64(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt64(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUInt64(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toUInt64(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT toUInt64(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT toUInt64(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT toUInt64(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT toUInt64(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toDate(0) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toDate(0) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } @@ -204,9 +204,9 @@ SELECT materialize(toInt8(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, t SELECT materialize(toInt8(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt8(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt8(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT materialize(toInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt16(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt16(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -221,9 +221,9 @@ SELECT materialize(toInt16(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toInt16(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt16(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt16(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT materialize(toInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt32(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt32(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -238,9 +238,9 @@ SELECT materialize(toInt32(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toInt32(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt32(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt32(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT materialize(toInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt64(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt64(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -255,9 +255,9 @@ SELECT materialize(toInt64(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toInt64(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt64(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt64(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT materialize(toInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt8(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt8(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -272,9 +272,9 @@ SELECT materialize(toUInt8(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt8(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt8(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt8(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toUInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toUInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT materialize(toUInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt16(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt16(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -289,9 +289,9 @@ SELECT materialize(toUInt16(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt16(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt16(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt16(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toUInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toUInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT materialize(toUInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt32(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt32(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -306,9 +306,9 @@ SELECT materialize(toUInt32(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt32(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt32(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt32(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toUInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toUInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT materialize(toUInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt64(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt64(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } @@ -323,9 +323,9 @@ SELECT materialize(toUInt64(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt64(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt64(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt64(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toUInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } -SELECT materialize(toUInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 48 } +SELECT materialize(toUInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toDate(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toDate(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } diff --git a/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.reference b/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.reference index 20076c05d5d1..b2e236400b0a 100644 --- a/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.reference +++ b/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.reference @@ -8,3 +8,4 @@ [0,2,4,6] [0,2,4,6] [0,2,4,6,8] +[['a']] diff --git a/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.sql b/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.sql index 9e2ef3f2ff65..76c3d485ccb8 100644 --- a/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.sql +++ b/tests/queries/0_stateless/00752_low_cardinality_lambda_argument.sql @@ -4,3 +4,8 @@ insert into lc_lambda select range(number) from system.numbers limit 10; select arrayFilter(x -> x % 2 == 0, arr) from lc_lambda; drop table if exists lc_lambda; +drop table if exists test_array; +CREATE TABLE test_array(resources_host Array(LowCardinality(String))) ENGINE = MergeTree() ORDER BY (resources_host); +insert into test_array values (['a']); +SELECT arrayMap(i -> [resources_host[i]], arrayEnumerate(resources_host)) FROM test_array; +drop table if exists test_array; diff --git a/tests/queries/0_stateless/00752_low_cardinality_mv_2.reference b/tests/queries/0_stateless/00752_low_cardinality_mv_2.reference index b362d0138a0b..2f1f0fc0b33b 100644 --- a/tests/queries/0_stateless/00752_low_cardinality_mv_2.reference +++ b/tests/queries/0_stateless/00752_low_cardinality_mv_2.reference @@ -1 +1,2 @@ +1 c 2018-10-10 15:45:00 3 10 2018-10-10 15:54:21 1 1 diff --git a/tests/queries/0_stateless/00752_low_cardinality_mv_2.sql b/tests/queries/0_stateless/00752_low_cardinality_mv_2.sql index 354fce8f8149..71950469a470 100644 --- a/tests/queries/0_stateless/00752_low_cardinality_mv_2.sql +++ b/tests/queries/0_stateless/00752_low_cardinality_mv_2.sql @@ -5,6 +5,7 @@ CREATE TABLE radacct ( radacctid UInt64, f3gppchargingid Nullable(String), f3gpp insert into radacct values (1, 'a', 'b', 'c', 'd', 'e', 2, 'a', 'b', 'c', 'd', 'e', 'f', 3, 4, 5, 6, 7, 'a', 'Stop', 'c', 'd', 'e', 'f', 'g', 'h', '2018-10-10 15:54:21', '2018-10-10 15:54:21', 8, 'a', 9, 10, 'a', 'b', '2018-10-10 15:54:21', 'a', 'b', 11, 12, '2018-10-10', 'a', 'b', 'c', 'd', 'e'); +SELECT any(acctstatustype = 'Stop') FROM radacct WHERE (acctstatustype = 'Stop') AND ((acctinputoctets + acctoutputoctets) > 0); create materialized view mv_traffic_by_tadig15min Engine=AggregatingMergeTree partition by tadig order by (ts,tadig) populate as select toStartOfFifteenMinutes(timestamp) ts,toDayOfWeek(timestamp) dow, tadig, sumState(acctinputoctets+acctoutputoctets) traffic_bytes,maxState(timestamp) last_stop, minState(radacctid) min_radacctid,maxState(radacctid) max_radacctid from radacct where acctstatustype='Stop' and acctinputoctets+acctoutputoctets > 0 group by tadig,ts,dow; select tadig, ts, dow, sumMerge(traffic_bytes), maxMerge(last_stop), minMerge(min_radacctid), maxMerge(max_radacctid) from mv_traffic_by_tadig15min group by tadig, ts, dow; diff --git a/tests/queries/0_stateless/00818_inner_join_bug_3567.reference b/tests/queries/0_stateless/00818_inner_join_bug_3567.reference index 7967cf7837ee..4c5e10c19b0d 100644 --- a/tests/queries/0_stateless/00818_inner_join_bug_3567.reference +++ b/tests/queries/0_stateless/00818_inner_join_bug_3567.reference @@ -1,5 +1,5 @@ -a 2018-01-01 00:00:00 0000-00-00 00:00:00 -b 2018-01-01 00:00:00 b b 2018-01-01 00:00:00 -c 2018-01-01 00:00:00 c c 2018-01-01 00:00:00 -b 2018-01-01 00:00:00 b b 2018-01-01 00:00:00 -c 2018-01-01 00:00:00 c c 2018-01-01 00:00:00 +a 2018-01-01 00:00:00 0000-00-00 00:00:00 +b 2018-01-01 00:00:00 b 2018-01-01 00:00:00 +c 2018-01-01 00:00:00 c 2018-01-01 00:00:00 +b 2018-01-01 00:00:00 b 2018-01-01 00:00:00 +c 2018-01-01 00:00:00 c 2018-01-01 00:00:00 diff --git a/tests/queries/0_stateless/00818_inner_join_bug_3567.sql b/tests/queries/0_stateless/00818_inner_join_bug_3567.sql index b8bd6d3384cf..c8a11124dc80 100644 --- a/tests/queries/0_stateless/00818_inner_join_bug_3567.sql +++ b/tests/queries/0_stateless/00818_inner_join_bug_3567.sql @@ -7,8 +7,8 @@ CREATE TABLE using2(c String, a String, d DateTime) ENGINE=MergeTree order by c; INSERT INTO using1 VALUES ('a', '2018-01-01 00:00:00') ('b', '2018-01-01 00:00:00') ('c', '2018-01-01 00:00:00'); INSERT INTO using2 VALUES ('d', 'd', '2018-01-01 00:00:00') ('b', 'b', '2018-01-01 00:00:00') ('c', 'c', '2018-01-01 00:00:00'); -SELECT * FROM using1 t1 ALL LEFT JOIN (SELECT *, c as a, d as b FROM using2) t2 USING (a, b) ORDER BY d; -SELECT * FROM using1 t1 ALL INNER JOIN (SELECT *, c as a, d as b FROM using2) t2 USING (a, b) ORDER BY d; +SELECT * FROM using1 t1 ALL LEFT JOIN (SELECT *, c, d as b FROM using2) t2 USING (a, b) ORDER BY d; +SELECT * FROM using1 t1 ALL INNER JOIN (SELECT *, c, d as b FROM using2) t2 USING (a, b) ORDER BY d; DROP TABLE using1; DROP TABLE using2; diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.reference b/tests/queries/0_stateless/00825_protobuf_format_input.reference index 884cc74c4e56..0c56bc4ebf05 100644 --- a/tests/queries/0_stateless/00825_protobuf_format_input.reference +++ b/tests/queries/0_stateless/00825_protobuf_format_input.reference @@ -8,3 +8,4 @@ a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +7495123456 0 0 2 4 3 9 +ok diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.sh b/tests/queries/0_stateless/00825_protobuf_format_input.sh index d28b70bb002e..1c915bc3f247 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_input.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_input.sh @@ -3,7 +3,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -set -e -o pipefail +set -eo pipefail # Run the client. $CLICKHOUSE_CLIENT --multiquery <<'EOF' @@ -48,5 +48,12 @@ source $CURDIR/00825_protobuf_format_input.insh $CLICKHOUSE_CLIENT --query "SELECT * FROM in_persons_00825 ORDER BY uuid;" $CLICKHOUSE_CLIENT --query "SELECT * FROM in_squares_00825 ORDER BY number;" +# Try to input malformed data. +set +eo pipefail +echo -ne '\xe0\x80\x3f\x0b' \ + | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" 2>&1 \ + | grep -qF "Protobuf messages are corrupted" && echo "ok" || echo "fail" +set -eo pipefail + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_persons_00825;" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_squares_00825;" diff --git a/tests/queries/0_stateless/00945_bloom_filter_index.sql b/tests/queries/0_stateless/00945_bloom_filter_index.sql index 6f93ae89a42c..d509b99229ab 100755 --- a/tests/queries/0_stateless/00945_bloom_filter_index.sql +++ b/tests/queries/0_stateless/00945_bloom_filter_index.sql @@ -43,7 +43,7 @@ SELECT COUNT() FROM bloom_filter_types_test WHERE f32 = 1 SETTINGS max_rows_to_r SELECT COUNT() FROM bloom_filter_types_test WHERE f64 = 1 SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_types_test WHERE date = '1970-01-02' SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_types_test WHERE date_time = toDateTime('1970-01-01 03:00:01', 'Europe/Moscow') SETTINGS max_rows_to_read = 6; -SELECT COUNT() FROM bloom_filter_types_test WHERE str = '1' SETTINGS max_rows_to_read = 6; +SELECT COUNT() FROM bloom_filter_types_test WHERE str = '1' SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_types_test WHERE fixed_string = toFixedString('1', 5) SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_types_test WHERE str IN ( SELECT str FROM bloom_filter_types_test); @@ -122,7 +122,7 @@ SELECT COUNT() FROM bloom_filter_null_types_test WHERE f32 = 1 SETTINGS max_rows SELECT COUNT() FROM bloom_filter_null_types_test WHERE f64 = 1 SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_null_types_test WHERE date = '1970-01-02' SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_null_types_test WHERE date_time = toDateTime('1970-01-01 03:00:01', 'Europe/Moscow') SETTINGS max_rows_to_read = 6; -SELECT COUNT() FROM bloom_filter_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 6; +SELECT COUNT() FROM bloom_filter_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_null_types_test WHERE fixed_string = toFixedString('1', 5) SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_null_types_test WHERE isNull(i8); @@ -150,7 +150,7 @@ CREATE TABLE bloom_filter_lc_null_types_test (order_key UInt64, str LowCardinali INSERT INTO bloom_filter_lc_null_types_test SELECT number AS order_key, toString(number) AS str, toFixedString(toString(number), 5) AS fixed_string FROM system.numbers LIMIT 100; INSERT INTO bloom_filter_lc_null_types_test SELECT 0 AS order_key, NULL AS str, NULL AS fixed_string; -SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 6; +SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE fixed_string = toFixedString('1', 5) SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE isNull(str); diff --git a/tests/queries/0_stateless/00948_values_interpreter_template.reference b/tests/queries/0_stateless/00948_values_interpreter_template.reference index 7559603fed65..e6888e214330 100644 --- a/tests/queries/0_stateless/00948_values_interpreter_template.reference +++ b/tests/queries/0_stateless/00948_values_interpreter_template.reference @@ -1,3 +1,7 @@ +1 Array(UInt64) Tuple(Nullable(UInt64), Nullable(Int64), Nullable(Float64), Nullable(String), Array(Int64)) Array(Tuple(Nullable(UInt64), Array(UInt64))) +2 Array(Int64) Tuple(Nullable(UInt64), Nullable(Int64), Nullable(UInt64), Nullable(String), Array(UInt64)) Array(Tuple(Nullable(UInt64), Array(Nothing))) +3 Array(UInt64) Tuple(Nullable(UInt64), Nullable(Int64), Nullable(Float64), Nullable(String), Array(Int64)) Array(Tuple(Nullable(UInt64), Array(UInt64))) +4 Array(Int64) Tuple(Nullable(UInt64), Nullable(Int64), Nullable(Float64), Nullable(String), Array(Int64)) Array(Tuple(Nullable(UInt64), Array(UInt64))) 1970-01-02 hello 6 -20 nan [3,2,1] 1970-01-03 world 7 2 inf [] 1970-01-04 test 6 -9223372036854775807 3.14 [5,4] diff --git a/tests/queries/0_stateless/00948_values_interpreter_template.sql b/tests/queries/0_stateless/00948_values_interpreter_template.sql index 2394146f454a..2886844ee20b 100644 --- a/tests/queries/0_stateless/00948_values_interpreter_template.sql +++ b/tests/queries/0_stateless/00948_values_interpreter_template.sql @@ -1,13 +1,18 @@ +DROP TABLE IF EXISTS type_names; DROP TABLE IF EXISTS values_template; DROP TABLE IF EXISTS values_template_nullable; DROP TABLE IF EXISTS values_template_fallback; +CREATE TABLE type_names (n UInt8, s1 String, s2 String, s3 String) ENGINE=Memory; CREATE TABLE values_template (d Date, s String, u UInt8, i Int64, f Float64, a Array(UInt8)) ENGINE = Memory; CREATE TABLE values_template_nullable (d Date, s Nullable(String), u Nullable(UInt8), a Array(Nullable(Float32))) ENGINE = Memory; CREATE TABLE values_template_fallback (n UInt8) ENGINE = Memory; SET input_format_values_interpret_expressions = 0; +-- checks type deduction +INSERT INTO type_names VALUES (1, toTypeName([1, 2]), toTypeName((256, -1, 3.14, 'str', [1, -1])), toTypeName([(1, [256]), (256, [1, 2])])), (2, toTypeName([1, -1]), toTypeName((256, -1, 3, 'str', [1, 2])), toTypeName([(256, []), (1, [])])); + --(1, lower(replaceAll(_STR_1, 'o', 'a')), _NUM_1 + _NUM_2 + _NUM_3, round(_NUM_4 / _NUM_5), _NUM_6 * CAST(_STR_7, 'Int8'), _ARR_8); -- _NUM_1: UInt64 -> Int64 -> UInt64 -- _NUM_4: Int64 -> UInt64 @@ -22,13 +27,17 @@ INSERT INTO values_template_fallback VALUES ([1]); -- { clientError 43 } INSERT INTO values_template_fallback VALUES (CAST(1, 'UInt8')), (CAST('2', 'UInt8')); SET input_format_values_accurate_types_of_literals = 0; + +INSERT INTO type_names VALUES (3, toTypeName([1, 2]), toTypeName((256, -1, 3.14, 'str', [1, -1])), toTypeName([(1, [256]), (256, [1, 2])])), (4, toTypeName([1, -1]), toTypeName((256, -1, 3, 'str', [1, 2])), toTypeName([(256, []), (1, [])])); SET input_format_values_interpret_expressions = 1; INSERT INTO values_template_fallback VALUES (1 + 2), (3 + +04), (5 + 6); INSERT INTO values_template_fallback VALUES (+020), (+030), (+040); +SELECT * FROM type_names ORDER BY n; SELECT * FROM values_template ORDER BY d; SELECT * FROM values_template_nullable ORDER BY d; SELECT * FROM values_template_fallback ORDER BY n; +DROP TABLE type_names; DROP TABLE values_template; DROP TABLE values_template_nullable; DROP TABLE values_template_fallback; diff --git a/tests/queries/0_stateless/00972_geohashesInBox.reference b/tests/queries/0_stateless/00972_geohashesInBox.reference index e6844fa8394d..92dab3cb04eb 100644 --- a/tests/queries/0_stateless/00972_geohashesInBox.reference +++ b/tests/queries/0_stateless/00972_geohashesInBox.reference @@ -37,4 +37,6 @@ zooming ['s7w1z0gs3y0z','s7w1z0gs3y1p','s7w1z0gs3y1r','s7w1z0gs3y1x','s7w1z0gs3y2b','s7w1z0gs3y2c','s7w1z0gs3y2f','s7w1z0gs3y2g','s7w1z0gs3y2u','s7w1z0gs3y2v','s7w1z0gs3y30','s7w1z0gs3y31','s7w1z0gs3y32','s7w1z0gs3y33','s7w1z0gs3y34','s7w1z0gs3y35','s7w1z0gs3y36','s7w1z0gs3y37','s7w1z0gs3y38','s7w1z0gs3y39','s7w1z0gs3y3d','s7w1z0gs3y3e','s7w1z0gs3y3h','s7w1z0gs3y3j','s7w1z0gs3y3k','s7w1z0gs3y3m','s7w1z0gs3y3s','s7w1z0gs3y3t'] ['s7w1z0gs3y0z','s7w1z0gs3y1p','s7w1z0gs3y1r','s7w1z0gs3y1x','s7w1z0gs3y2b','s7w1z0gs3y2c','s7w1z0gs3y2f','s7w1z0gs3y2g','s7w1z0gs3y2u','s7w1z0gs3y2v','s7w1z0gs3y30','s7w1z0gs3y31','s7w1z0gs3y32','s7w1z0gs3y33','s7w1z0gs3y34','s7w1z0gs3y35','s7w1z0gs3y36','s7w1z0gs3y37','s7w1z0gs3y38','s7w1z0gs3y39','s7w1z0gs3y3d','s7w1z0gs3y3e','s7w1z0gs3y3h','s7w1z0gs3y3j','s7w1z0gs3y3k','s7w1z0gs3y3m','s7w1z0gs3y3s','s7w1z0gs3y3t'] ['s7w1z0gs3y0z','s7w1z0gs3y1p','s7w1z0gs3y1r','s7w1z0gs3y1x','s7w1z0gs3y2b','s7w1z0gs3y2c','s7w1z0gs3y2f','s7w1z0gs3y2g','s7w1z0gs3y2u','s7w1z0gs3y2v','s7w1z0gs3y30','s7w1z0gs3y31','s7w1z0gs3y32','s7w1z0gs3y33','s7w1z0gs3y34','s7w1z0gs3y35','s7w1z0gs3y36','s7w1z0gs3y37','s7w1z0gs3y38','s7w1z0gs3y39','s7w1z0gs3y3d','s7w1z0gs3y3e','s7w1z0gs3y3h','s7w1z0gs3y3j','s7w1z0gs3y3k','s7w1z0gs3y3m','s7w1z0gs3y3s','s7w1z0gs3y3t'] +input values are clamped to -90..90, -180..180 range +32768 errors diff --git a/tests/queries/0_stateless/00972_geohashesInBox.sql b/tests/queries/0_stateless/00972_geohashesInBox.sql index f382bf234ac8..d52a03b055ee 100644 --- a/tests/queries/0_stateless/00972_geohashesInBox.sql +++ b/tests/queries/0_stateless/00972_geohashesInBox.sql @@ -5,41 +5,46 @@ -- except for the cases when JS-version produces result outside of given region, -- typically at wrap points: poles, 0-latitude and 0-longitude. -select 'center'; +SELECT 'center'; SELECT arraySort(geohashesInBox(-1.0, -1.0, 1.0, 1.0, 3)); SELECT arraySort(geohashesInBox(-0.1, -0.1, 0.1, 0.1, 5)); SELECT arraySort(geohashesInBox(-0.01, -0.01, 0.01, 0.01, 5)); -select 'north pole'; +SELECT 'north pole'; SELECT arraySort(geohashesInBox(-180.0, 89.0, -179.0, 90.0, 3)); SELECT arraySort(geohashesInBox(-1.0, 89.0, 0.0, 90.0, 3)); SELECT arraySort(geohashesInBox(0.0, 89.0, 1.0, 90.0, 3)); SELECT arraySort(geohashesInBox(179.0, 89.0, 180.0, 90.0, 3)); -select 'south pole'; +SELECT 'south pole'; SELECT arraySort(geohashesInBox(-180.0, -90.0, -179.0, -89.0, 3)); SELECT arraySort(geohashesInBox(-1.0, -90.0, 0.0, -89.0, 3)); SELECT arraySort(geohashesInBox(0.0, -90.0, 1.0, -89.0, 3)); SELECT arraySort(geohashesInBox(179.0, -90.0, 180.0, -89.0, 3)); -select 'wrap point around equator'; +SELECT 'wrap point around equator'; SELECT arraySort(geohashesInBox(179.0, -1.0, 180.0, 0.0, 3)); SELECT arraySort(geohashesInBox(179.0, 0.0, 180.0, 1.0, 3)); SELECT arraySort(geohashesInBox(-180.0, -1.0, -179.0, 0.0, 3)); SELECT arraySort(geohashesInBox(-180.0, 0.0, -179.0, 1.0, 3)); -select 'arbitrary values in all 4 quarters'; +SELECT 'arbitrary values in all 4 quarters'; SELECT arraySort(geohashesInBox(98.36, 7.88, 98.37, 7.89, 6)); SELECT arraySort(geohashesInBox(53.8, 27.6, 53.9, 27.7, 5)); SELECT arraySort(geohashesInBox(-49.26, -25.38, -49.25, -25.37, 6)); SELECT arraySort(geohashesInBox(23.11, -82.37, 23.12, -82.36, 6)); -select 'small range always produces array of length 1'; -SELECT lon/5 - 180 as lon1, lat/5 - 90 as lat1, lon1 as lon2, lat1 as lat2, geohashesInBox(lon1, lat1, lon2, lat2, 1) as g FROM (SELECT arrayJoin(range(360*5)) as lon, arrayJoin(range(180*5)) as lat) WHERE length(g) != 1; -SELECT lon/5 - 40 as lon1, lat/5 - 20 as lat1, lon1 as lon2, lat1 as lat2, geohashesInBox(lon1, lat1, lon2, lat2, 12) as g FROM (SELECT arrayJoin(range(80*5)) as lon, arrayJoin(range(10*5)) as lat) WHERE length(g) != 1; -SELECT lon/5 - 40 as lon1, lat/5 - 20 as lat1, lon1 + 0.0000000001 as lon2, lat1 + 0.0000000001 as lat2, geohashesInBox(lon1, lat1, lon2, lat2, 1) as g FROM (SELECT arrayJoin(range(80*5)) as lon, arrayJoin(range(10*5)) as lat) WHERE length(g) != 1; +SELECT 'small range always produces array of length 1'; +SELECT lon/5 - 180 AS lon1, lat/5 - 90 AS lat1, lon1 AS lon2, lat1 AS lat2, geohashesInBox(lon1, lat1, lon2, lat2, 1) AS g +FROM (SELECT arrayJoin(range(360*5)) AS lon, arrayJoin(range(180*5)) AS lat) WHERE length(g) != 1; -select 'zooming'; +SELECT lon/5 - 40 AS lon1, lat/5 - 20 AS lat1, lon1 AS lon2, lat1 AS lat2, geohashesInBox(lon1, lat1, lon2, lat2, 12) AS g +FROM (SELECT arrayJoin(range(80*5)) AS lon, arrayJoin(range(10*5)) AS lat) WHERE length(g) != 1; + +SELECT lon/5 - 40 AS lon1, lat/5 - 20 AS lat1, lon1 + 0.0000000001 AS lon2, lat1 + 0.0000000001 AS lat2, geohashesInBox(lon1, lat1, lon2, lat2, 1) AS g +FROM (SELECT arrayJoin(range(80*5)) AS lon, arrayJoin(range(10*5)) AS lat) WHERE length(g) != 1; + +SELECT 'zooming'; SELECT arraySort(geohashesInBox(20.0, 20.0, 21.0, 21.0, 2)); SELECT arraySort(geohashesInBox(20.0, 20.0, 21.0, 21.0, 3)); SELECT arraySort(geohashesInBox(20.0, 20.0, 21.0, 21.0, 4)); @@ -56,8 +61,12 @@ SELECT arraySort(geohashesInBox(20.0, 20.0, 20.000001, 20.000001, 12)); SELECT arraySort(geohashesInBox(20.0, 20.0, 20.000001, 20.000001, 13)); SELECT arraySort(geohashesInBox(20.0, 20.0, 20.000001, 20.000001, 14)); -select 'errors'; +SELECT 'input values are clamped to -90..90, -180..180 range'; +SELECT length(geohashesInBox(-inf, -inf, inf, inf, 3)); + +SELECT 'errors'; SELECT geohashesInBox(); -- { serverError 42 } -- not enough arguments SELECT geohashesInBox(1, 2, 3, 4, 5); -- { serverError 43 } -- wrong types of arguments SELECT geohashesInBox(toFloat32(1.0), 2.0, 3.0, 4.0, 5); -- { serverError 43 } -- all lats and longs should be of the same type SELECT geohashesInBox(24.48, 40.56, 24.785, 40.81, 12); -- { serverError 128 } -- to many elements in array + diff --git a/tests/queries/0_stateless/00975_values_list.reference b/tests/queries/0_stateless/00975_values_list.reference index eee9e0a0ca56..f8ada08d1302 100644 --- a/tests/queries/0_stateless/00975_values_list.reference +++ b/tests/queries/0_stateless/00975_values_list.reference @@ -11,3 +11,4 @@ abracadabra 23 23 23 24 24 24 1.6660 a b +\N diff --git a/tests/queries/0_stateless/00975_values_list.sql b/tests/queries/0_stateless/00975_values_list.sql index ad30cec21e98..40c868989669 100644 --- a/tests/queries/0_stateless/00975_values_list.sql +++ b/tests/queries/0_stateless/00975_values_list.sql @@ -11,4 +11,9 @@ SELECT * FROM VALUES('s String', ('abra'), ('cadabra'), ('abracadabra')); SELECT * FROM VALUES('n UInt64, s String, ss String', (1 + 22, '23', toString(23)), (toUInt64('24'), '24', concat('2', '4'))); SELECT * FROM VALUES('a Decimal(4, 4), b String, c String', (divide(toDecimal32(5, 3), 3), 'a', 'b')); + +SELECT * FROM VALUES('x Float64', toUInt64(-1)); -- { serverError 69; } +SELECT * FROM VALUES('x Float64', NULL); -- { serverError 53; } +SELECT * FROM VALUES('x Nullable(Float64)', NULL); + DROP TABLE values_list; diff --git a/tests/queries/0_stateless/01015_insert_values_parametrized.reference b/tests/queries/0_stateless/01015_insert_values_parametrized.reference index c887e5feb5f0..fc04eb5bc225 100644 --- a/tests/queries/0_stateless/01015_insert_values_parametrized.reference +++ b/tests/queries/0_stateless/01015_insert_values_parametrized.reference @@ -3,3 +3,4 @@ 2 testparam [0.3] 3 paramparam [] 4 evaluateparam [0.2] +5 param [0.2,0.3] diff --git a/tests/queries/0_stateless/01015_insert_values_parametrized.sh b/tests/queries/0_stateless/01015_insert_values_parametrized.sh index 8edda6629b05..ac3833016632 100755 --- a/tests/queries/0_stateless/01015_insert_values_parametrized.sh +++ b/tests/queries/0_stateless/01015_insert_values_parametrized.sh @@ -15,6 +15,9 @@ $CLICKHOUSE_CLIENT --input_format_values_deduce_templates_of_expressions=1 --inp $CLICKHOUSE_CLIENT --input_format_values_deduce_templates_of_expressions=0 --input_format_values_interpret_expressions=1 --param_p_n="-1" --param_p_s="param" --param_p_a="[0.2,0.3]" --query="INSERT INTO insert_values_parametrized VALUES \ (5 + {p_n:Int8}, lower(concat('Evaluate', {p_s:String})), arrayIntersect([0, 0.2, 0.6], {p_a:Array(Nullable(Float32))}))" +$CLICKHOUSE_CLIENT --param_p_n="5" --param_p_s="param" --param_p_a="[0.2,0.3]" --query="INSERT INTO insert_values_parametrized VALUES \ +({p_n:Int8}, {p_s:String}, {p_a:Array(Nullable(Float32))})" + $CLICKHOUSE_CLIENT --query="SELECT * FROM insert_values_parametrized ORDER BY n"; $CLICKHOUSE_CLIENT --query="DROP TABLE insert_values_parametrized"; diff --git a/tests/queries/0_stateless/01055_prewhere_bugs.reference b/tests/queries/0_stateless/01055_prewhere_bugs.reference index cd0e6a397a15..19de4641cd61 100644 --- a/tests/queries/0_stateless/01055_prewhere_bugs.reference +++ b/tests/queries/0_stateless/01055_prewhere_bugs.reference @@ -1,2 +1,3 @@ 43 1 + Nullable(UInt8) Nullable(Int32) diff --git a/tests/queries/0_stateless/01055_prewhere_bugs.sql b/tests/queries/0_stateless/01055_prewhere_bugs.sql index d9a0256ce52a..3929356ce55c 100644 --- a/tests/queries/0_stateless/01055_prewhere_bugs.sql +++ b/tests/queries/0_stateless/01055_prewhere_bugs.sql @@ -12,6 +12,7 @@ CREATE TABLE test_prewhere_column_type (`a` LowCardinality(String), `x` Nullable INSERT INTO test_prewhere_column_type VALUES ('', 2); SELECT a, y FROM test_prewhere_column_type prewhere (x = 2) AS y; +SELECT a, toTypeName(x = 2), toTypeName(x) FROM test_prewhere_column_type where (x = 2) AS y; DROP TABLE test_prewhere_default_column; DROP TABLE test_prewhere_column_type; diff --git a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference index 019e95cb359b..bd1322029796 100644 --- a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference +++ b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference @@ -4,7 +4,7 @@ a 2 1 0 a 3 1 0 b 13 2 0 b 15 2 0 -SELECT \n co, \n co2, \n co3, \n num\nFROM \n(\n SELECT \n co, \n co2, \n co3, \n count() AS num\n FROM \n (\n SELECT \n 1 AS co, \n 2 AS co2, \n 3 AS co3\n )\n GROUP BY \n co, \n co2, \n co3\n WITH CUBE\n HAVING (co != 0) AND (co2 != 2)\n)\nWHERE (co != 0) AND (co2 != 2) +SELECT \n co, \n co2, \n co3, \n num\nFROM \n(\n SELECT \n co, \n co2, \n co3, \n count() AS num\n FROM \n (\n SELECT \n 1 AS co, \n 2 AS co2, \n 3 AS co3\n )\n GROUP BY \n co, \n co2, \n co3\n WITH CUBE\n HAVING (co2 != 2) AND (co != 0)\n)\nWHERE (co != 0) AND (co2 != 2) 1 0 3 1 1 0 0 1 SELECT alias AS name\nFROM \n(\n SELECT name AS alias\n FROM system.settings\n WHERE alias = \'enable_optimize_predicate_expression\'\n)\nANY INNER JOIN \n(\n SELECT name\n FROM system.settings\n) USING (name)\nWHERE name = \'enable_optimize_predicate_expression\' @@ -26,3 +26,4 @@ SELECT dummy\nFROM \n(\n SELECT dummy\n FROM system.one\n WHERE arrayMa 0 SELECT \n id, \n value, \n value_1\nFROM \n(\n SELECT \n 1 AS id, \n 2 AS value\n)\nALL INNER JOIN \n(\n SELECT \n 1 AS id, \n 3 AS value_1\n) USING (id)\nWHERE arrayMap(x -> ((x + value) + value_1), [1]) = [6] 1 2 3 +SELECT dummy\nFROM system.one\nWHERE (dummy > 0) AND (dummy < 0) diff --git a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql index db6c78e3cc6d..18552a6591d4 100644 --- a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql +++ b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql @@ -74,3 +74,13 @@ SELECT * FROM (SELECT * FROM system.one) WHERE arrayMap(x -> x + 1, [dummy]) = [ ANALYZE SELECT * FROM (SELECT 1 AS id, 2 AS value) INNER JOIN (SELECT 1 AS id, 3 AS value_1) USING id WHERE arrayMap(x -> x + value + value_1, [1]) = [6]; SELECT * FROM (SELECT 1 AS id, 2 AS value) INNER JOIN (SELECT 1 AS id, 3 AS value_1) USING id WHERE arrayMap(x -> x + value + value_1, [1]) = [6]; + +-- check order is preserved +ANALYZE SELECT * FROM system.one HAVING dummy > 0 AND dummy < 0; + +-- from #10613 +SELECT name, count() AS cnt +FROM remote('127.{1,2}', system.settings) +GROUP BY name +HAVING (max(value) > '9') AND (min(changed) = 0) +FORMAT Null; diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index b6522b128954..4ae0ed851376 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -105,11 +105,14 @@ sleep 1 counter=0 while [[ $($CLICKHOUSE_CLIENT --query "select * from system.mutations where table like 'concurrent_mutate_mt_%' and is_done=0" 2>&1) ]]; do - if [ "$counter" -gt 20 ] + if [ "$counter" -gt 120 ] then break fi sleep 1 + for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_mutate_mt_$i" 2> /dev/null + done counter=$(($counter + 1)) done diff --git a/tests/queries/0_stateless/01086_odbc_roundtrip.sh b/tests/queries/0_stateless/01086_odbc_roundtrip.sh new file mode 100755 index 000000000000..71ea517f4dd4 --- /dev/null +++ b/tests/queries/0_stateless/01086_odbc_roundtrip.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CUR_DIR/../shell_config.sh + + +for i in $(seq 1 10); do + ${CLICKHOUSE_CLIENT} -q "select count() > 1 as ok from (select * from odbc('DSN={ClickHouse DSN (ANSI)}','system','tables'))" 2>/dev/null && break + sleep 0.1 +done + +${CLICKHOUSE_CLIENT} --query "select count() > 1 as ok from (select * from odbc('DSN={ClickHouse DSN (Unicode)}','system','tables'))" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS test_01086" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE test_01086" + + +${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_01086.t (x UInt8, y Float32, z String) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test_01086.t VALUES (1,0.1,'a я'),(2,0.2,'b ą'),(3,0.3,'c d')" + +${CLICKHOUSE_CLIENT} --query "SELECT * FROM odbc('DSN={ClickHouse DSN (ANSI)}','test_01086','t') ORDER BY x" + +${CLICKHOUSE_CLIENT} --query "SELECT * FROM odbc('DSN={ClickHouse DSN (Unicode)}','test_01086','t') ORDER BY x" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE test_01086;" diff --git a/tests/queries/0_stateless/01086_odbc_roundtrip.sql b/tests/queries/0_stateless/01086_odbc_roundtrip.sql deleted file mode 100644 index 2c31711d8955..000000000000 --- a/tests/queries/0_stateless/01086_odbc_roundtrip.sql +++ /dev/null @@ -1,14 +0,0 @@ -select count() > 1 as ok from (select * from odbc('DSN={ClickHouse DSN (ANSI)}','system','tables')); -select count() > 1 as ok from (select * from odbc('DSN={ClickHouse DSN (Unicode)}','system','tables')); - -DROP DATABASE IF EXISTS test_01086; -CREATE DATABASE test_01086; -USE test_01086; - -CREATE TABLE t (x UInt8, y Float32, z String) ENGINE = Memory; -INSERT INTO t VALUES (1,0.1,'a я'),(2,0.2,'b ą'),(3,0.3,'c d'); - -select * from odbc('DSN={ClickHouse DSN (ANSI)}','test_01086','t') ORDER BY x; -select * from odbc('DSN={ClickHouse DSN (Unicode)}','test_01086','t') ORDER BY x; - -DROP DATABASE test_01086; diff --git a/tests/queries/0_stateless/01091_num_threads.sql b/tests/queries/0_stateless/01091_num_threads.sql index 876a2d15d1a1..7b9c284f7590 100644 --- a/tests/queries/0_stateless/01091_num_threads.sql +++ b/tests/queries/0_stateless/01091_num_threads.sql @@ -1,14 +1,14 @@ set log_queries=1; set log_query_threads=1; -SELECT 1; +WITH 01091 AS id SELECT 1; SYSTEM FLUSH LOGS; WITH ( SELECT query_id FROM system.query_log - WHERE (query = 'SELECT 1') AND (event_date >= (today() - 1)) + WHERE (query LIKE 'WITH 1091%') AND (event_date >= (today() - 1)) ORDER BY event_time DESC LIMIT 1 ) AS id @@ -16,14 +16,14 @@ SELECT uniqExact(thread_id) FROM system.query_thread_log WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master_thread_id); -select sum(number) from numbers(1000000); +with 01091 as id select sum(number) from numbers(1000000); SYSTEM FLUSH LOGS; WITH ( SELECT query_id FROM system.query_log - WHERE (query = 'SELECT sum(number) FROM numbers(1000000)') AND (event_date >= (today() - 1)) + WHERE (query LIKE 'WITH 1091% AS id SELECT sum(number) FROM numbers(1000000)%') AND (event_date >= (today() - 1)) ORDER BY event_time DESC LIMIT 1 ) AS id @@ -31,14 +31,14 @@ SELECT uniqExact(thread_id) FROM system.query_thread_log WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master_thread_id); -select sum(number) from numbers_mt(1000000); +with 01091 as id select sum(number) from numbers_mt(1000000); SYSTEM FLUSH LOGS; WITH ( SELECT query_id FROM system.query_log - WHERE (query = 'SELECT sum(number) FROM numbers_mt(1000000)') AND (event_date >= (today() - 1)) + WHERE (query LIKE 'WITH 1091% AS id SELECT sum(number) FROM numbers_mt(1000000)%') AND (event_date >= (today() - 1)) ORDER BY event_time DESC LIMIT 1 ) AS id diff --git a/tests/queries/0_stateless/01104_distributed_numbers_test.sql b/tests/queries/0_stateless/01104_distributed_numbers_test.sql index b301c0ac00f6..7f56a4e08fd3 100644 --- a/tests/queries/0_stateless/01104_distributed_numbers_test.sql +++ b/tests/queries/0_stateless/01104_distributed_numbers_test.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS d_numbers; -CREATE TABLE d_numbers (number UInt32) ENGINE = Distributed(test_cluster_two_shards_localhost, system, numbers, rand()); +CREATE TABLE d_numbers (number UInt32) ENGINE = Distributed(test_cluster_two_shards, system, numbers, rand()); SET experimental_use_processors = 1; diff --git a/tests/queries/0_stateless/01104_distributed_one_test.sql b/tests/queries/0_stateless/01104_distributed_one_test.sql index 92b4a83ebf36..5d427bdc4e15 100644 --- a/tests/queries/0_stateless/01104_distributed_one_test.sql +++ b/tests/queries/0_stateless/01104_distributed_one_test.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS d_one; -CREATE TABLE d_one (dummy UInt8) ENGINE = Distributed(test_cluster_two_shards_localhost, system, one, rand()); +CREATE TABLE d_one (dummy UInt8) ENGINE = Distributed(test_cluster_two_shards, system, one, rand()); SELECT 'local_0', toUInt8(1) AS dummy FROM system.one AS o WHERE o.dummy = 0; SELECT 'local_1', toUInt8(1) AS dummy FROM system.one AS o WHERE o.dummy = 1; diff --git a/tests/queries/0_stateless/01116_cross_count_asterisks.sql b/tests/queries/0_stateless/01116_cross_count_asterisks.sql deleted file mode 100644 index 1fb8b0b0e664..000000000000 --- a/tests/queries/0_stateless/01116_cross_count_asterisks.sql +++ /dev/null @@ -1,29 +0,0 @@ -SET multiple_joins_rewriter_version = 2; - -SELECT count(*) -FROM numbers(2) AS n1, numbers(3) AS n2, numbers(4) AS n3 -WHERE (n1.number = n2.number) AND (n2.number = n3.number); - -SELECT count(*) c FROM ( - SELECT count(*), count(*) as c - FROM numbers(2) AS n1, numbers(3) AS n2, numbers(4) AS n3 - WHERE (n1.number = n2.number) AND (n2.number = n3.number) - AND (SELECT count(*) FROM numbers(1)) = 1 -) -WHERE (SELECT count(*) FROM numbers(2)) = 2 -HAVING c IN(SELECT count(*) c FROM numbers(1)); - -SET multiple_joins_rewriter_version = 1; - -SELECT count(*) -FROM numbers(2) AS n1, numbers(3) AS n2, numbers(4) AS n3 -WHERE (n1.number = n2.number) AND (n2.number = n3.number); - -SELECT count(*) c FROM ( - SELECT count(*), count(*) as c - FROM numbers(2) AS n1, numbers(3) AS n2, numbers(4) AS n3 - WHERE (n1.number = n2.number) AND (n2.number = n3.number) - AND (SELECT count(*) FROM numbers(1)) = 1 -) -WHERE (SELECT count(*) FROM numbers(2)) = 2 -HAVING c IN(SELECT count(*) c FROM numbers(1)); diff --git a/tests/queries/0_stateless/01137_order_by_func.reference b/tests/queries/0_stateless/01137_order_by_func.reference new file mode 100644 index 000000000000..66d4d94db721 --- /dev/null +++ b/tests/queries/0_stateless/01137_order_by_func.reference @@ -0,0 +1,8 @@ +2020-05-05 01:00:00 0 +2020-05-05 01:00:00 1 +2020-05-05 01:00:00 2 +2020-05-05 01:00:00 3 +2020-05-05 01:00:00 4 +111 9999999 9999999 +111 9999998 9999998 +111 9999997 9999997 diff --git a/tests/queries/0_stateless/01137_order_by_func.sql b/tests/queries/0_stateless/01137_order_by_func.sql new file mode 100644 index 000000000000..682b2d391cee --- /dev/null +++ b/tests/queries/0_stateless/01137_order_by_func.sql @@ -0,0 +1,25 @@ +DROP TABLE IF EXISTS pk_func; +CREATE TABLE pk_func(d DateTime, ui UInt32) ENGINE = MergeTree ORDER BY toDate(d); + +INSERT INTO pk_func SELECT '2020-05-05 01:00:00', number FROM numbers(1000000); +INSERT INTO pk_func SELECT '2020-05-06 01:00:00', number FROM numbers(1000000); +INSERT INTO pk_func SELECT '2020-05-07 01:00:00', number FROM numbers(1000000); + +SELECT * FROM pk_func ORDER BY toDate(d), ui LIMIT 5; + +DROP TABLE pk_func; + +DROP TABLE IF EXISTS nORX; +CREATE TABLE nORX (`A` Int64, `B` Int64, `V` Int64) ENGINE = MergeTree ORDER BY (A, negate(B)); +INSERT INTO nORX SELECT 111, number, number FROM numbers(10000000); + +SELECT * +FROM nORX +WHERE B >= 1000 +ORDER BY + A ASC, + -B ASC +LIMIT 3 +SETTINGS max_threads = 1; + +DROP TABLE nORX; diff --git a/tests/queries/0_stateless/01137_order_by_func_final.reference b/tests/queries/0_stateless/01137_order_by_func_final.reference new file mode 100644 index 000000000000..c97316543dad --- /dev/null +++ b/tests/queries/0_stateless/01137_order_by_func_final.reference @@ -0,0 +1,3 @@ +2020-05-05 704982704 +2020-05-06 704982704 +2020-05-07 704982704 diff --git a/tests/queries/0_stateless/01137_order_by_func_final.sql b/tests/queries/0_stateless/01137_order_by_func_final.sql new file mode 100644 index 000000000000..32a9085e9ee5 --- /dev/null +++ b/tests/queries/0_stateless/01137_order_by_func_final.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS pk_func; +CREATE TABLE pk_func(d DateTime, ui UInt32) ENGINE = SummingMergeTree ORDER BY toDate(d); + +INSERT INTO pk_func SELECT '2020-05-05 01:00:00', number FROM numbers(100000); +INSERT INTO pk_func SELECT '2020-05-06 01:00:00', number FROM numbers(100000); +INSERT INTO pk_func SELECT '2020-05-07 01:00:00', number FROM numbers(100000); + +SELECT toDate(d), ui FROM pk_func FINAL; + +DROP TABLE pk_func; diff --git a/tests/queries/0_stateless/01137_sample_final.reference b/tests/queries/0_stateless/01137_sample_final.reference new file mode 100644 index 000000000000..91dfe0101612 --- /dev/null +++ b/tests/queries/0_stateless/01137_sample_final.reference @@ -0,0 +1,10 @@ +8 8 +9 9 +10 10 +14 14 +15 15 +9140302661501632497 +9199082625845137542 +8769270213041934496 +4926958392161000708 +89922286675368115 diff --git a/tests/queries/0_stateless/01137_sample_final.sql b/tests/queries/0_stateless/01137_sample_final.sql new file mode 100644 index 000000000000..99fac5147678 --- /dev/null +++ b/tests/queries/0_stateless/01137_sample_final.sql @@ -0,0 +1,13 @@ +drop table if exists tab; + +create table tab (x UInt64, v UInt64) engine = ReplacingMergeTree(v) order by (x, sipHash64(x)) sample by sipHash64(x); +insert into tab select number, number from numbers(1000); +select * from tab final sample 1/2 order by x limit 5; + +drop table tab; + +create table tab (x UInt64, v UInt64) engine = ReplacingMergeTree(v) order by (x, sipHash64(x)) sample by sipHash64(x); +insert into tab select number, number from numbers(1000); +select sipHash64(x) from tab sample 1/2 order by x, sipHash64(x) limit 5; + +drop table tab; diff --git a/tests/queries/0_stateless/01140_select_from_storage_join_fix.reference b/tests/queries/0_stateless/01140_select_from_storage_join_fix.reference new file mode 100644 index 000000000000..101a270ad39b --- /dev/null +++ b/tests/queries/0_stateless/01140_select_from_storage_join_fix.reference @@ -0,0 +1,8 @@ +1 s 1 String String +2 s 2 String String +3 s 3 Nullable(String) String +4 s 4 String Nullable(String) +1 s 1 String String +2 s 2 String String +3 s 3 Nullable(String) String +4 s 4 String Nullable(String) diff --git a/tests/queries/0_stateless/01140_select_from_storage_join_fix.sql b/tests/queries/0_stateless/01140_select_from_storage_join_fix.sql new file mode 100644 index 000000000000..4e64c90f56de --- /dev/null +++ b/tests/queries/0_stateless/01140_select_from_storage_join_fix.sql @@ -0,0 +1,42 @@ +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS t4; + +CREATE TABLE t1 (id String, name String, value UInt32) +ENGINE = Join(ANY, LEFT, id) +SETTINGS join_use_nulls = 1; + +CREATE TABLE t2 (id String, name String, value UInt32) +ENGINE = Join(ANY, LEFT, id) +SETTINGS join_use_nulls = 0; + +CREATE TABLE t3 (id Nullable(String), name String, value UInt32) +ENGINE = Join(ANY, LEFT, id) +SETTINGS join_use_nulls = 1; + +CREATE TABLE t4 (id String, name Nullable(String), value UInt32) +ENGINE = Join(ANY, LEFT, id) +SETTINGS join_use_nulls = 0; + +insert into t1 values('1', 's', 1); +insert into t2 values('2', 's', 2); +insert into t3 values('3', 's', 3); +insert into t4 values('4', 's', 4); + +select *, toTypeName(id), toTypeName(name) from t1; +select *, toTypeName(id), toTypeName(name) from t2; +select *, toTypeName(id), toTypeName(name) from t3; +select *, toTypeName(id), toTypeName(name) from t4; + +SET join_use_nulls = 1; + +select *, toTypeName(id), toTypeName(name) from t1; +select *, toTypeName(id), toTypeName(name) from t2; +select *, toTypeName(id), toTypeName(name) from t3; +select *, toTypeName(id), toTypeName(name) from t4; + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; diff --git a/tests/queries/0_stateless/01141_join_get_negative.reference b/tests/queries/0_stateless/01141_join_get_negative.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01141_join_get_negative.sql b/tests/queries/0_stateless/01141_join_get_negative.sql new file mode 100644 index 000000000000..e165d34e4604 --- /dev/null +++ b/tests/queries/0_stateless/01141_join_get_negative.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; + +CREATE TABLE t1 (`s` String, `x` Array(UInt8), `k` UInt64) ENGINE = Join(ANY, LEFT, k); +CREATE TABLE t2 (`s` String, `x` Array(UInt8), `k` UInt64) ENGINE = Join(ANY, INNER, k); + +SELECT joinGet('t1', '', number) FROM numbers(2); -- { serverError 16 } +SELECT joinGet('t2', 's', number) FROM numbers(2); -- { serverError 264 } + +DROP TABLE t1; +DROP TABLE t2; diff --git a/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.reference b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.reference new file mode 100644 index 000000000000..d1b29b46df69 --- /dev/null +++ b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.reference @@ -0,0 +1,29 @@ +1 l \N Nullable(String) +2 \N Nullable(String) +1 l \N Nullable(String) +2 \N Nullable(String) +- +1 l \N Nullable(String) +0 \N Nullable(String) +0 \N Nullable(String) +1 l \N Nullable(String) +- +1 l \N Nullable(String) +0 \N Nullable(String) +0 \N Nullable(String) +1 l \N Nullable(String) +- +1 l \N Nullable(String) +2 \N Nullable(String) +1 l \N Nullable(String) +2 \N Nullable(String) +- +1 l \N Nullable(String) +\N \N Nullable(String) +1 l \N Nullable(String) +\N \N Nullable(String) +- +1 l \N Nullable(String) +\N \N Nullable(String) +1 l \N Nullable(String) +\N \N Nullable(String) diff --git a/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql new file mode 100644 index 000000000000..edaf2870e898 --- /dev/null +++ b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql @@ -0,0 +1,50 @@ +DROP TABLE IF EXISTS t; +DROP TABLE IF EXISTS nr; + +CREATE TABLE t (`x` UInt32, `lc` LowCardinality(String)) ENGINE = Memory; +CREATE TABLE nr (`x` Nullable(UInt32), `lc` Nullable(String)) ENGINE = Memory; + +INSERT INTO t VALUES (1, 'l'); +INSERT INTO nr VALUES (2, NULL); + +SET join_use_nulls = 0; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (x) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +SELECT '-'; + +SET join_use_nulls = 1; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (x) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + + +DROP TABLE t; +DROP TABLE nr; diff --git a/tests/queries/0_stateless/01145_with_fill_const.reference b/tests/queries/0_stateless/01145_with_fill_const.reference new file mode 100644 index 000000000000..fa72c3c5993b --- /dev/null +++ b/tests/queries/0_stateless/01145_with_fill_const.reference @@ -0,0 +1,20 @@ +2020-06-16 00:00:00 +2020-06-16 00:30:00 +2020-06-16 01:00:00 +2020-06-16 01:30:00 +2020-06-16 02:00:00 +2020-06-16 02:30:00 +2020-06-16 03:00:00 +2020-06-16 03:30:00 +2020-06-16 04:00:00 +2020-06-16 04:30:00 +2020-06-16 05:00:00 +2020-06-16 05:30:00 +2020-06-16 06:00:00 +2020-06-16 06:30:00 +2020-06-16 07:00:00 +2020-06-16 07:30:00 +2020-06-16 08:00:00 +2020-06-16 08:30:00 +2020-06-16 09:00:00 +2020-06-16 09:30:00 diff --git a/tests/queries/0_stateless/01145_with_fill_const.sql b/tests/queries/0_stateless/01145_with_fill_const.sql new file mode 100644 index 000000000000..531d202c02ae --- /dev/null +++ b/tests/queries/0_stateless/01145_with_fill_const.sql @@ -0,0 +1,6 @@ +WITH toDateTime('2020-06-16 03:00:00') AS date_time +SELECT date_time ORDER BY date_time ASC +WITH FILL + FROM toDateTime('2020-06-16 00:00:00') + TO toDateTime('2020-06-16 10:00:00') + STEP 1800; diff --git a/tests/queries/0_stateless/01193_metadata_loading.reference b/tests/queries/0_stateless/01193_metadata_loading.reference new file mode 100644 index 000000000000..8ff246325ac7 --- /dev/null +++ b/tests/queries/0_stateless/01193_metadata_loading.reference @@ -0,0 +1,5 @@ +10000 0 2020-06-25 hello [1,2] [3,4] +10000 1 2020-06-26 word [10,20] [30,40] +ok +8000 0 2020-06-25 hello [1,2] [3,4] +8000 1 2020-06-26 word [10,20] [30,40] diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh new file mode 100755 index 000000000000..b7811e30ac45 --- /dev/null +++ b/tests/queries/0_stateless/01193_metadata_loading.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +# it is the worst way of making performance test, nevertheless it can detect significant slowdown and some other issues, that usually found by stress test + +db="test_01193_$RANDOM" + +declare -A engines +engines[0]="Memory" +engines[1]="File(CSV)" +engines[2]="Log" +engines[3]="StripeLog" +engines[4]="MergeTree ORDER BY i" + +tables=1000 +threads=10 +count_multiplier=1 +max_time_ms=1000 + +debug_or_sanitizer_build=`$CLICKHOUSE_CLIENT -q "WITH ((SELECT value FROM system.build_options WHERE name='BUILD_TYPE') AS build, (SELECT value FROM system.build_options WHERE name='CXX_FLAGS') as flags) SELECT build='Debug' OR flags LIKE '%fsanitize%'"` + +if [[ debug_or_sanitizer_build -eq 1 ]]; then tables=100; count_multiplier=10; max_time_ms=1500; fi + +create_tables() { + for i in $(seq 1 $tables); do + engine=${engines[$((i % ${#engines[@]}))]} + $CLICKHOUSE_CLIENT -q "CREATE TABLE $db.table_$1_$i (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=$engine" + $CLICKHOUSE_CLIENT -q "INSERT INTO $db.table_$1_$i VALUES (0, '2020-06-25', 'hello', [1, 2], [3, 4]), (1, '2020-06-26', 'word', [10, 20], [30, 40])" + done +} + +$CLICKHOUSE_CLIENT -q "CREATE DATABASE $db" + +for i in $(seq 1 $threads); do + create_tables $i & +done +wait + +$CLICKHOUSE_CLIENT -q "CREATE TABLE $db.table_merge (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=Merge('$db', '^table_')" +$CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" + +db_engine=`$CLICKHOUSE_CLIENT -q "SELECT engine FROM system.databases WHERE name='$db'"` + +$CLICKHOUSE_CLIENT -q "DETACH DATABASE $db" + +# get real time, grep seconds, remove point, remove leading zeros +elapsed_ms=`{ time $CLICKHOUSE_CLIENT -q "ATTACH DATABASE $db ENGINE=$db_engine"; } 2>&1 | grep real | grep -Po "0m\K[0-9\.]*" | tr -d '.' | sed "s/^0*//"` +$CLICKHOUSE_CLIENT -q "SELECT '01193_metadata_loading', $elapsed_ms FORMAT Null" # it will be printed to server log + +if [[ $elapsed_ms -le $max_time_ms ]]; then echo ok; fi + +$CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" + +$CLICKHOUSE_CLIENT -q "DROP DATABASE $db" diff --git a/tests/queries/0_stateless/01194_http_query_id.reference b/tests/queries/0_stateless/01194_http_query_id.reference new file mode 100644 index 000000000000..b8626c4cff28 --- /dev/null +++ b/tests/queries/0_stateless/01194_http_query_id.reference @@ -0,0 +1 @@ +4 diff --git a/tests/queries/0_stateless/01194_http_query_id.sh b/tests/queries/0_stateless/01194_http_query_id.sh new file mode 100755 index 000000000000..381ae67f88f7 --- /dev/null +++ b/tests/queries/0_stateless/01194_http_query_id.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +url="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?session_id=test_01194" +rnd=$RANDOM + +${CLICKHOUSE_CURL} -sS "$url&query=SELECT+$rnd,1" > /dev/null +${CLICKHOUSE_CURL} -sS "$url&query=SELECT+$rnd,2" > /dev/null +${CLICKHOUSE_CURL} -sS "$url" --data "SELECT $rnd,3" > /dev/null +${CLICKHOUSE_CURL} -sS "$url" --data "SELECT $rnd,4" > /dev/null + +${CLICKHOUSE_CURL} -sS "$url" --data "SYSTEM FLUSH LOGS" + +${CLICKHOUSE_CURL} -sS "$url&query=SELECT+count(DISTINCT+query_id)+FROM+system.query_log+WHERE+query+LIKE+'SELECT+$rnd%25'" diff --git a/tests/queries/0_stateless/01238_http_memory_tracking.reference b/tests/queries/0_stateless/01238_http_memory_tracking.reference new file mode 100644 index 000000000000..83b33d238dab --- /dev/null +++ b/tests/queries/0_stateless/01238_http_memory_tracking.reference @@ -0,0 +1 @@ +1000 diff --git a/tests/queries/0_stateless/01238_http_memory_tracking.sh b/tests/queries/0_stateless/01238_http_memory_tracking.sh new file mode 100755 index 000000000000..508a15a536f0 --- /dev/null +++ b/tests/queries/0_stateless/01238_http_memory_tracking.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -o pipefail + +# This is needed to keep at least one running query for user for the time of test. +# (1k http queries takes ~1 second, let's run for 5x more to avoid flaps) +${CLICKHOUSE_CLIENT} --format Null -n <<<'SELECT sleepEachRow(1) FROM numbers(5)' & + +# ignore "yes: standard output: Broken pipe" +yes 'SELECT 1' 2>/dev/null | { + head -n1000 +} | { + xargs -i ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&max_memory_usage_for_user=$((1<<30))" -d '{}' +} | grep -x -c 1 + +wait diff --git a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference index 04ae001675f1..facc265593ca 100644 --- a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference +++ b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference @@ -1,3 +1,3 @@ -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64, \n `i32` Int32, \n `f64` Float64, \n `d` Decimal(10, 2), \n `s` String, \n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), \n `dt` Date, \n INDEX bloom_filter_a i32 TYPE bloom_filter(0.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64, \n `i32` Int32, \n `f64` Float64, \n `d` Decimal(10, 2), \n `s` String, \n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), \n `dt` Date, \n INDEX bloom_filter_a i32 TYPE bloom_filter(0.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64, \n `i32` Int32, \n `f64` Float64, \n `d` Decimal(10, 2), \n `s` String, \n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), \n `dt` Date, \n INDEX bloom_filter_a i32 TYPE bloom_filter(1.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.bloom_filter_idx_good (`u64` UInt64, `i32` Int32, `f64` Float64, `d` Decimal(10, 2), `s` String, `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), `dt` Date, INDEX bloom_filter_a i32 TYPE bloom_filter(0.) GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192 +CREATE TABLE default.bloom_filter_idx_good (`u64` UInt64, `i32` Int32, `f64` Float64, `d` Decimal(10, 2), `s` String, `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), `dt` Date, INDEX bloom_filter_a i32 TYPE bloom_filter(0.) GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192 +CREATE TABLE default.bloom_filter_idx_good (`u64` UInt64, `i32` Int32, `f64` Float64, `d` Decimal(10, 2), `s` String, `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), `dt` Date, INDEX bloom_filter_a i32 TYPE bloom_filter(1.) GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql b/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql index c65bf668d717..ced6615c57f0 100644 --- a/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql +++ b/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql @@ -6,7 +6,7 @@ WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT * WHERE D WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT * WHERE materialize(S) = DT64; -- {serverError 44} SELECT * WHERE toDateTime64(123.345, 3) == 'ABCD'; -- {serverError 131} -- invalid DateTime64 string -SELECT * WHERE toDateTime64(123.345, 3) == '2020-02-05 14:34:12.33333333333333333333333333333333333333333333333333333333'; -- {serverError 131} -- invalid string length +SELECT * WHERE toDateTime64(123.345, 3) == '2020-02-05 14:34:12.33333333333333333333333333333333333333333333333333333333'; SELECT 'in SELECT'; WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT DT64 = S; diff --git a/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.reference b/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql b/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql new file mode 100644 index 000000000000..8de0f40229ca --- /dev/null +++ b/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS data_01283; + +CREATE TABLE data_01283 engine=MergeTree() +ORDER BY key +PARTITION BY key +AS SELECT number key FROM numbers(10); + +SET log_queries=1; +SELECT * FROM data_01283 LIMIT 1 FORMAT Null; +SET log_queries=0; +SYSTEM FLUSH LOGS; + +-- 1 for PullingAsyncPipelineExecutor::pull +-- 1 for AsynchronousBlockInputStream +SELECT + throwIf(count() != 1, 'no query was logged'), + throwIf(length(thread_ids) != 2, 'too many threads used') +FROM system.query_log +WHERE type = 'QueryFinish' AND query LIKE '%data_01283 LIMIT 1%' +GROUP BY thread_ids +FORMAT Null; diff --git a/tests/queries/0_stateless/01284_view_and_extremes_bug.reference b/tests/queries/0_stateless/01284_view_and_extremes_bug.reference new file mode 100644 index 000000000000..216e97ce0822 --- /dev/null +++ b/tests/queries/0_stateless/01284_view_and_extremes_bug.reference @@ -0,0 +1 @@ +World diff --git a/tests/queries/0_stateless/01284_view_and_extremes_bug.sql b/tests/queries/0_stateless/01284_view_and_extremes_bug.sql new file mode 100644 index 000000000000..c444441a2585 --- /dev/null +++ b/tests/queries/0_stateless/01284_view_and_extremes_bug.sql @@ -0,0 +1,4 @@ +drop table if exists view_bug_const; +CREATE VIEW view_bug_const AS SELECT 'World' AS hello FROM (SELECT number FROM system.numbers LIMIT 1) AS n1 JOIN (SELECT number FROM system.numbers LIMIT 1) AS n2 USING (number); +select * from view_bug_const; +drop table if exists view_bug_const; diff --git a/tests/queries/0_stateless/01287_max_execution_speed.reference b/tests/queries/0_stateless/01287_max_execution_speed.reference new file mode 100644 index 000000000000..2e85f1f53355 --- /dev/null +++ b/tests/queries/0_stateless/01287_max_execution_speed.reference @@ -0,0 +1,8 @@ +Ok (1) +Ok (2) +2000000 +1 +Ok (3) +2000000 +1 +Ok (4) diff --git a/tests/queries/0_stateless/01287_max_execution_speed.sql b/tests/queries/0_stateless/01287_max_execution_speed.sql new file mode 100644 index 000000000000..7e8f6681c84f --- /dev/null +++ b/tests/queries/0_stateless/01287_max_execution_speed.sql @@ -0,0 +1,44 @@ +SET min_execution_speed = 100000000000, timeout_before_checking_execution_speed = 0.1; +SELECT count() FROM system.numbers; -- { serverError 160 } +SELECT 'Ok (1)'; +SET min_execution_speed = 0; + +SET min_execution_speed_bytes = 800000000000, timeout_before_checking_execution_speed = 0.1; +SELECT count() FROM system.numbers; -- { serverError 160 } +SELECT 'Ok (2)'; +SET min_execution_speed_bytes = 0; + +SET max_execution_speed = 1000000; +SET max_block_size = 100; + +CREATE TEMPORARY TABLE times (t DateTime); + +INSERT INTO times SELECT now(); +SELECT count() FROM numbers(2000000); +INSERT INTO times SELECT now(); + +SELECT max(t) - min(t) >= 1 FROM times; +SELECT 'Ok (3)'; +SET max_execution_speed = 0; + +SET max_execution_speed_bytes = 8000000; +TRUNCATE TABLE times; + +INSERT INTO times SELECT now(); +SELECT count() FROM numbers(2000000); +INSERT INTO times SELECT now(); + +SELECT max(t) - min(t) >= 1 FROM times; +SELECT 'Ok (4)'; +SET max_execution_speed_bytes = 0; + +-- Note that 'min_execution_speed' does not count sleeping due to throttling +-- with 'max_execution_speed' and similar limits like 'priority' and 'max_network_bandwidth' + +-- Note: I have to disable this part of the test because it actually can work slower under sanitizers, +-- with debug builds and in presense of random system hickups in our CI environment. + +--SET max_execution_speed = 1000000, min_execution_speed = 2000000; +-- And this query will work despite the fact that the above settings look contradictory. +--SELECT count() FROM numbers(1000000); +--SELECT 'Ok (5)'; diff --git a/tests/queries/0_stateless/01288_shard_max_network_bandwidth.reference b/tests/queries/0_stateless/01288_shard_max_network_bandwidth.reference new file mode 100644 index 000000000000..0d66ea1aee95 --- /dev/null +++ b/tests/queries/0_stateless/01288_shard_max_network_bandwidth.reference @@ -0,0 +1,2 @@ +0 +1 diff --git a/tests/queries/0_stateless/01288_shard_max_network_bandwidth.sql b/tests/queries/0_stateless/01288_shard_max_network_bandwidth.sql new file mode 100644 index 000000000000..09c043784bbb --- /dev/null +++ b/tests/queries/0_stateless/01288_shard_max_network_bandwidth.sql @@ -0,0 +1,15 @@ +-- Limit to 10 MB/sec +SET max_network_bandwidth = 10000000; + +-- Lower max_block_size, so we can start throttling sooner. Otherwise query will be executed too quickly. +SET max_block_size = 100; + +CREATE TEMPORARY TABLE times (t DateTime); + +-- rand64 is uncompressable data. Each number will take 8 bytes of bandwidth. +-- This query should execute in no less than 1.6 seconds if throttled. +INSERT INTO times SELECT now(); +SELECT sum(ignore(*)) FROM (SELECT rand64() FROM remote('127.0.0.{2,3}', numbers(2000000))); +INSERT INTO times SELECT now(); + +SELECT max(t) - min(t) >= 1 FROM times; diff --git a/tests/queries/0_stateless/01289_min_execution_speed_not_too_early.reference b/tests/queries/0_stateless/01289_min_execution_speed_not_too_early.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01289_min_execution_speed_not_too_early.sql b/tests/queries/0_stateless/01289_min_execution_speed_not_too_early.sql new file mode 100644 index 000000000000..e3a18d0c5157 --- /dev/null +++ b/tests/queries/0_stateless/01289_min_execution_speed_not_too_early.sql @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS ES; + +create table ES(A String) Engine=MergeTree order by tuple(); +insert into ES select toString(number) from numbers(10000000); + +SET max_execution_time = 100, max_execution_speed = 1000000; +SET max_threads = 1; +SET max_block_size = 1000000; + +-- Exception about execution speed is not thrown from these queries. +SELECT * FROM ES LIMIT 1 format Null; +SELECT * FROM ES LIMIT 10 format Null; +SELECT * FROM ES LIMIT 100 format Null; +SELECT * FROM ES LIMIT 1000 format Null; +SELECT * FROM ES LIMIT 10000 format Null; +SELECT * FROM ES LIMIT 100000 format Null; +SELECT * FROM ES LIMIT 1000000 format Null; + +DROP TABLE ES; diff --git a/tests/queries/0_stateless/01290_empty_array_index_analysis.reference b/tests/queries/0_stateless/01290_empty_array_index_analysis.reference new file mode 100644 index 000000000000..5037a64c0f08 --- /dev/null +++ b/tests/queries/0_stateless/01290_empty_array_index_analysis.reference @@ -0,0 +1,50 @@ +--- notEmpty + ['a'] 2 + ['a','b','c'] 3 + ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'] 4 +--- empty + [] 1 +--- = [] + [] 1 +--- != [] + ['a'] 2 + ['a','b','c'] 3 + ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'] 4 +--- > [] + ['a'] 2 + ['a','b','c'] 3 + ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'] 4 +--- < [] +--- >= [] + [] 1 + ['a'] 2 + ['a','b','c'] 3 + ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'] 4 +--- <= [] + [] 1 +--- +--- notEmpty + ['a'] 2 + ['a','b','c'] 3 + ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'] 4 +--- empty + [] 1 +--- = [] + [] 1 +--- != [] + ['a'] 2 + ['a','b','c'] 3 + ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'] 4 +--- > [] + ['a'] 2 + ['a','b','c'] 3 + ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'] 4 +--- < [] +--- >= [] + [] 1 + ['a'] 2 + ['a','b','c'] 3 + ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'] 4 +--- <= [] + [] 1 +--- diff --git a/tests/queries/0_stateless/01290_empty_array_index_analysis.sql b/tests/queries/0_stateless/01290_empty_array_index_analysis.sql new file mode 100644 index 000000000000..b1b6067945dd --- /dev/null +++ b/tests/queries/0_stateless/01290_empty_array_index_analysis.sql @@ -0,0 +1,66 @@ +drop table if exists count_lc_test; + +CREATE TABLE count_lc_test +( + `s` LowCardinality(String), + `arr` Array(LowCardinality(String)), + `num` UInt64 +) +ENGINE = MergeTree +ORDER BY (s, arr); + +INSERT INTO count_lc_test(num, arr) VALUES (1,[]),(2,['a']),(3,['a','b','c']),(4,['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']); + +SELECT '--- notEmpty'; +select * from count_lc_test where notEmpty(arr); +SELECT '--- empty'; +select * from count_lc_test where empty(arr); +SELECT '--- = []'; +select * from count_lc_test where arr = []; +SELECT '--- != []'; +select * from count_lc_test where arr != []; +SELECT '--- > []'; +select * from count_lc_test where arr > []; +SELECT '--- < []'; +select * from count_lc_test where arr < []; +SELECT '--- >= []'; +select * from count_lc_test where arr >= []; +SELECT '--- <= []'; +select * from count_lc_test where arr <= []; +SELECT '---'; + +DROP TABLE count_lc_test; + + +drop table if exists count_lc_test; + +CREATE TABLE count_lc_test +( + `s` LowCardinality(String), + `arr` Array(String), + `num` UInt64 +) +ENGINE = MergeTree +ORDER BY (s, arr); + +INSERT INTO count_lc_test(num, arr) VALUES (1,[]),(2,['a']),(3,['a','b','c']),(4,['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']); + +SELECT '--- notEmpty'; +select * from count_lc_test where notEmpty(arr); +SELECT '--- empty'; +select * from count_lc_test where empty(arr); +SELECT '--- = []'; +select * from count_lc_test where arr = []; +SELECT '--- != []'; +select * from count_lc_test where arr != []; +SELECT '--- > []'; +select * from count_lc_test where arr > []; +SELECT '--- < []'; +select * from count_lc_test where arr < []; +SELECT '--- >= []'; +select * from count_lc_test where arr >= []; +SELECT '--- <= []'; +select * from count_lc_test where arr <= []; +SELECT '---'; + +DROP TABLE count_lc_test; diff --git a/tests/queries/0_stateless/01290_max_execution_speed_distributed.reference b/tests/queries/0_stateless/01290_max_execution_speed_distributed.reference new file mode 100644 index 000000000000..ad0e80d8e690 --- /dev/null +++ b/tests/queries/0_stateless/01290_max_execution_speed_distributed.reference @@ -0,0 +1,3 @@ +2000000 +1 +1 diff --git a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql new file mode 100644 index 000000000000..8282390ca90b --- /dev/null +++ b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql @@ -0,0 +1,13 @@ +SET max_execution_speed = 1000000, timeout_before_checking_execution_speed = 0.001, max_block_size = 100; + +CREATE TEMPORARY TABLE times (t DateTime); + +INSERT INTO times SELECT now(); +SELECT count('special query for 01290_max_execution_speed_distributed') FROM remote('127.0.0.{2,3}', numbers(1000000)); +INSERT INTO times SELECT now(); + +SELECT max(t) - min(t) >= 1 FROM times; + +-- Check that the query was also throttled on "remote" servers. +SYSTEM FLUSH LOGS; +SELECT DISTINCT query_duration_ms >= 500 FROM system.query_log WHERE event_date >= yesterday() AND query LIKE '%special query for 01290_max_execution_speed_distributed%' AND type = 2; diff --git a/tests/queries/0_stateless/01291_unsupported_conversion_from_decimal.reference b/tests/queries/0_stateless/01291_unsupported_conversion_from_decimal.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01291_unsupported_conversion_from_decimal.sql b/tests/queries/0_stateless/01291_unsupported_conversion_from_decimal.sql new file mode 100644 index 000000000000..256c64249018 --- /dev/null +++ b/tests/queries/0_stateless/01291_unsupported_conversion_from_decimal.sql @@ -0,0 +1,5 @@ +SELECT toIntervalSecond(now64()); -- { serverError 70 } +SELECT CAST(now64() AS IntervalSecond); -- { serverError 70 } + +SELECT toIntervalSecond(now64()); -- { serverError 70 } +SELECT CAST(now64() AS IntervalSecond); -- { serverError 70 } diff --git a/tests/queries/0_stateless/01292_quantile_array_bug.reference b/tests/queries/0_stateless/01292_quantile_array_bug.reference new file mode 100644 index 000000000000..36abe23e89c7 --- /dev/null +++ b/tests/queries/0_stateless/01292_quantile_array_bug.reference @@ -0,0 +1 @@ +[7] diff --git a/tests/queries/0_stateless/01292_quantile_array_bug.sql b/tests/queries/0_stateless/01292_quantile_array_bug.sql new file mode 100644 index 000000000000..ecb1028d569b --- /dev/null +++ b/tests/queries/0_stateless/01292_quantile_array_bug.sql @@ -0,0 +1 @@ +select quantilesExactWeightedArray(0.5)(range(10), range(10)) diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.reference b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.sh b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.sh new file mode 100755 index 000000000000..4d4292472da8 --- /dev/null +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.sh @@ -0,0 +1,88 @@ +#!/usr/bin/expect -f + +log_user 0 +set timeout 60 +spawn clickhouse-client +match_max 100000 + +expect ":) " + +send -- "SELECT 1\r" +expect "│ 1 │" +expect ":) " + +send -- "SELECT 1\\G\r" +expect "Row 1:" +expect "1: 1" +expect ":) " + +send -- "SELECT 1\\\r" +expect ":-] " +send -- ", 2\r" +expect "│ 1 │ 2 │" +expect ":) " + +send -- "SELECT 1\\\r" +expect ":-] " +send -- ", 2\\G\r" +expect "Row 1:" +expect "1: 1" +expect "2: 2" +expect ":) " + +send -- "" +expect eof + +set timeout 60 +spawn clickhouse-client --multiline +match_max 100000 + +expect ":) " + +send -- "SELECT 1;\r" +expect "│ 1 │" +expect ":) " + +send -- "SELECT 1\\G\r" +expect "Row 1:" +expect "1: 1" +expect ":) " + +send -- "SELECT 1; \r" +expect "│ 1 │" +expect ":) " + +send -- "SELECT 1\\G \r" +expect "Row 1:" +expect "1: 1" +expect ":) " + +send -- "SELECT 1\r" +expect ":-] " +send -- ";\r" +expect "│ 1 │" +expect ":) " + +send -- "SELECT 1\r" +expect ":-] " +send -- "\\G\r" +expect "Row 1:" +expect "1: 1" +expect ":) " + +send -- "SELECT 1\r" +expect ":-] " +send -- ", 2;\r" +expect "│ 1 │ 2 │" +expect ":) " + +send -- "SELECT 1\r" +expect ":-] " +send -- ", 2\\G\r" +expect "Row 1:" +expect "1: 1" +expect "2: 2" +expect ":) " + +send -- "" +expect eof diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.reference b/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.sh b/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.sh new file mode 100755 index 000000000000..3b98caeff2ec --- /dev/null +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.sh @@ -0,0 +1,34 @@ +#!/usr/bin/expect -f + +log_user 0 +set timeout 60 +spawn clickhouse-client +match_max 100000 + +expect ":) " + +send -- "SELECT 1\r" +expect "│ 1 │" +expect ":) " + +send -- "SELECT 1\\G\r" +expect "Row 1:" +expect "1: 1" +expect ":) " + +send -- "SELECT 1\\\r" +expect ":-] " +send -- ", 2\r" +expect "│ 1 │ 2 │" +expect ":) " + +send -- "SELECT 1\\\r" +expect ":-] " +send -- ", 2\\G\r" +expect "Row 1:" +expect "1: 1" +expect "2: 2" +expect ":) " + +send -- "" +expect eof diff --git a/tests/queries/0_stateless/01293_external_sorting_limit_bug.reference b/tests/queries/0_stateless/01293_external_sorting_limit_bug.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01293_external_sorting_limit_bug.sql b/tests/queries/0_stateless/01293_external_sorting_limit_bug.sql new file mode 100644 index 000000000000..570ec4b8c3ce --- /dev/null +++ b/tests/queries/0_stateless/01293_external_sorting_limit_bug.sql @@ -0,0 +1 @@ +SELECT number FROM (SELECT number FROM system.numbers LIMIT 999990) ORDER BY number ASC LIMIT 100, 65535 SETTINGS max_bytes_before_external_sort = 1000000 format Null diff --git a/tests/queries/0_stateless/01295_aggregation_bug_11413.reference b/tests/queries/0_stateless/01295_aggregation_bug_11413.reference new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/tests/queries/0_stateless/01295_aggregation_bug_11413.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01295_aggregation_bug_11413.sql b/tests/queries/0_stateless/01295_aggregation_bug_11413.sql new file mode 100644 index 000000000000..ec43be9eab3d --- /dev/null +++ b/tests/queries/0_stateless/01295_aggregation_bug_11413.sql @@ -0,0 +1 @@ +SELECT 1 FROM remote('127.0.0.{1,2}', numbers(99)) GROUP BY materialize(1) HAVING count() > 0 AND argMax(1, tuple(0)) diff --git a/tests/queries/0_stateless/01296_codecs_bad_arguments.reference b/tests/queries/0_stateless/01296_codecs_bad_arguments.reference new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/tests/queries/0_stateless/01296_codecs_bad_arguments.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01296_codecs_bad_arguments.sql b/tests/queries/0_stateless/01296_codecs_bad_arguments.sql new file mode 100644 index 000000000000..d7eb53300ecd --- /dev/null +++ b/tests/queries/0_stateless/01296_codecs_bad_arguments.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS delta_table; +DROP TABLE IF EXISTS zstd_table; +DROP TABLE IF EXISTS lz4_table; + +CREATE TABLE delta_table (`id` UInt64 CODEC(Delta(tuple()))) ENGINE = MergeTree() ORDER BY tuple(); --{serverError 433} +CREATE TABLE zstd_table (`id` UInt64 CODEC(ZSTD(tuple()))) ENGINE = MergeTree() ORDER BY tuple(); --{serverError 433} +CREATE TABLE lz4_table (`id` UInt64 CODEC(LZ4HC(tuple()))) ENGINE = MergeTree() ORDER BY tuple(); --{serverError 433} + +CREATE TABLE lz4_table (`id` UInt64 CODEC(LZ4(tuple()))) ENGINE = MergeTree() ORDER BY tuple(); --{serverError 378} + +SELECT 1; + +DROP TABLE IF EXISTS delta_table; +DROP TABLE IF EXISTS zstd_table; +DROP TABLE IF EXISTS lz4_table; diff --git a/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.reference b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.reference new file mode 100644 index 000000000000..b20e7415f52d --- /dev/null +++ b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.reference @@ -0,0 +1,2 @@ +Memory limit (for query) exceeded +Ok diff --git a/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh new file mode 100755 index 000000000000..633fa5ce315a --- /dev/null +++ b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +function test() +{ + for i in {1..1000}; do + $CLICKHOUSE_CLIENT --max_memory_usage 1G <<< "SELECT uniqExactState(number) FROM system.numbers_mt GROUP BY number % 10"; + done +} + +export -f test; + +# If the memory leak exists, it will lead to OOM fairly quickly. +timeout 30 bash -c test 2>&1 | grep -o -F 'Memory limit (for query) exceeded' | uniq +echo 'Ok' diff --git a/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.reference b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.reference new file mode 100644 index 000000000000..7326d9603970 --- /dev/null +++ b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.reference @@ -0,0 +1 @@ +Ok diff --git a/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh new file mode 100755 index 000000000000..cd2fec408abf --- /dev/null +++ b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +function test() +{ + for i in {1..250}; do + $CLICKHOUSE_CLIENT --query "SELECT groupArrayIfState(('Hello, world' AS s) || s || s || s || s || s || s || s || s || s, NOT throwIf(number > 10000000, 'Ok')) FROM system.numbers_mt GROUP BY number % 10"; + done +} + +export -f test; + +# If the memory leak exists, it will lead to OOM fairly quickly. +timeout 30 bash -c test 2>&1 | grep -o -F 'Ok' | uniq diff --git a/tests/queries/0_stateless/01304_direct_io.reference b/tests/queries/0_stateless/01304_direct_io.reference new file mode 100644 index 000000000000..ec7a223ddc2b --- /dev/null +++ b/tests/queries/0_stateless/01304_direct_io.reference @@ -0,0 +1 @@ +Loaded 1 queries. diff --git a/tests/queries/0_stateless/01304_direct_io.sh b/tests/queries/0_stateless/01304_direct_io.sh new file mode 100755 index 000000000000..32091acd5ebd --- /dev/null +++ b/tests/queries/0_stateless/01304_direct_io.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --multiquery --query " + DROP TABLE IF EXISTS bug; + CREATE TABLE bug (UserID UInt64, Date Date) ENGINE = MergeTree ORDER BY Date; + INSERT INTO bug SELECT rand64(), '2020-06-07' FROM numbers(50000000); + OPTIMIZE TABLE bug FINAL;" + +$CLICKHOUSE_BENCHMARK --database $CLICKHOUSE_DATABASE --iterations 10 --max_threads 100 --min_bytes_to_use_direct_io 1 <<< "SELECT sum(UserID) FROM bug PREWHERE NOT ignore(Date)" 1>/dev/null 2>$CLICKHOUSE_TMP/err +cat $CLICKHOUSE_TMP/err | grep Exception +cat $CLICKHOUSE_TMP/err | grep Loaded + +$CLICKHOUSE_CLIENT --multiquery --query " + DROP TABLE bug;" diff --git a/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.reference b/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.reference new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.sql b/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.sql new file mode 100644 index 000000000000..535dee5ebbe7 --- /dev/null +++ b/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.sql @@ -0,0 +1,5 @@ +drop table if exists h; +create table h (EventDate Date, CounterID UInt64, WatchID UInt64) engine = MergeTree order by (CounterID, EventDate); +insert into h values ('2020-06-10', 16671268, 1); +SELECT count() from h ARRAY JOIN [1] AS a PREWHERE WatchID IN (SELECT toUInt64(1)) WHERE (EventDate = '2020-06-10') AND (CounterID = 16671268); +drop table if exists h; diff --git a/tests/queries/0_stateless/01305_buffer_final_bug.reference b/tests/queries/0_stateless/01305_buffer_final_bug.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01305_buffer_final_bug.sql b/tests/queries/0_stateless/01305_buffer_final_bug.sql new file mode 100644 index 000000000000..8d1586932ce7 --- /dev/null +++ b/tests/queries/0_stateless/01305_buffer_final_bug.sql @@ -0,0 +1,11 @@ +drop table if exists t; +drop table if exists t_buf; + +create table t (x UInt64) engine = MergeTree order by (x, intHash64(x)) sample by intHash64(x); +insert into t select number from numbers(10000); +create table t_buf as t engine = Buffer(currentDatabase(), 't', 16, 20, 100, 100000, 10000000, 50000000, 250000000); +insert into t_buf values (1); +select count() from t_buf sample 1/2 format Null; + +drop table if exists t_buf; +drop table if exists t; diff --git a/tests/queries/0_stateless/01305_nullable-prewhere_bug.reference b/tests/queries/0_stateless/01305_nullable-prewhere_bug.reference new file mode 100644 index 000000000000..bd81ae06cec3 --- /dev/null +++ b/tests/queries/0_stateless/01305_nullable-prewhere_bug.reference @@ -0,0 +1 @@ +some_field_value 1 diff --git a/tests/queries/0_stateless/01305_nullable-prewhere_bug.sql b/tests/queries/0_stateless/01305_nullable-prewhere_bug.sql new file mode 100644 index 000000000000..35d3762660f6 --- /dev/null +++ b/tests/queries/0_stateless/01305_nullable-prewhere_bug.sql @@ -0,0 +1,5 @@ +drop table if exists data; +CREATE TABLE data (ts DateTime, field String, num_field Nullable(Float64)) ENGINE = MergeTree() PARTITION BY ts ORDER BY ts; +insert into data values(toDateTime('2020-05-14 02:08:00'),'some_field_value',7.); +SELECT field, countIf(num_field > 6.0) FROM data PREWHERE (num_field>6.0) GROUP BY field; +drop table if exists data; diff --git a/tests/queries/0_stateless/01116_cross_count_asterisks.reference b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.reference similarity index 50% rename from tests/queries/0_stateless/01116_cross_count_asterisks.reference rename to tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.reference index 8347b144a351..98fb6a686563 100644 --- a/tests/queries/0_stateless/01116_cross_count_asterisks.reference +++ b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.reference @@ -1,4 +1,4 @@ -2 1 -2 +1 +1 1 diff --git a/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql new file mode 100644 index 000000000000..832f7140af25 --- /dev/null +++ b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql @@ -0,0 +1,8 @@ +DROP TABLE IF EXISTS test_01307; +CREATE TABLE test_01307 (id UInt64, val String, INDEX ind val TYPE bloom_filter() GRANULARITY 1) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 2; +INSERT INTO test_01307 (id, val) select number as id, toString(number) as val from numbers(4); +SELECT count() FROM test_01307 WHERE identity(val) = '2'; +SELECT count() FROM test_01307 WHERE val = '2'; +OPTIMIZE TABLE test_01307 FINAL; +SELECT count() FROM test_01307 WHERE identity(val) = '2'; +SELECT count() FROM test_01307 WHERE val = '2'; diff --git a/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.reference b/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.reference new file mode 100644 index 000000000000..61150aca43cf --- /dev/null +++ b/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.reference @@ -0,0 +1,3 @@ +3 +2 +3 diff --git a/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.sql b/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.sql new file mode 100644 index 000000000000..c105885cb604 --- /dev/null +++ b/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS t; + +CREATE TABLE t (x UInt8) ENGINE = MergeTree ORDER BY x; +INSERT INTO t VALUES (1), (2), (3); + +SELECT count() FROM t; +CREATE ROW POLICY filter ON t USING (x % 2 = 1) TO ALL; +SELECT count() FROM t; +DROP ROW POLICY filter ON t; +SELECT count() FROM t; + +DROP TABLE t; diff --git a/tests/queries/0_stateless/01312_case_insensitive_regexp.reference b/tests/queries/0_stateless/01312_case_insensitive_regexp.reference new file mode 100644 index 000000000000..c18b4e9b0829 --- /dev/null +++ b/tests/queries/0_stateless/01312_case_insensitive_regexp.reference @@ -0,0 +1,8 @@ +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01312_case_insensitive_regexp.sql b/tests/queries/0_stateless/01312_case_insensitive_regexp.sql new file mode 100644 index 000000000000..ca13989599d9 --- /dev/null +++ b/tests/queries/0_stateless/01312_case_insensitive_regexp.sql @@ -0,0 +1,8 @@ +SELECT match('Too late', 'Too late'); +select match('Too late', '(?i)Too late'); +select match('Too late', '(?i)too late'); +select match('Too late', '(?i:too late)'); +select match('Too late', '(?i)to{2} late'); +select match('Too late', '(?i)to(?)o late'); +select match('Too late', '(?i)to+ late'); +select match('Too late', '(?i)to(?:o|o) late'); diff --git a/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.reference b/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.reference new file mode 100644 index 000000000000..9524f73c87d8 --- /dev/null +++ b/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.reference @@ -0,0 +1,6 @@ +90001 +2 +waiting default mutation_table 0000000000 +is_done parts_to_do +0 1 +MUTATE_PART 0_0_0_0_2 diff --git a/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.sh b/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.sh new file mode 100755 index 000000000000..3dc8b34fff64 --- /dev/null +++ b/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS mutation_table" + +$CLICKHOUSE_CLIENT --query " + CREATE TABLE mutation_table( + key UInt64, + value String + ) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/mutation_table', '1') + ORDER BY key + PARTITION BY key % 10 +" + +$CLICKHOUSE_CLIENT --query "INSERT INTO mutation_table select number, toString(number) from numbers(100000) where number % 10 != 0" + +$CLICKHOUSE_CLIENT --query "INSERT INTO mutation_table VALUES(0, 'hello')" + +$CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM mutation_table" + +$CLICKHOUSE_CLIENT --query "ALTER TABLE mutation_table MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=0" + +first_mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations where table='mutation_table' and database='$CLICKHOUSE_DATABASE'") + +# Here we have long sleeps, but they shouldn't lead to flaps. We just check that +# background mutation finalization function will be triggered at least once. In +# rare cases this test doesn't check anything, but will report OK. +sleep 7 + +$CLICKHOUSE_CLIENT --query "ALTER TABLE mutation_table MODIFY COLUMN value UInt32 SETTINGS replication_alter_partitions_sync=0" + + +#### just check that both mutations started +check_query="SELECT count() FROM system.mutations WHERE table='mutation_table' and database='$CLICKHOUSE_DATABASE'" + +query_result=`$CLICKHOUSE_CLIENT --query="$check_query" 2>&1` + +while [ "$query_result" != "2" ] +do + query_result=`$CLICKHOUSE_CLIENT --query="$check_query" 2>&1` + sleep 0.5 +done + +echo $query_result + +$CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$first_mutation_id'" + +sleep 7 + +$CLICKHOUSE_CLIENT --query "SELECT is_done, parts_to_do FROM system.mutations where table='mutation_table' and database='$CLICKHOUSE_DATABASE' FORMAT TSVWithNames" + +$CLICKHOUSE_CLIENT --query "SELECT type, new_part_name FROM system.replication_queue WHERE table='mutation_table' and database='$CLICKHOUSE_DATABASE'" + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS mutation_table" diff --git a/tests/queries/0_stateless/01323_if_with_nulls.reference b/tests/queries/0_stateless/01323_if_with_nulls.reference new file mode 100644 index 000000000000..6bf2d206e0b6 --- /dev/null +++ b/tests/queries/0_stateless/01323_if_with_nulls.reference @@ -0,0 +1,20 @@ +\N Nullable(UInt8) +0 Nullable(UInt8) +\N Nullable(UInt8) +0 Nullable(UInt8) +0 ok +\N ok +0 ok +\N ok +ok +ok +ok +ok +ok +ok +ok +Nullable(UInt8) \N 1 ok ok ok +Nullable(UInt8) \N 1 ok ok ok +Nullable(UInt8) \N 1 ok ok ok +Nullable(UInt8) \N 1 ok ok ok +\N 1 Nullable(Int8) \N ok diff --git a/tests/queries/0_stateless/01323_if_with_nulls.sql b/tests/queries/0_stateless/01323_if_with_nulls.sql new file mode 100644 index 000000000000..f2cd943988f4 --- /dev/null +++ b/tests/queries/0_stateless/01323_if_with_nulls.sql @@ -0,0 +1,38 @@ +SELECT if(1 = 0, toNullable(toUInt8(0)), NULL) AS x, toTypeName(x); +SELECT if(1 = 1, toNullable(toUInt8(0)), NULL) AS x, toTypeName(x); +SELECT if(1 = 1, NULL, toNullable(toUInt8(0))) AS x, toTypeName(x); +SELECT if(1 = 0, NULL, toNullable(toUInt8(0))) AS x, toTypeName(x); + +SELECT if(toUInt8(0), NULL, toNullable(toUInt8(0))) AS x, if(x = 0, 'ok', 'fail'); +SELECT if(toUInt8(1), NULL, toNullable(toUInt8(0))) AS x, if(x = 0, 'fail', 'ok'); +SELECT if(toUInt8(1), toNullable(toUInt8(0)), NULL) AS x, if(x = 0, 'ok', 'fail'); +SELECT if(toUInt8(0), toNullable(toUInt8(0)), NULL) AS x, if(x = 0, 'fail', 'ok'); + +SELECT if(x = 0, 'ok', 'fail') FROM (SELECT toNullable(toUInt8(0)) AS x); +SELECT if(x = 0, 'fail', 'ok') FROM (SELECT CAST(NULL, 'Nullable(UInt8)') AS x); +SELECT if(x = 0, 'fail', 'ok') FROM (SELECT materialize(CAST(NULL, 'Nullable(UInt8)')) AS x); + +SELECT if(x = 0, 'ok', 'fail') FROM (SELECT if(toUInt8(1), toNullable(toUInt8(0)), NULL) AS x); +SELECT if(x = 0, 'fail', 'ok') FROM (SELECT if(toUInt8(0), toNullable(toUInt8(0)), NULL) AS x); + +SELECT if(x = 0, 'ok', 'fail') FROM (SELECT if(toUInt8(0), NULL, toNullable(toUInt8(0))) AS x); +SELECT if(x = 0, 'fail', 'ok') FROM (SELECT if(toUInt8(1), NULL, toNullable(toUInt8(0))) AS x); + +SELECT toTypeName(x), x, isNull(x), if(x = 0, 'fail', 'ok'), if(x = 1, 'fail', 'ok'), if(x >= 0, 'fail', 'ok') +FROM (SELECT CAST(NULL, 'Nullable(UInt8)') AS x); + +SELECT toTypeName(x), x, isNull(x), if(x = 0, 'fail', 'ok'), if(x = 1, 'fail', 'ok'), if(x >= 0, 'fail', 'ok') +FROM (SELECT materialize(CAST(NULL, 'Nullable(UInt8)')) AS x); + +SELECT toTypeName(x), x, isNull(x), if(x = 0, 'fail', 'ok'), if(x = 1, 'fail', 'ok'), if(x >= 0, 'fail', 'ok') +FROM (SELECT if(1 = 0, toNullable(toUInt8(0)), NULL) AS x); + +SELECT toTypeName(x), x, isNull(x), if(x = 0, 'fail', 'ok'), if(x = 1, 'fail', 'ok'), if(x >= 0, 'fail', 'ok') +FROM (SELECT materialize(if(1 = 0, toNullable(toUInt8(0)), NULL)) AS x); + +SET join_use_nulls = 1; + +SELECT b_num, isNull(b_num), toTypeName(b_num), b_num = 0, if(b_num = 0, 'fail', 'ok') +FROM (SELECT 1 k, toInt8(1) a_num) AS x +LEFT JOIN (SELECT 2 k, toInt8(1) b_num) AS y +USING (k); diff --git a/tests/queries/0_stateless/01323_too_many_threads_bug.reference b/tests/queries/0_stateless/01323_too_many_threads_bug.reference new file mode 100644 index 000000000000..6ed281c757a9 --- /dev/null +++ b/tests/queries/0_stateless/01323_too_many_threads_bug.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/01323_too_many_threads_bug.sql b/tests/queries/0_stateless/01323_too_many_threads_bug.sql new file mode 100644 index 000000000000..62758e17b97f --- /dev/null +++ b/tests/queries/0_stateless/01323_too_many_threads_bug.sql @@ -0,0 +1,19 @@ +drop table if exists table_01323_many_parts; + +create table table_01323_many_parts (x UInt64) engine = MergeTree order by x partition by x % 100; +set max_partitions_per_insert_block = 100; +insert into table_01323_many_parts select number from numbers(100000); + +set max_threads = 16; +set log_queries = 1; +select x from table_01323_many_parts limit 10 format Null; + +system flush logs; +select length(thread_ids) <= 4 from system.query_log where event_date >= today() - 1 and lower(query) like '%select x from table_01323_many_parts%' and type = 'QueryFinish' order by query_start_time desc limit 1; + +select x from table_01323_many_parts order by x limit 10 format Null; + +system flush logs; +select length(thread_ids) <= 20 from system.query_log where event_date >= today() - 1 and lower(query) like '%select x from table_01323_many_parts order by x%' and type = 'QueryFinish' order by query_start_time desc limit 1; + +drop table if exists table_01323_many_parts; diff --git a/tests/queries/0_stateless/01330_array_join_in_higher_order_function.reference b/tests/queries/0_stateless/01330_array_join_in_higher_order_function.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01330_array_join_in_higher_order_function.sql b/tests/queries/0_stateless/01330_array_join_in_higher_order_function.sql new file mode 100644 index 000000000000..456b24a03d04 --- /dev/null +++ b/tests/queries/0_stateless/01330_array_join_in_higher_order_function.sql @@ -0,0 +1 @@ +SELECT arrayMap(x -> arrayJoin([x, 1]), [1, 2]); -- { serverError 36 } diff --git a/tests/queries/0_stateless/01333_select_abc_asterisk.reference b/tests/queries/0_stateless/01333_select_abc_asterisk.reference new file mode 100644 index 000000000000..573541ac9702 --- /dev/null +++ b/tests/queries/0_stateless/01333_select_abc_asterisk.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/01333_select_abc_asterisk.sql b/tests/queries/0_stateless/01333_select_abc_asterisk.sql new file mode 100644 index 000000000000..e59829131d64 --- /dev/null +++ b/tests/queries/0_stateless/01333_select_abc_asterisk.sql @@ -0,0 +1,6 @@ +select *; + +--error: should be failed for abc.*; +select abc.*; --{serverError 47} +select *, abc.*; --{serverError 47} +select abc.*, *; --{serverError 47} diff --git a/tests/queries/0_stateless/01340_datetime64_fpe.reference b/tests/queries/0_stateless/01340_datetime64_fpe.reference new file mode 100644 index 000000000000..0a99fbafde48 --- /dev/null +++ b/tests/queries/0_stateless/01340_datetime64_fpe.reference @@ -0,0 +1,22 @@ +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 +2011-11-11 11:11:11 diff --git a/tests/queries/0_stateless/01340_datetime64_fpe.sql b/tests/queries/0_stateless/01340_datetime64_fpe.sql new file mode 100644 index 000000000000..3e76e3164b1c --- /dev/null +++ b/tests/queries/0_stateless/01340_datetime64_fpe.sql @@ -0,0 +1,71 @@ +WITH toDateTime64('2019-09-16 19:20:12.3456789102019-09-16 19:20:12.345678910', 0) AS dt64 SELECT dt64; -- { serverError 6 } + +SELECT toDateTime64('2011-11-11 11:11:11.1234567890123456789', 0); +SELECT toDateTime64('2011-11-11 11:11:11.-12345678901234567890', 0); -- { serverError 6 } + + +SELECT toDateTime64('2011-11-11 11:11:11.1', 0); +SELECT toDateTime64('2011-11-11 11:11:11.11', 0); +SELECT toDateTime64('2011-11-11 11:11:11.111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.1111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.11111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.1111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.11111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.1111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.11111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.111111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.1111111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.11111111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.111111111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.1111111111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.11111111111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.111111111111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.1111111111111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.11111111111111111111', 0); +SELECT toDateTime64('2011-11-11 11:11:11.111111111111111111111', 0); + +SELECT toDateTime64('2011-11-11 11:11:11.-1', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-11', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-1111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-11111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-1111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-11111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-1111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-11111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-111111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-1111111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-11111111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-111111111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-1111111111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-11111111111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-111111111111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-1111111111111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-11111111111111111111', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.-111111111111111111111', 0); -- { serverError 6 } + +SELECT toDateTime64('2011-11-11 11:11:11.+1', 0); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.++11', 10); -- { serverError 69 } +SELECT toDateTime64('2011-11-11 11:11:11.+111', 3); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.+++1111', 5); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.+11111', 7); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.+++++111111', 2); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.+1111111', 1); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.++++++11111111', 8); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.+111111111', 9); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.+++++++1111111111', 6); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.+11111111111', 4); -- { serverError 6 } +SELECT toDateTime64('2011-11-11 11:11:11.++++++++111111111111', 11); -- { serverError 69 } +SELECT toDateTime64('2011-11-11 11:11:11.+1111111111111', 15); -- { serverError 69 } +SELECT toDateTime64('2011-11-11 11:11:11.+++++++++11111111111111', 13); -- { serverError 69 } +SELECT toDateTime64('2011-11-11 11:11:11.+111111111111111', 12); -- { serverError 69 } +SELECT toDateTime64('2011-11-11 11:11:11.++++++++++1111111111111111', 16); -- { serverError 69 } +SELECT toDateTime64('2011-11-11 11:11:11.+11111111111111111', 14); -- { serverError 69 } +SELECT toDateTime64('2011-11-11 11:11:11.+++++++++++111111111111111111', 15); -- { serverError 69 } +SELECT toDateTime64('2011-11-11 11:11:11.+1111111111111111111', 17); -- { serverError 69 } +SELECT toDateTime64('2011-11-11 11:11:11.++++++++++++11111111111111111111', 19); -- { serverError 69 } +SELECT toDateTime64('2011-11-11 11:11:11.+111111111111111111111', 18); -- { serverError 69 } diff --git a/tests/queries/0_stateless/01341_datetime64_wrong_supertype.reference b/tests/queries/0_stateless/01341_datetime64_wrong_supertype.reference new file mode 100644 index 000000000000..09076dd68dba --- /dev/null +++ b/tests/queries/0_stateless/01341_datetime64_wrong_supertype.reference @@ -0,0 +1 @@ +['2000-01-01 01:01:01.123000','2000-01-01 01:01:01.123456'] diff --git a/tests/queries/0_stateless/01341_datetime64_wrong_supertype.sql b/tests/queries/0_stateless/01341_datetime64_wrong_supertype.sql new file mode 100644 index 000000000000..39acaf93e347 --- /dev/null +++ b/tests/queries/0_stateless/01341_datetime64_wrong_supertype.sql @@ -0,0 +1 @@ +SELECT [toDateTime64('2000-01-01 01:01:01.123', 3), toDateTime64('2000-01-01 01:01:01.123456', 6)]; diff --git a/tests/queries/0_stateless/01342_query_parameters_alias.reference b/tests/queries/0_stateless/01342_query_parameters_alias.reference new file mode 100644 index 000000000000..dffa16eaec2b --- /dev/null +++ b/tests/queries/0_stateless/01342_query_parameters_alias.reference @@ -0,0 +1,3 @@ +a +Nullable(Nothing) +\N diff --git a/tests/queries/0_stateless/01342_query_parameters_alias.sh b/tests/queries/0_stateless/01342_query_parameters_alias.sh new file mode 100755 index 000000000000..1f46f6b388e0 --- /dev/null +++ b/tests/queries/0_stateless/01342_query_parameters_alias.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --param_x '\N' --query 'SELECT {x:Nullable(Nothing)} as a' --format TSVWithNamesAndTypes diff --git a/tests/queries/0_stateless/01350_intdiv_nontrivial_fpe.reference b/tests/queries/0_stateless/01350_intdiv_nontrivial_fpe.reference new file mode 100644 index 000000000000..ff5ca85eb556 --- /dev/null +++ b/tests/queries/0_stateless/01350_intdiv_nontrivial_fpe.reference @@ -0,0 +1,3 @@ +-36170086419038336 +-140739635871744 +-2147483648 diff --git a/tests/queries/0_stateless/01350_intdiv_nontrivial_fpe.sql b/tests/queries/0_stateless/01350_intdiv_nontrivial_fpe.sql new file mode 100644 index 000000000000..29dfb2c3fda1 --- /dev/null +++ b/tests/queries/0_stateless/01350_intdiv_nontrivial_fpe.sql @@ -0,0 +1,5 @@ +select intDiv(-9223372036854775808, 255); +select intDiv(-9223372036854775808, 65535); +select intDiv(-9223372036854775808, 4294967295); +select intDiv(-9223372036854775808, 18446744073709551615); -- { serverError 153 } +select intDiv(-9223372036854775808, -1); -- { serverError 153 } diff --git a/tests/queries/0_stateless/01352_generate_random_overflow.reference b/tests/queries/0_stateless/01352_generate_random_overflow.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01352_generate_random_overflow.sql b/tests/queries/0_stateless/01352_generate_random_overflow.sql new file mode 100644 index 000000000000..d49f8cb26878 --- /dev/null +++ b/tests/queries/0_stateless/01352_generate_random_overflow.sql @@ -0,0 +1 @@ +SELECT i FROM generateRandom('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 1025, 65535, 9223372036854775807) LIMIT 10; -- { serverError 128 } diff --git a/tests/queries/0_stateless/01353_neighbor_overflow.reference b/tests/queries/0_stateless/01353_neighbor_overflow.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01353_neighbor_overflow.sql b/tests/queries/0_stateless/01353_neighbor_overflow.sql new file mode 100644 index 000000000000..dee18d52ae68 --- /dev/null +++ b/tests/queries/0_stateless/01353_neighbor_overflow.sql @@ -0,0 +1,2 @@ +SELECT neighbor(toString(number), -9223372036854775808) FROM numbers(100); -- { serverError 69 } +WITH neighbor(toString(number), toInt64(rand64())) AS x SELECT * FROM system.numbers WHERE NOT ignore(x); -- { serverError 69 } diff --git a/tests/queries/0_stateless/01353_nullable_tuple.reference b/tests/queries/0_stateless/01353_nullable_tuple.reference new file mode 100644 index 000000000000..b79391827802 --- /dev/null +++ b/tests/queries/0_stateless/01353_nullable_tuple.reference @@ -0,0 +1,92 @@ +single argument +1 +0 +1 +0 +1 +0 +- 1 +1 +1 +1 +0 +0 +0 +0 +0 +0 +1 +1 +1 +- 2 +1 +1 +1 +0 +0 +0 +0 +0 +1 +1 +1 +1 +- 3 +1 +1 +1 +1 +1 +1 +- 4 +\N +\N +\N +\N +\N +\N +two arguments +1 +1 +1 +1 +1 +1 +- 1 +0 +0 +0 +0 +0 +0 +- 2 +1 +1 +1 +1 +1 +1 +- 3 +\N +\N +\N +\N +\N +1 +\N +\N +0 +many arguments +1 +1 +0 +0 +1 +0 +1 +\N +\N +\N +\N +\N +\N diff --git a/tests/queries/0_stateless/01353_nullable_tuple.sql b/tests/queries/0_stateless/01353_nullable_tuple.sql new file mode 100644 index 000000000000..f757e2c42d7f --- /dev/null +++ b/tests/queries/0_stateless/01353_nullable_tuple.sql @@ -0,0 +1,107 @@ +select 'single argument'; +select tuple(number) = tuple(number) from numbers(1); +select tuple(number) = tuple(number + 1) from numbers(1); +select tuple(toNullable(number)) = tuple(number) from numbers(1); +select tuple(toNullable(number)) = tuple(number + 1) from numbers(1); +select tuple(toNullable(number)) = tuple(toNullable(number)) from numbers(1); +select tuple(toNullable(number)) = tuple(toNullable(number + 1)) from numbers(1); +select '- 1'; +select tuple(toNullable(number)) < tuple(number + 1) from numbers(1); +select tuple(number) < tuple(toNullable(number + 1)) from numbers(1); +select tuple(toNullable(number)) < tuple(toNullable(number + 1)) from numbers(1); + +select tuple(toNullable(number)) > tuple(number + 1) from numbers(1); +select tuple(number) > tuple(toNullable(number + 1)) from numbers(1); +select tuple(toNullable(number)) > tuple(toNullable(number + 1)) from numbers(1); + +select tuple(toNullable(number + 1)) < tuple(number) from numbers(1); +select tuple(number + 1) < tuple(toNullable(number)) from numbers(1); +select tuple(toNullable(number + 1)) < tuple(toNullable(number + 1)) from numbers(1); + +select tuple(toNullable(number + 1)) > tuple(number) from numbers(1); +select tuple(number + 1) > tuple(toNullable(number)) from numbers(1); +select tuple(toNullable(number + 1)) > tuple(toNullable(number)) from numbers(1); + +select '- 2'; +select tuple(toNullable(number)) <= tuple(number + 1) from numbers(1); +select tuple(number) <= tuple(toNullable(number + 1)) from numbers(1); +select tuple(toNullable(number)) <= tuple(toNullable(number + 1)) from numbers(1); + +select tuple(toNullable(number)) >= tuple(number + 1) from numbers(1); +select tuple(number) > tuple(toNullable(number + 1)) from numbers(1); +select tuple(toNullable(number)) >= tuple(toNullable(number + 1)) from numbers(1); + +select tuple(toNullable(number + 1)) <= tuple(number) from numbers(1); +select tuple(number + 1) <= tuple(toNullable(number)) from numbers(1); +select tuple(toNullable(number + 1)) <= tuple(toNullable(number + 1)) from numbers(1); + +select tuple(toNullable(number + 1)) >= tuple(number) from numbers(1); +select tuple(number + 1) >= tuple(toNullable(number)) from numbers(1); +select tuple(toNullable(number + 1)) >= tuple(toNullable(number)) from numbers(1); + +select '- 3'; +select tuple(toNullable(number)) <= tuple(number) from numbers(1); +select tuple(number) <= tuple(toNullable(number)) from numbers(1); +select tuple(toNullable(number)) <= tuple(toNullable(number)) from numbers(1); + +select tuple(toNullable(number)) >= tuple(number) from numbers(1); +select tuple(number) >= tuple(toNullable(number)) from numbers(1); +select tuple(toNullable(number)) >= tuple(toNullable(number)) from numbers(1); + +select '- 4'; +select tuple(number) = tuple(materialize(toUInt64OrNull(''))) from numbers(1); +select tuple(materialize(toUInt64OrNull(''))) = tuple(materialize(toUInt64OrNull(''))) from numbers(1); +select tuple(number) <= tuple(materialize(toUInt64OrNull(''))) from numbers(1); +select tuple(materialize(toUInt64OrNull(''))) <= tuple(materialize(toUInt64OrNull(''))) from numbers(1); +select tuple(number) >= tuple(materialize(toUInt64OrNull(''))) from numbers(1); +select tuple(materialize(toUInt64OrNull(''))) >= tuple(materialize(toUInt64OrNull(''))) from numbers(1); + +select 'two arguments'; +select tuple(toNullable(number), number) = tuple(number, number) from numbers(1); +select tuple(toNullable(number), toNullable(number)) = tuple(number, number) from numbers(1); +select tuple(toNullable(number), toNullable(number)) = tuple(toNullable(number), number) from numbers(1); +select tuple(toNullable(number), toNullable(number)) = tuple(toNullable(number), toNullable(number)) from numbers(1); +select tuple(number, toNullable(number)) = tuple(toNullable(number), toNullable(number)) from numbers(1); +select tuple(number, toNullable(number)) = tuple(toNullable(number), number) from numbers(1); + +select '- 1'; +select tuple(toNullable(number), number) < tuple(number, number) from numbers(1); +select tuple(toNullable(number), toNullable(number)) < tuple(number, number) from numbers(1); +select tuple(toNullable(number), toNullable(number)) < tuple(toNullable(number), number) from numbers(1); +select tuple(toNullable(number), toNullable(number)) < tuple(toNullable(number), toNullable(number)) from numbers(1); +select tuple(number, toNullable(number)) < tuple(toNullable(number), toNullable(number)) from numbers(1); +select tuple(number, toNullable(number)) < tuple(toNullable(number), number) from numbers(1); + +select '- 2'; +select tuple(toNullable(number), number) < tuple(number, number + 1) from numbers(1); +select tuple(toNullable(number), toNullable(number)) < tuple(number, number + 1) from numbers(1); +select tuple(toNullable(number), toNullable(number)) < tuple(toNullable(number + 1), number) from numbers(1); +select tuple(toNullable(number), toNullable(number)) < tuple(toNullable(number + 1), toNullable(number)) from numbers(1); +select tuple(number, toNullable(number)) < tuple(toNullable(number), toNullable(number + 1)) from numbers(1); +select tuple(number, toNullable(number)) < tuple(toNullable(number), number + 1) from numbers(1); + +select '- 3'; +select tuple(materialize(toUInt64OrNull('')), number) = tuple(number, number) from numbers(1); +select tuple(materialize(toUInt64OrNull('')), number) = tuple(number, toUInt64OrNull('')) from numbers(1); +select tuple(materialize(toUInt64OrNull('')), toUInt64OrNull('')) = tuple(toUInt64OrNull(''), toUInt64OrNull('')) from numbers(1); +select tuple(number, materialize(toUInt64OrNull(''))) < tuple(number, number) from numbers(1); +select tuple(number, materialize(toUInt64OrNull(''))) <= tuple(number, number) from numbers(1); +select tuple(number, materialize(toUInt64OrNull(''))) < tuple(number + 1, number) from numbers(1); +select tuple(number, materialize(toUInt64OrNull(''))) > tuple(number, number) from numbers(1); +select tuple(number, materialize(toUInt64OrNull(''))) >= tuple(number, number) from numbers(1); +select tuple(number, materialize(toUInt64OrNull(''))) > tuple(number + 1, number) from numbers(1); + +select 'many arguments'; +select tuple(toNullable(number), number, number) = tuple(number, number, number) from numbers(1); +select tuple(toNullable(number), materialize('a'), number) = tuple(number, materialize('a'), number) from numbers(1); +select tuple(toNullable(number), materialize('a'), number) = tuple(number, materialize('a'), number + 1) from numbers(1); +select tuple(toNullable(number), number, number) < tuple(number, number, number) from numbers(1); +select tuple(toNullable(number), number, number) <= tuple(number, number, number) from numbers(1); +select tuple(toNullable(number), materialize('a'), number) < tuple(number, materialize('a'), number) from numbers(1); +select tuple(toNullable(number), materialize('a'), number) < tuple(number, materialize('a'), number + 1) from numbers(1); +select tuple(toNullable(number), number, materialize(toUInt64OrNull(''))) = tuple(number, number, number) from numbers(1); +select tuple(toNullable(number), materialize('a'), materialize(toUInt64OrNull(''))) = tuple(number, materialize('a'), number) from numbers(1); +select tuple(toNullable(number), materialize('a'), materialize(toUInt64OrNull(''))) = tuple(number, materialize('a'), number + 1) from numbers(1); +select tuple(toNullable(number), number, materialize(toUInt64OrNull(''))) <= tuple(number, number, number) from numbers(1); +select tuple(toNullable(number), materialize('a'), materialize(toUInt64OrNull(''))) <= tuple(number, materialize('a'), number) from numbers(1); +select tuple(toNullable(number), materialize('a'), materialize(toUInt64OrNull(''))) <= tuple(number, materialize('a'), number + 1) from numbers(1); diff --git a/tests/queries/0_stateless/01353_topk_enum.reference b/tests/queries/0_stateless/01353_topk_enum.reference new file mode 100644 index 000000000000..d650850c4342 --- /dev/null +++ b/tests/queries/0_stateless/01353_topk_enum.reference @@ -0,0 +1 @@ +['test','world','hello',''] diff --git a/tests/queries/0_stateless/01353_topk_enum.sql b/tests/queries/0_stateless/01353_topk_enum.sql new file mode 100644 index 000000000000..ba048401b237 --- /dev/null +++ b/tests/queries/0_stateless/01353_topk_enum.sql @@ -0,0 +1 @@ +WITH CAST(round(sqrt(number)) % 4 AS Enum('' = 0, 'hello' = 1, 'world' = 2, 'test' = 3)) AS x SELECT topK(10)(x) FROM numbers(1000); diff --git a/tests/queries/0_stateless/01354_tuple_low_cardinality_array_mapped_bug.reference b/tests/queries/0_stateless/01354_tuple_low_cardinality_array_mapped_bug.reference new file mode 100644 index 000000000000..aa47d0d46d47 --- /dev/null +++ b/tests/queries/0_stateless/01354_tuple_low_cardinality_array_mapped_bug.reference @@ -0,0 +1,2 @@ +0 +0 diff --git a/tests/queries/0_stateless/01354_tuple_low_cardinality_array_mapped_bug.sql b/tests/queries/0_stateless/01354_tuple_low_cardinality_array_mapped_bug.sql new file mode 100644 index 000000000000..80a1a7c46ebf --- /dev/null +++ b/tests/queries/0_stateless/01354_tuple_low_cardinality_array_mapped_bug.sql @@ -0,0 +1,8 @@ +SELECT arrayExists(x -> ((x.1) = 'pattern'), cast([tuple('a', 1)] as Array(Tuple(LowCardinality(String), UInt8)))); + +DROP TABLE IF EXISTS table; +CREATE TABLE table (id Int32, values Array(Tuple(LowCardinality(String), Int32)), date Date) ENGINE MergeTree() PARTITION BY toYYYYMM(date) ORDER BY (id, date); + +SELECT count(*) FROM table WHERE (arrayExists(x -> ((x.1) = toLowCardinality('pattern')), values) = 1); + +DROP TABLE IF EXISTS table; diff --git a/tests/queries/0_stateless/01355_defaultValueOfArgumentType_bug.reference b/tests/queries/0_stateless/01355_defaultValueOfArgumentType_bug.reference new file mode 100644 index 000000000000..4165503c4b59 --- /dev/null +++ b/tests/queries/0_stateless/01355_defaultValueOfArgumentType_bug.reference @@ -0,0 +1 @@ + LowCardinality(String) diff --git a/tests/queries/0_stateless/01355_defaultValueOfArgumentType_bug.sql b/tests/queries/0_stateless/01355_defaultValueOfArgumentType_bug.sql new file mode 100644 index 000000000000..e3168eb09a06 --- /dev/null +++ b/tests/queries/0_stateless/01355_defaultValueOfArgumentType_bug.sql @@ -0,0 +1,4 @@ +SELECT + materialize(toLowCardinality('')) AS lc, + toTypeName(lc) +WHERE lc = defaultValueOfArgumentType(lc) diff --git a/tests/queries/0_stateless/01355_if_fixed_string.reference b/tests/queries/0_stateless/01355_if_fixed_string.reference new file mode 100644 index 000000000000..43c8af518b4c --- /dev/null +++ b/tests/queries/0_stateless/01355_if_fixed_string.reference @@ -0,0 +1,40 @@ +0\0\0\0\0 String +1\0 String +-2\0\0\0 String +3\0 String +-4\0\0\0 String +5\0 String +-6\0\0\0 String +7\0 String +-8\0\0\0 String +9\0 String +0\0 FixedString(2) +1\0 FixedString(2) +-2 FixedString(2) +3\0 FixedString(2) +-4 FixedString(2) +5\0 FixedString(2) +-6 FixedString(2) +7\0 FixedString(2) +-8 FixedString(2) +9\0 FixedString(2) +0 String +1 String +-2 String +3 String +-4 String +5 String +-6 String +7 String +-8 String +9 String +0\0 FixedString(2) +1\0 FixedString(2) +-2 FixedString(2) +3\0 FixedString(2) +-4 FixedString(2) +5\0 FixedString(2) +-6 FixedString(2) +7\0 FixedString(2) +-8 FixedString(2) +9\0 FixedString(2) diff --git a/tests/queries/0_stateless/01355_if_fixed_string.sql b/tests/queries/0_stateless/01355_if_fixed_string.sql new file mode 100644 index 000000000000..a0afcc5f1972 --- /dev/null +++ b/tests/queries/0_stateless/01355_if_fixed_string.sql @@ -0,0 +1,5 @@ +SELECT if(number % 2, toFixedString(toString(number), 2), toFixedString(toString(-number), 5)) AS x, toTypeName(x) FROM system.numbers LIMIT 10; +SELECT if(number % 2, toFixedString(toString(number), 2), toFixedString(toString(-number), 2)) AS x, toTypeName(x) FROM system.numbers LIMIT 10; + +SELECT multiIf(number % 2, toFixedString(toString(number), 2), toFixedString(toString(-number), 5)) AS x, toTypeName(x) FROM system.numbers LIMIT 10; +SELECT multiIf(number % 2, toFixedString(toString(number), 2), toFixedString(toString(-number), 2)) AS x, toTypeName(x) FROM system.numbers LIMIT 10; diff --git a/tests/queries/0_stateless/01356_state_resample.reference b/tests/queries/0_stateless/01356_state_resample.reference new file mode 100644 index 000000000000..40c606b4a68f --- /dev/null +++ b/tests/queries/0_stateless/01356_state_resample.reference @@ -0,0 +1,8 @@ +[900,910,920,930,940,950,960,970,980,990,1000,1010,1020,1030,1040,1050,1060,1070,1080,1090] +[900,910,920,930,940,950,960,970,980,990,1000,1010,1020,1030,1040,1050,1060,1070,1080,1090] +[360,243,306,372,252,315,384,261,324,396,270,333,408,279,342,420,288,351,432,297] +[300,364,246,309,376,255,318,388,264,327,400,273,336,412,282,345,424,291,354,436] +[240,303,368,249,312,380,258,321,392,267,330,404,276,339,416,285,348,428,294,357] +[[0,20,40],[1,21,41],[2,22,42],[3,23,43],[4,24,44],[5,25,45],[6,26,46],[7,27,47],[8,28,48],[9,29,49],[10,30],[11,31],[12,32],[13,33],[14,34],[15,35],[16,36],[17,37],[18,38],[19,39]] +[[0,20,40],[1,21,41],[2,22,42],[3,23,43],[4,24,44],[5,25,45],[6,26,46],[7,27,47],[8,28,48],[9,29,49],[10,30],[11,31],[12,32],[13,33],[14,34],[15,35],[16,36],[17,37],[18,38],[19,39]] +[1800,1820,1840,1860,1880,1900,1920,1940,1960,1980,2000,2020,2040,2060,2080,2100,2120,2140,2160,2180] diff --git a/tests/queries/0_stateless/01356_state_resample.sql b/tests/queries/0_stateless/01356_state_resample.sql new file mode 100644 index 000000000000..6be28e19d87b --- /dev/null +++ b/tests/queries/0_stateless/01356_state_resample.sql @@ -0,0 +1,14 @@ +select sumResample(0, 20, 1)(number, number % 20) from numbers(200); +select arrayMap(x -> finalizeAggregation(x), state) from (select sumStateResample(0, 20, 1)(number, number % 20) as state from numbers(200)); +select arrayMap(x -> finalizeAggregation(x), state) from +( + select sumStateResample(0,20,1)(number, number%20) as state from numbers(200) group by number % 3 +); + +select groupArrayResample(0, 20, 1)(number, number % 20) from numbers(50); +select arrayMap(x -> finalizeAggregation(x), state) from (select groupArrayStateResample(0, 20, 1)(number, number % 20) state from numbers(50)); + +select arrayMap(x -> finalizeAggregation(x), state) from +( + select sumStateResample(0, 20, 1)(number, number % 20) as state from remote('127.0.0.{1,2}', numbers(200)) +); diff --git a/tests/queries/0_stateless/01356_view_threads.reference b/tests/queries/0_stateless/01356_view_threads.reference new file mode 100644 index 000000000000..4e9079198d52 --- /dev/null +++ b/tests/queries/0_stateless/01356_view_threads.reference @@ -0,0 +1,3 @@ +0 249999500000 +1 250000000000 +1 diff --git a/tests/queries/0_stateless/01356_view_threads.sql b/tests/queries/0_stateless/01356_view_threads.sql new file mode 100644 index 000000000000..5290ec555aff --- /dev/null +++ b/tests/queries/0_stateless/01356_view_threads.sql @@ -0,0 +1,12 @@ +drop table if exists table_01356_view_threads; + +create view table_01356_view_threads as select number % 10 as g, sum(number) as s from numbers_mt(1000000) group by g; + +set log_queries = 1; +set max_threads = 16; +select g % 2 as gg, sum(s) from table_01356_view_threads group by gg order by gg; + +system flush logs; +select length(thread_ids) >= 16 from system.query_log where event_date >= today() - 1 and lower(query) like '%select g % 2 as gg, sum(s) from table_01356_view_threads group by gg order by gg%' and type = 'QueryFinish' order by query_start_time desc limit 1; + +drop table if exists table_01356_view_threads; diff --git a/tests/queries/0_stateless/01356_wrong_filter-type_bug.reference b/tests/queries/0_stateless/01356_wrong_filter-type_bug.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01356_wrong_filter-type_bug.sql b/tests/queries/0_stateless/01356_wrong_filter-type_bug.sql new file mode 100644 index 000000000000..b3f48967ba2c --- /dev/null +++ b/tests/queries/0_stateless/01356_wrong_filter-type_bug.sql @@ -0,0 +1,9 @@ +drop table if exists t0; + +CREATE TABLE t0 (`c0` String, `c1` Int32 CODEC(NONE), `c2` Int32) ENGINE = MergeTree() ORDER BY tuple(); +insert into t0 values ('a', 1, 2); + +SELECT t0.c2, t0.c1, t0.c0 FROM t0 PREWHERE t0.c0 ORDER BY ((t0.c2)>=(t0.c1)), (((- (((t0.c0)>(t0.c0))))) IS NULL) FORMAT TabSeparatedWithNamesAndTypes; -- {serverError 59} +SELECT t0.c2, t0.c1, t0.c0 FROM t0 WHERE t0.c0 ORDER BY ((t0.c2)>=(t0.c1)), (((- (((t0.c0)>(t0.c0))))) IS NULL) FORMAT TabSeparatedWithNamesAndTypes settings optimize_move_to_prewhere=0; -- {serverError 59} + +drop table if exists t0; diff --git a/tests/queries/0_stateless/01358_mutation_delete_null_rows.reference b/tests/queries/0_stateless/01358_mutation_delete_null_rows.reference new file mode 100644 index 000000000000..126874237b7e --- /dev/null +++ b/tests/queries/0_stateless/01358_mutation_delete_null_rows.reference @@ -0,0 +1,20 @@ +-------- +0 1 x=0 +1 0 x<>0 +3 0 x<>0 +\N \N x<>0 +-------- +2020-01-01 2 0 leave +2020-01-02 aaa 0 1 delete +2020-01-03 2 0 leave +2020-01-04 2 0 leave +2020-01-05 \N 2 0 leave +2020-01-06 aaa 0 1 delete +2020-01-07 aaa 0 1 delete +2020-01-08 aaa \N \N leave +-------- +2020-01-01 2 +2020-01-03 2 +2020-01-04 2 +2020-01-05 \N 2 +2020-01-08 aaa \N diff --git a/tests/queries/0_stateless/01358_mutation_delete_null_rows.sql b/tests/queries/0_stateless/01358_mutation_delete_null_rows.sql new file mode 100644 index 000000000000..e8aabf1aa375 --- /dev/null +++ b/tests/queries/0_stateless/01358_mutation_delete_null_rows.sql @@ -0,0 +1,26 @@ +select '--------'; +SELECT arrayJoin([0, 1, 3, NULL]) AS x, x = 0, if(x = 0, 'x=0', 'x<>0') ORDER BY x; + +select '--------'; +drop table if exists mutation_delete_null_rows; + +CREATE TABLE mutation_delete_null_rows +( + `EventDate` Date, + `CounterID` Nullable(String), + `UserID` Nullable(UInt32) +) +ENGINE = MergeTree() +ORDER BY EventDate; + +INSERT INTO mutation_delete_null_rows VALUES ('2020-01-01', '', 2)('2020-01-02', 'aaa', 0); +INSERT INTO mutation_delete_null_rows VALUES ('2020-01-03', '', 2)('2020-01-04', '', 2)('2020-01-05', NULL, 2)('2020-01-06', 'aaa', 0)('2020-01-07', 'aaa', 0)('2020-01-08', 'aaa', NULL); + +SELECT *,UserID = 0 as UserIDEquals0, if(UserID = 0, 'delete', 'leave') as verdict FROM mutation_delete_null_rows ORDER BY EventDate; + +ALTER TABLE mutation_delete_null_rows DELETE WHERE UserID = 0 SETTINGS mutations_sync=1; + +select '--------'; +SELECT * FROM mutation_delete_null_rows ORDER BY EventDate; + +drop table mutation_delete_null_rows; diff --git a/tests/queries/0_stateless/01358_union_threads_bug.reference b/tests/queries/0_stateless/01358_union_threads_bug.reference new file mode 100644 index 000000000000..4397f4e2fdd5 --- /dev/null +++ b/tests/queries/0_stateless/01358_union_threads_bug.reference @@ -0,0 +1,2 @@ +300 +1 diff --git a/tests/queries/0_stateless/01358_union_threads_bug.sql b/tests/queries/0_stateless/01358_union_threads_bug.sql new file mode 100644 index 000000000000..0c07364f41ca --- /dev/null +++ b/tests/queries/0_stateless/01358_union_threads_bug.sql @@ -0,0 +1,7 @@ +set log_queries = 1; +set max_threads = 16; + +SELECT count() FROM (SELECT number FROM numbers_mt(1000000) ORDER BY number DESC LIMIT 100 UNION ALL SELECT number FROM numbers_mt(1000000) ORDER BY number DESC LIMIT 100 UNION ALL SELECT number FROM numbers_mt(1000000) ORDER BY number DESC LIMIT 100); + +system flush logs; +select length(thread_ids) >= 16 from system.query_log where event_date >= today() - 1 and query like '%SELECT count() FROM (SELECT number FROM numbers_mt(1000000) ORDER BY number DESC LIMIT 100 UNION ALL SELECT number FROM numbers_mt(1000000) ORDER BY number DESC LIMIT 100 UNION ALL SELECT number FROM numbers_mt(1000000) ORDER BY number DESC LIMIT 100)%' and type = 'QueryFinish' order by query_start_time desc limit 1; diff --git a/tests/queries/0_stateless/01359_geodistance_loop.reference b/tests/queries/0_stateless/01359_geodistance_loop.reference new file mode 100644 index 000000000000..8484d062f57d --- /dev/null +++ b/tests/queries/0_stateless/01359_geodistance_loop.reference @@ -0,0 +1 @@ +inf diff --git a/tests/queries/0_stateless/01359_geodistance_loop.sql b/tests/queries/0_stateless/01359_geodistance_loop.sql new file mode 100644 index 000000000000..4c555a2538e6 --- /dev/null +++ b/tests/queries/0_stateless/01359_geodistance_loop.sql @@ -0,0 +1 @@ +SELECT geoDistance(0., 0., -inf, 1.); diff --git a/tests/queries/0_stateless/01360_division_overflow.reference b/tests/queries/0_stateless/01360_division_overflow.reference new file mode 100644 index 000000000000..a4acafeee604 --- /dev/null +++ b/tests/queries/0_stateless/01360_division_overflow.reference @@ -0,0 +1,6 @@ +0 +0 +0 +1 +3 +5 diff --git a/tests/queries/0_stateless/01360_division_overflow.sql b/tests/queries/0_stateless/01360_division_overflow.sql new file mode 100644 index 000000000000..75601a365366 --- /dev/null +++ b/tests/queries/0_stateless/01360_division_overflow.sql @@ -0,0 +1,5 @@ +select intDiv(materialize(toInt32(1)), 0x100000000); +select intDiv(materialize(toInt32(1)), -0x100000000); +select intDiv(materialize(toInt32(1)), -9223372036854775808); +select materialize(toInt32(1)) % -9223372036854775808; +select value % -9223372036854775808 from (select toInt32(arrayJoin([3, 5])) value); diff --git a/tests/queries/0_stateless/01361_buffer_table_flush_with_materialized_view.reference b/tests/queries/0_stateless/01361_buffer_table_flush_with_materialized_view.reference new file mode 100644 index 000000000000..083edaac2489 --- /dev/null +++ b/tests/queries/0_stateless/01361_buffer_table_flush_with_materialized_view.reference @@ -0,0 +1,3 @@ +2 +2 +2 diff --git a/tests/queries/0_stateless/01361_buffer_table_flush_with_materialized_view.sql b/tests/queries/0_stateless/01361_buffer_table_flush_with_materialized_view.sql new file mode 100644 index 000000000000..424c38d5590b --- /dev/null +++ b/tests/queries/0_stateless/01361_buffer_table_flush_with_materialized_view.sql @@ -0,0 +1,37 @@ +DROP TABLE IF EXISTS t1_01361; +DROP TABLE IF EXISTS t2_01361; +DROP TABLE IF EXISTS mv1_01361; +DROP TABLE IF EXISTS b1_01361; + +CREATE TABLE t1_01361 ( + i UInt32, + time DateTime +) ENGINE = MergeTree() +PARTITION BY time +ORDER BY time; + +CREATE TABLE t2_01361 ( + i UInt32, + time DateTime +) ENGINE = MergeTree() +PARTITION BY time +ORDER BY time; + +CREATE MATERIALIZED VIEW mv1_01361 +TO t2_01361 +AS SELECT * FROM (SELECT * FROM t1_01361); + +CREATE TABLE b1_01361 AS t1_01361 +ENGINE = Buffer(currentDatabase(), t1_01361, 1, 0, 0, 1, 1, 1, 1); + +INSERT INTO b1_01361 VALUES (1, now()); +INSERT INTO b1_01361 VALUES (2, now()); + +SELECT count() FROM b1_01361; +SELECT count() FROM t1_01361; +SELECT count() FROM t2_01361; + +DROP TABLE IF EXISTS t1_01361; +DROP TABLE IF EXISTS t2_01361; +DROP TABLE IF EXISTS mv1_01361; +DROP TABLE IF EXISTS b1_01361; diff --git a/tests/queries/0_stateless/01373_is_zero_or_null.reference b/tests/queries/0_stateless/01373_is_zero_or_null.reference new file mode 100644 index 000000000000..d9caaa2089ad --- /dev/null +++ b/tests/queries/0_stateless/01373_is_zero_or_null.reference @@ -0,0 +1,22 @@ +1 1 +1 1 +0 0 +\N 1 +--- +1 1 +1 1 +0 0 +--- +hello +world +--- +hello +world +--- +hello +world +\N +--- +3 +--- +4 diff --git a/tests/queries/0_stateless/01373_is_zero_or_null.sql b/tests/queries/0_stateless/01373_is_zero_or_null.sql new file mode 100644 index 000000000000..d83b4f58fffe --- /dev/null +++ b/tests/queries/0_stateless/01373_is_zero_or_null.sql @@ -0,0 +1,29 @@ +SELECT NOT x, isZeroOrNull(x) FROM (SELECT arrayJoin([1, 2, 3, NULL]) = 3 AS x); +SELECT '---'; +SELECT NOT x, isZeroOrNull(x) FROM (SELECT arrayJoin([1, 2, 3]) = 3 AS x); +SELECT '---'; +CREATE TEMPORARY TABLE test (x Nullable(String)); +INSERT INTO test VALUES ('hello'), ('world'), ('xyz'), (NULL); + +SELECT * FROM test WHERE x != 'xyz'; +SELECT '---'; +SELECT * FROM test WHERE NOT x = 'xyz'; +SELECT '---'; +SELECT * FROM test WHERE isZeroOrNull(x = 'xyz'); +SELECT '---'; + +SELECT count() FROM +( + SELECT * FROM test WHERE x != 'xyz' + UNION ALL + SELECT * FROM test WHERE NOT x != 'xyz' +); + +SELECT '---'; + +SELECT count() FROM +( + SELECT * FROM test WHERE x != 'xyz' + UNION ALL + SELECT * FROM test WHERE isZeroOrNull(x != 'xyz') +); diff --git a/tests/queries/0_stateless/01374_if_nullable_filimonov.reference b/tests/queries/0_stateless/01374_if_nullable_filimonov.reference new file mode 100644 index 000000000000..ebe52278bb31 --- /dev/null +++ b/tests/queries/0_stateless/01374_if_nullable_filimonov.reference @@ -0,0 +1,8 @@ +2 0 leave +0 1 delete +\N \N leave +--- +0 1 Definitely x = 0 +1 0 We cannot say that x = 0 +3 0 We cannot say that x = 0 +\N \N We cannot say that x = 0 diff --git a/tests/queries/0_stateless/01374_if_nullable_filimonov.sql b/tests/queries/0_stateless/01374_if_nullable_filimonov.sql new file mode 100644 index 000000000000..0fadfb85fe4b --- /dev/null +++ b/tests/queries/0_stateless/01374_if_nullable_filimonov.sql @@ -0,0 +1,9 @@ +SELECT + UserID, + UserID = 0, + if(UserID = 0, 'delete', 'leave') +FROM VALUES('UserID Nullable(UInt8)', (2), (0), (NULL)); + +SELECT '---'; + +SELECT arrayJoin([0, 1, 3, NULL]) AS x, x = 0, if(x = 0, 'Definitely x = 0', 'We cannot say that x = 0'); diff --git a/tests/queries/0_stateless/01375_GROUP_BY_injective_elimination_dictGet_BAD_ARGUMENTS.reference b/tests/queries/0_stateless/01375_GROUP_BY_injective_elimination_dictGet_BAD_ARGUMENTS.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01375_GROUP_BY_injective_elimination_dictGet_BAD_ARGUMENTS.sql b/tests/queries/0_stateless/01375_GROUP_BY_injective_elimination_dictGet_BAD_ARGUMENTS.sql new file mode 100644 index 000000000000..88a2b25c2dba --- /dev/null +++ b/tests/queries/0_stateless/01375_GROUP_BY_injective_elimination_dictGet_BAD_ARGUMENTS.sql @@ -0,0 +1 @@ +SELECT dictGetString(concat('default', '.countryId'), 'country', toUInt64(number)) AS country FROM numbers(2) GROUP BY country; -- { serverError 36; } diff --git a/tests/queries/0_stateless/01375_storage_file_write_prefix.reference b/tests/queries/0_stateless/01375_storage_file_write_prefix.reference new file mode 100644 index 000000000000..ed9a18b93463 --- /dev/null +++ b/tests/queries/0_stateless/01375_storage_file_write_prefix.reference @@ -0,0 +1,30 @@ +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 diff --git a/tests/queries/0_stateless/01375_storage_file_write_prefix.sql b/tests/queries/0_stateless/01375_storage_file_write_prefix.sql new file mode 100644 index 000000000000..b335db03f6c6 --- /dev/null +++ b/tests/queries/0_stateless/01375_storage_file_write_prefix.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS tmp_01375; +DROP TABLE IF EXISTS table_csv_01375; + +CREATE TABLE tmp_01375 (n UInt32, s String) ENGINE = Memory; +CREATE TABLE table_csv_01375 AS tmp_01375 ENGINE = File(CSVWithNames); + +INSERT INTO table_csv_01375 SELECT number as n, toString(n) as s FROM numbers(10); +INSERT INTO table_csv_01375 SELECT number as n, toString(n) as s FROM numbers(10); +INSERT INTO table_csv_01375 SELECT number as n, toString(n) as s FROM numbers(10); + +SELECT * FROM table_csv_01375; + +DROP TABLE IF EXISTS tmp_01375; +DROP TABLE IF EXISTS table_csv_01375; diff --git a/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.reference b/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.reference new file mode 100644 index 000000000000..9459d4ba2a0d --- /dev/null +++ b/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.reference @@ -0,0 +1 @@ +1.1 diff --git a/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.sql b/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.sql new file mode 100644 index 000000000000..1c7a4d16f054 --- /dev/null +++ b/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.sql @@ -0,0 +1,31 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/11469 +SELECT dictGet('default.countryId', 'country', toUInt64(number)) AS country FROM numbers(2) GROUP BY country; -- { serverError 36; } + + +-- with real dictionary +DROP TABLE IF EXISTS dictdb_01376.table_for_dict; +DROP DICTIONARY IF EXISTS dictdb_01376.dict_exists; +DROP DATABASE IF EXISTS dictdb_01376; + +CREATE DATABASE dictdb_01376 ENGINE = Ordinary; + +CREATE TABLE dictdb_01376.table_for_dict +( + key_column UInt64, + value Float64 +) +ENGINE = Memory(); + +INSERT INTO dictdb_01376.table_for_dict VALUES (1, 1.1); + +CREATE DICTIONARY IF NOT EXISTS dictdb_01376.dict_exists +( + key_column UInt64, + value Float64 DEFAULT 77.77 +) +PRIMARY KEY key_column +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' DB 'dictdb_01376')) +LIFETIME(1) +LAYOUT(FLAT()); + +SELECT dictGet('dictdb_01376.dict_exists', 'value', toUInt64(1)) as val FROM numbers(2) GROUP BY val; diff --git a/tests/queries/0_stateless/01376_array_fill_empty.reference b/tests/queries/0_stateless/01376_array_fill_empty.reference new file mode 100644 index 000000000000..70e8ac460741 --- /dev/null +++ b/tests/queries/0_stateless/01376_array_fill_empty.reference @@ -0,0 +1,4 @@ +[] +[] +[] +[] diff --git a/tests/queries/0_stateless/01376_array_fill_empty.sql b/tests/queries/0_stateless/01376_array_fill_empty.sql new file mode 100644 index 000000000000..38eea8a51c10 --- /dev/null +++ b/tests/queries/0_stateless/01376_array_fill_empty.sql @@ -0,0 +1,4 @@ +SELECT arrayFill(x -> (x < 10), []); +SELECT arrayFill(x -> (x < 10), emptyArrayUInt8()); +SELECT arrayFill(x -> 1, []); +SELECT arrayFill(x -> 0, []); diff --git a/tests/queries/0_stateless/01376_null_logical.reference b/tests/queries/0_stateless/01376_null_logical.reference new file mode 100644 index 000000000000..4b849dd057d7 --- /dev/null +++ b/tests/queries/0_stateless/01376_null_logical.reference @@ -0,0 +1,8 @@ +1 +1 +0 +0 +\N +\N +\N +\N diff --git a/tests/queries/0_stateless/01376_null_logical.sql b/tests/queries/0_stateless/01376_null_logical.sql new file mode 100644 index 000000000000..d8c7a3224457 --- /dev/null +++ b/tests/queries/0_stateless/01376_null_logical.sql @@ -0,0 +1,11 @@ +SELECT NULL OR 1; +SELECT materialize(NULL) OR materialize(1); + +SELECT NULL AND 0; +SELECT materialize(NULL) AND materialize(0); + +SELECT NULL OR 0; +SELECT materialize(NULL) OR materialize(0); + +SELECT NULL AND 1; +SELECT materialize(NULL) AND materialize(1); diff --git a/tests/queries/0_stateless/01377_supertype_low_cardinality.reference b/tests/queries/0_stateless/01377_supertype_low_cardinality.reference new file mode 100644 index 000000000000..89c8606dd33c --- /dev/null +++ b/tests/queries/0_stateless/01377_supertype_low_cardinality.reference @@ -0,0 +1,71 @@ +hello +hello +String +String +--- +--- +hello +hello +hello +hello +--- +hello +hello +- +hello +hello +- +hello +hello +- +hello +hello +- +hello +hello +- +hello +hello +- +hello +hello +- +hello +hello +- +hello +hello +- +hello +hello +- +hello +hello +- +hello +hello +--- +hello +hello +hello +- +hello +hello +hello +- +hello +hello +hello +- +hello +hello +hello +--- +hello +hello +hello +hello +--- +['abc','def'] +['abc','def'] +['abc','def'] diff --git a/tests/queries/0_stateless/01377_supertype_low_cardinality.sql b/tests/queries/0_stateless/01377_supertype_low_cardinality.sql new file mode 100644 index 000000000000..9e7ed1f98b2c --- /dev/null +++ b/tests/queries/0_stateless/01377_supertype_low_cardinality.sql @@ -0,0 +1,71 @@ +SELECT 'hello' UNION ALL SELECT toLowCardinality('hello'); +SELECT toTypeName(x) FROM (SELECT 'hello' AS x UNION ALL SELECT toLowCardinality('hello')); + +SELECT '---'; + +create temporary table t1(a String); +create temporary table t2(a LowCardinality(String)); +select a from t1 union all select a from t2; + +SELECT '---'; + +CREATE TEMPORARY TABLE a (x String); +CREATE TEMPORARY TABLE b (x LowCardinality(String)); +CREATE TEMPORARY TABLE c (x Nullable(String)); +CREATE TEMPORARY TABLE d (x LowCardinality(Nullable(String))); + +INSERT INTO a VALUES ('hello'); +INSERT INTO b VALUES ('hello'); +INSERT INTO c VALUES ('hello'); +INSERT INTO d VALUES ('hello'); + +SELECT x FROM a; +SELECT x FROM b; +SELECT x FROM c; +SELECT x FROM d; + +SELECT '---'; + +SELECT x FROM a UNION ALL SELECT x FROM b; +SELECT '-'; +SELECT x FROM a UNION ALL SELECT x FROM c; +SELECT '-'; +SELECT x FROM a UNION ALL SELECT x FROM d; +SELECT '-'; +SELECT x FROM b UNION ALL SELECT x FROM a; +SELECT '-'; +SELECT x FROM b UNION ALL SELECT x FROM c; +SELECT '-'; +SELECT x FROM b UNION ALL SELECT x FROM d; +SELECT '-'; +SELECT x FROM c UNION ALL SELECT x FROM a; +SELECT '-'; +SELECT x FROM c UNION ALL SELECT x FROM b; +SELECT '-'; +SELECT x FROM c UNION ALL SELECT x FROM d; +SELECT '-'; +SELECT x FROM d UNION ALL SELECT x FROM a; +SELECT '-'; +SELECT x FROM d UNION ALL SELECT x FROM c; +SELECT '-'; +SELECT x FROM d UNION ALL SELECT x FROM b; + +SELECT '---'; + +SELECT x FROM b UNION ALL SELECT x FROM c UNION ALL SELECT x FROM d; +SELECT '-'; +SELECT x FROM a UNION ALL SELECT x FROM c UNION ALL SELECT x FROM d; +SELECT '-'; +SELECT x FROM a UNION ALL SELECT x FROM b UNION ALL SELECT x FROM d; +SELECT '-'; +SELECT x FROM a UNION ALL SELECT x FROM b UNION ALL SELECT x FROM c; + +SELECT '---'; + +SELECT x FROM a UNION ALL SELECT x FROM b UNION ALL SELECT x FROM c UNION ALL SELECT x FROM d; + +SELECT '---'; + +SELECT [CAST('abc' AS LowCardinality(String)), CAST('def' AS Nullable(String))]; +SELECT [CAST('abc' AS LowCardinality(String)), CAST('def' AS FixedString(3))]; +SELECT [CAST('abc' AS LowCardinality(String)), CAST('def' AS LowCardinality(FixedString(3)))]; diff --git a/tests/queries/0_stateless/01379_with_fill_several_columns.reference b/tests/queries/0_stateless/01379_with_fill_several_columns.reference new file mode 100644 index 000000000000..f3362a47aab0 --- /dev/null +++ b/tests/queries/0_stateless/01379_with_fill_several_columns.reference @@ -0,0 +1,21 @@ +1970-01-11 1970-01-02 original +0000-00-00 1970-01-03 +0000-00-00 1970-01-04 +1970-02-10 1970-01-05 original +0000-00-00 1970-01-06 +0000-00-00 1970-01-07 +1970-03-12 1970-01-08 original +=============== +1970-01-11 1970-01-02 original +1970-01-16 0000-00-00 +1970-01-21 0000-00-00 +1970-01-26 0000-00-00 +1970-01-31 0000-00-00 +1970-02-05 0000-00-00 +1970-02-10 1970-01-05 original +1970-02-15 0000-00-00 +1970-02-20 0000-00-00 +1970-02-25 0000-00-00 +1970-03-02 0000-00-00 +1970-03-07 0000-00-00 +1970-03-12 1970-01-08 original diff --git a/tests/queries/0_stateless/01379_with_fill_several_columns.sql b/tests/queries/0_stateless/01379_with_fill_several_columns.sql new file mode 100644 index 000000000000..5d1cb4e68281 --- /dev/null +++ b/tests/queries/0_stateless/01379_with_fill_several_columns.sql @@ -0,0 +1,21 @@ +SELECT + toDate((number * 10) * 86400) AS d1, + toDate(number * 86400) AS d2, + 'original' AS source +FROM numbers(10) +WHERE (number % 3) = 1 +ORDER BY + d2 WITH FILL, + d1 WITH FILL STEP 5; + +SELECT '==============='; + +SELECT + toDate((number * 10) * 86400) AS d1, + toDate(number * 86400) AS d2, + 'original' AS source +FROM numbers(10) +WHERE (number % 3) = 1 +ORDER BY + d1 WITH FILL STEP 5, + d2 WITH FILL; \ No newline at end of file diff --git a/tests/queries/0_stateless/01389_filter_by_virtual_columns.reference b/tests/queries/0_stateless/01389_filter_by_virtual_columns.reference new file mode 100644 index 000000000000..573541ac9702 --- /dev/null +++ b/tests/queries/0_stateless/01389_filter_by_virtual_columns.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/01389_filter_by_virtual_columns.sql b/tests/queries/0_stateless/01389_filter_by_virtual_columns.sql new file mode 100644 index 000000000000..43ce3ad40e5f --- /dev/null +++ b/tests/queries/0_stateless/01389_filter_by_virtual_columns.sql @@ -0,0 +1,2 @@ +SELECT count() FROM system.parts WHERE table = NULL AND database = currentDatabase(); +SELECT DISTINCT marks FROM system.parts WHERE (table = NULL) AND (database = currentDatabase()) AND (active = 1); diff --git a/tests/queries/0_stateless/01397_in_bad_arguments.reference b/tests/queries/0_stateless/01397_in_bad_arguments.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01397_in_bad_arguments.sql b/tests/queries/0_stateless/01397_in_bad_arguments.sql new file mode 100644 index 000000000000..4854abad091e --- /dev/null +++ b/tests/queries/0_stateless/01397_in_bad_arguments.sql @@ -0,0 +1,4 @@ +select in((1, 1, 1, 1)); -- { serverError 42 } +select in(1); -- { serverError 42 } +select in(); -- { serverError 42 } +select in(1, 2, 3); -- { serverError 42 } diff --git a/tests/queries/0_stateless/01412_cache_dictionary_race.reference b/tests/queries/0_stateless/01412_cache_dictionary_race.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/01412_cache_dictionary_race.sh b/tests/queries/0_stateless/01412_cache_dictionary_race.sh new file mode 100755 index 000000000000..f94b997f829f --- /dev/null +++ b/tests/queries/0_stateless/01412_cache_dictionary_race.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ordinary_db" + +$CLICKHOUSE_CLIENT --query "CREATE DATABASE ordinary_db" + +$CLICKHOUSE_CLIENT -n -q " + +CREATE DICTIONARY ordinary_db.dict1 +( + key_column UInt64 DEFAULT 0, + second_column UInt8 DEFAULT 1, + third_column String DEFAULT 'qqq' +) +PRIMARY KEY key_column +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'view_for_dict' PASSWORD '' DB 'ordinary_db')) +LIFETIME(MIN 1 MAX 3) +LAYOUT(CACHE(SIZE_IN_CELLS 3)); +" + +function dict_get_thread() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null + done +} + + +function drop_create_table_thread() +{ + while true; do + $CLICKHOUSE_CLIENT -n --query "CREATE TABLE ordinary_db.table_for_dict_real ( + key_column UInt64, + second_column UInt8, + third_column String + ) + ENGINE MergeTree() ORDER BY tuple(); + INSERT INTO ordinary_db.table_for_dict_real SELECT number, number, toString(number) from numbers(2); + CREATE VIEW ordinary_db.view_for_dict AS SELECT key_column, second_column, third_column from ordinary_db.table_for_dict_real WHERE sleepEachRow(1) == 0; +" + sleep 10 + + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS ordinary_db.table_for_dict_real" + sleep 10 + done +} + +export -f dict_get_thread; +export -f drop_create_table_thread; + +TIMEOUT=30 + +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & + + +timeout $TIMEOUT bash -c drop_create_table_thread 2> /dev/null & + +wait + + + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ordinary_db" diff --git a/tests/queries/0_stateless/disabled_01360_materialized_view_with_join_on_query_log.reference b/tests/queries/0_stateless/disabled_01360_materialized_view_with_join_on_query_log.reference new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/tests/queries/0_stateless/disabled_01360_materialized_view_with_join_on_query_log.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/disabled_01360_materialized_view_with_join_on_query_log.sql b/tests/queries/0_stateless/disabled_01360_materialized_view_with_join_on_query_log.sql new file mode 100644 index 000000000000..96932dac65f3 --- /dev/null +++ b/tests/queries/0_stateless/disabled_01360_materialized_view_with_join_on_query_log.sql @@ -0,0 +1,56 @@ +-- To work correctly this test require improvement from https://github.com/ClickHouse/ClickHouse/pull/11686 which we won't backport. +SELECT 1; +--DROP TABLE IF EXISTS slow_log; +--DROP TABLE IF EXISTS expected_times; +-- +--CREATE TABLE expected_times (QUERY_GROUP_ID String, max_query_duration_ms UInt64) Engine=Memory; +--INSERT INTO expected_times VALUES('main_dashboard_top_query', 100), ('main_dashboard_bottom_query', 100); +-- +--SET log_queries=1; +--SELECT 1; +--SYSTEM FLUSH LOGS; + +--CREATE MATERIALIZED VIEW slow_log Engine=Memory AS +--( +-- SELECT * FROM +-- ( +-- SELECT +-- extract(query,'/\\*\\s*QUERY_GROUP_ID:(.*?)\\s*\\*/') as QUERY_GROUP_ID, +-- * +-- FROM system.query_log +-- WHERE type<>1 and event_date >= yesterday() and event_time > now() - 120 +-- ) as ql +-- INNER JOIN expected_times USING (QUERY_GROUP_ID) +-- WHERE query_duration_ms > max_query_duration_ms +--); +-- +--SELECT 1 /* QUERY_GROUP_ID:main_dashboard_top_query */; +--SELECT 1 /* QUERY_GROUP_ID:main_dashboard_bottom_query */; +-- +--SELECT 1 WHERE not ignore(sleep(0.105)) /* QUERY_GROUP_ID:main_dashboard_top_query */; +--SELECT 1 WHERE not ignore(sleep(0.105)) /* QUERY_GROUP_ID:main_dashboard_bottom_query */; +-- +--SET log_queries=0; +--SYSTEM FLUSH LOGS; +-- +--SELECT '=== system.query_log ==='; +-- +--SELECT +-- extract(query,'/\\*\\s*QUERY_GROUP_ID:(.*?)\\s*\\*/') as QUERY_GROUP_ID, +-- count() +--FROM system.query_log +--WHERE type<>1 and event_date >= yesterday() and event_time > now() - 20 and QUERY_GROUP_ID<>'' +--GROUP BY QUERY_GROUP_ID +--ORDER BY QUERY_GROUP_ID; +-- +--SELECT '=== slowlog ==='; +-- +--SELECT +-- QUERY_GROUP_ID, +-- count() +--FROM slow_log +--GROUP BY QUERY_GROUP_ID +--ORDER BY QUERY_GROUP_ID; +-- +--DROP TABLE slow_log; +--DROP TABLE expected_times; diff --git a/tests/queries/1_stateful/00156_max_execution_speed_sample_merge.reference b/tests/queries/1_stateful/00156_max_execution_speed_sample_merge.reference new file mode 100644 index 000000000000..53bb58224b96 --- /dev/null +++ b/tests/queries/1_stateful/00156_max_execution_speed_sample_merge.reference @@ -0,0 +1,4 @@ +4392010 +1 +4392010 +1 diff --git a/tests/queries/1_stateful/00156_max_execution_speed_sample_merge.sql b/tests/queries/1_stateful/00156_max_execution_speed_sample_merge.sql new file mode 100644 index 000000000000..37e91296f14c --- /dev/null +++ b/tests/queries/1_stateful/00156_max_execution_speed_sample_merge.sql @@ -0,0 +1,16 @@ +SET max_execution_speed = 4000000, timeout_before_checking_execution_speed = 0.001; + +CREATE TEMPORARY TABLE times (t DateTime); + +INSERT INTO times SELECT now(); +SELECT count() FROM test.hits SAMPLE 1 / 2; +INSERT INTO times SELECT now(); + +SELECT max(t) - min(t) >= 1 FROM times; +TRUNCATE TABLE times; + +INSERT INTO times SELECT now(); +SELECT count() FROM merge(test, '^hits$') SAMPLE 1 / 2; +INSERT INTO times SELECT now(); + +SELECT max(t) - min(t) >= 1 FROM times; diff --git a/utils/release/release_lib.sh b/utils/release/release_lib.sh index 9a3a258f15a5..7752af5adf06 100644 --- a/utils/release/release_lib.sh +++ b/utils/release/release_lib.sh @@ -12,10 +12,10 @@ function gen_version_string { function get_version { if [ -z "$VERSION_MAJOR" ] && [ -z "$VERSION_MINOR" ] && [ -z "$VERSION_PATCH" ]; then BASEDIR=$(dirname "${BASH_SOURCE[0]}")/../../ - VERSION_REVISION=`grep "set(VERSION_REVISION" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'` - VERSION_MAJOR=`grep "set(VERSION_MAJOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'` - VERSION_MINOR=`grep "set(VERSION_MINOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'` - VERSION_PATCH=`grep "set(VERSION_PATCH" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'` + VERSION_REVISION=`grep "SET(VERSION_REVISION" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'` + VERSION_MAJOR=`grep "SET(VERSION_MAJOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'` + VERSION_MINOR=`grep "SET(VERSION_MINOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'` + VERSION_PATCH=`grep "SET(VERSION_PATCH" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'` fi VERSION_PREFIX="${VERSION_PREFIX:-v}" VERSION_POSTFIX_TAG="${VERSION_POSTFIX:--testing}" @@ -90,26 +90,26 @@ function gen_revision_author { git_describe=`git describe` git_hash=`git rev-parse HEAD` - sed -i -e "s/set(VERSION_REVISION [^) ]*/set(VERSION_REVISION $VERSION_REVISION/g;" \ - -e "s/set(VERSION_DESCRIBE [^) ]*/set(VERSION_DESCRIBE $git_describe/g;" \ - -e "s/set(VERSION_GITHASH [^) ]*/set(VERSION_GITHASH $git_hash/g;" \ - -e "s/set(VERSION_MAJOR [^) ]*/set(VERSION_MAJOR $VERSION_MAJOR/g;" \ - -e "s/set(VERSION_MINOR [^) ]*/set(VERSION_MINOR $VERSION_MINOR/g;" \ - -e "s/set(VERSION_PATCH [^) ]*/set(VERSION_PATCH $VERSION_PATCH/g;" \ - -e "s/set(VERSION_STRING [^) ]*/set(VERSION_STRING $VERSION_STRING/g;" \ - dbms/cmake/version.cmake + sed -i -e "s/SET(VERSION_REVISION [^) ]*/SET(VERSION_REVISION $VERSION_REVISION/g;" \ + -e "s/SET(VERSION_DESCRIBE [^) ]*/SET(VERSION_DESCRIBE $git_describe/g;" \ + -e "s/SET(VERSION_GITHASH [^) ]*/SET(VERSION_GITHASH $git_hash/g;" \ + -e "s/SET(VERSION_MAJOR [^) ]*/SET(VERSION_MAJOR $VERSION_MAJOR/g;" \ + -e "s/SET(VERSION_MINOR [^) ]*/SET(VERSION_MINOR $VERSION_MINOR/g;" \ + -e "s/SET(VERSION_PATCH [^) ]*/SET(VERSION_PATCH $VERSION_PATCH/g;" \ + -e "s/SET(VERSION_STRING [^) ]*/SET(VERSION_STRING $VERSION_STRING/g;" \ + cmake/autogenerated_versions.txt gen_changelog "$VERSION_STRING" "" "$AUTHOR" "" gen_dockerfiles "$VERSION_STRING" src/Storages/System/StorageSystemContributors.sh ||: - git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" cmake/version.cmake debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp + git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" autogenerated_versions.txt debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp if [ -z $NO_PUSH ]; then git push fi echo "Generated version: ${VERSION_STRING}, revision: ${VERSION_REVISION}." - # Second tag for correct version information in version.cmake inside tag + # Second tag for correct version information in autogenerated_versions.txt inside tag if git tag --force -a "$tag" -m "$tag" then if [ -z $NO_PUSH ]; then diff --git a/utils/zookeeper-cli/zookeeper-cli.cpp b/utils/zookeeper-cli/zookeeper-cli.cpp index 44140423a15e..d4cac387ec35 100644 --- a/utils/zookeeper-cli/zookeeper-cli.cpp +++ b/utils/zookeeper-cli/zookeeper-cli.cpp @@ -70,7 +70,7 @@ int main(int argc, char ** argv) Logger::root().setLevel("trace"); zkutil::ZooKeeper zk(argv[1]); - LineReader lr({}, '\\'); + LineReader lr({}, false, {"\\"}, {}); do {