From 22f357f6ffe2549a6e660e0e1bb4023d56e7e107 Mon Sep 17 00:00:00 2001 From: Oleg Galizin Date: Mon, 19 Aug 2024 12:51:36 +0300 Subject: [PATCH 1/9] params --- clickhouse/base/wire_format.cpp | 73 +++++++++++++++++++++++++++++++++ clickhouse/base/wire_format.h | 1 + clickhouse/client.cpp | 50 ++++++++++++++++++++-- clickhouse/query.h | 14 +++++++ tests/simple/main.cpp | 45 ++++++++++++++++++++ 5 files changed, 179 insertions(+), 4 deletions(-) diff --git a/clickhouse/base/wire_format.cpp b/clickhouse/base/wire_format.cpp index 62a21833..5a9e8ff2 100644 --- a/clickhouse/base/wire_format.cpp +++ b/clickhouse/base/wire_format.cpp @@ -1,3 +1,4 @@ +#include #include "wire_format.h" #include "input.h" @@ -99,4 +100,76 @@ bool WireFormat::SkipString(InputStream& input) { return false; } +const char quoted_chars[] = {'\0', '\b', '\t', '\n', '\'', '\\'}; + +inline const char* find_quoted_chars(const char* start, const char* end) { + while (start < end) { + char c = *start; + for (unsigned i = 0; i < sizeof(quoted_chars); i++) { + if (quoted_chars[i] == c) return start; + } + start++; + } + return nullptr; +} + +void WireFormat::WriteQuotedString(OutputStream& output, std::string_view value) { + auto size = value.size(); + const char* start = value.data(); + const char* end = start + size; + const char* quoted_char = find_quoted_chars(start, end); + if (quoted_char == nullptr) { + WriteVarint64(output, size + 2); + WriteAll(output, "'", 1); + WriteAll(output, start, size); + WriteAll(output, "'", 1); + return; + } + + // calculate quoted chars count + int quoted_count = 1; + const char* next_quoted_char = quoted_char + 1; + while ((next_quoted_char = find_quoted_chars(next_quoted_char, end))) { + quoted_count++; + next_quoted_char++; + } + WriteVarint64(output, size + 2 + 3 * quoted_count); // length + + WriteAll(output, "'", 1); + + do { + auto write_size = quoted_char - start; + WriteAll(output, start, write_size); + WriteAll(output, "\\", 1); + char c = quoted_char[0]; + switch (c) { + case '\0': + WriteAll(output, "x00", 3); + break; + case '\b': + WriteAll(output, "x08", 3); + break; + case '\t': + WriteAll(output, "\\\\t", 3); + break; + case '\n': + WriteAll(output, "\\\\\n", 3); + break; + case '\'': + WriteAll(output, "x27", 3); + break; + case '\\': + WriteAll(output, "\\\\\\", 3); + break; + default: + assert(false); + WriteAll(output, "x3F", 3); // out ? + } + start = quoted_char + 1; + quoted_char = find_quoted_chars(start, end); + } while (quoted_char); + + WriteAll(output, start, end - start); + WriteAll(output, "'", 1); +} } diff --git a/clickhouse/base/wire_format.h b/clickhouse/base/wire_format.h index 6ff53528..8707b2d6 100644 --- a/clickhouse/base/wire_format.h +++ b/clickhouse/base/wire_format.h @@ -22,6 +22,7 @@ class WireFormat { static void WriteFixed(OutputStream& output, const T& value); static void WriteBytes(OutputStream& output, const void* buf, size_t len); static void WriteString(OutputStream& output, std::string_view value); + static void WriteQuotedString(OutputStream& output, std::string_view value); static void WriteUInt64(OutputStream& output, const uint64_t value); static void WriteVarint64(OutputStream& output, uint64_t value); diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index 01ee70b4..ce63e3c0 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -38,8 +38,13 @@ #define DBMS_MIN_REVISION_WITH_DISTRIBUTED_DEPTH 54448 #define DBMS_MIN_REVISION_WITH_INITIAL_QUERY_START_TIME 54449 #define DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS 54451 +#define DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS 54453 +#define DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION 54454 // Client can get some fields in JSon format +#define DBMS_MIN_PROTOCOL_VERSION_WITH_ADDENDUM 54458 // send quota key after handshake +#define DBMS_MIN_PROTOCOL_REVISION_WITH_QUOTA_KEY 54458 // the same +#define DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS 54459 -#define DMBS_PROTOCOL_REVISION DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS +#define DMBS_PROTOCOL_REVISION DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS namespace clickhouse { @@ -433,6 +438,11 @@ bool Client::Impl::Handshake() { if (!ReceiveHello()) { return false; } + + if (server_info_.revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_ADDENDUM) { + WireFormat::WriteString(*output_, std::string()); + } + return true; } @@ -502,7 +512,7 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { return false; } } - if constexpr (DMBS_PROTOCOL_REVISION >= DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO) + if (server_info_.revision >= DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO) { if (!WireFormat::ReadUInt64(*input_, &info.written_rows)) { return false; @@ -589,7 +599,7 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { bool Client::Impl::ReadBlock(InputStream& input, Block* block) { // Additional information about block. - if constexpr (DMBS_PROTOCOL_REVISION >= DBMS_MIN_REVISION_WITH_BLOCK_INFO) { + if (server_info_.revision >= DBMS_MIN_REVISION_WITH_BLOCK_INFO) { uint64_t num; BlockInfo info; @@ -635,6 +645,16 @@ bool Client::Impl::ReadBlock(InputStream& input, Block* block) { if (!WireFormat::ReadString(input, &type)) { return false; } + + if (server_info_.revision >= DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION) { + uint8_t custom_format_len; + if (!WireFormat::ReadFixed(input, &custom_format_len)) { + return false; + } + if (custom_format_len > 0) { + throw UnimplementedError(std::string("unsupported custom serialization")); + } + } if (ColumnRef col = CreateColumnByType(type, create_column_settings)) { if (num_rows && !col->Load(&input, num_rows)) { @@ -653,7 +673,7 @@ bool Client::Impl::ReadBlock(InputStream& input, Block* block) { bool Client::Impl::ReceiveData() { Block block; - if constexpr (DMBS_PROTOCOL_REVISION >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES) { + if (server_info_.revision >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES) { if (!WireFormat::SkipString(*input_)) { return false; } @@ -793,6 +813,11 @@ void Client::Impl::SendQuery(const Query& query) { throw UnimplementedError(std::string("Can't send open telemetry tracing context to a server, server version is too old")); } } + if (server_info_.revision >= DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS) { + WireFormat::WriteUInt64(*output_, 0); + WireFormat::WriteUInt64(*output_, 0); + WireFormat::WriteUInt64(*output_, 0); + } } /// Per query settings @@ -817,6 +842,18 @@ void Client::Impl::SendQuery(const Query& query) { WireFormat::WriteUInt64(*output_, Stages::Complete); WireFormat::WriteUInt64(*output_, compression_); WireFormat::WriteString(*output_, query.GetText()); + + //Send params after query text + if (server_info_.revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS) { + for(const auto& [name, value] : query.GetParams()) { + // params is like query settings + WireFormat::WriteString(*output_, name); + WireFormat::WriteVarint64(*output_, 2); // Custom + WireFormat::WriteQuotedString(*output_, value); + } + WireFormat::WriteString(*output_, std::string()); // empty string after last param + } + // Send empty block as marker of // end of data SendData(Block()); @@ -842,6 +879,11 @@ void Client::Impl::WriteBlock(const Block& block, OutputStream& output) { WireFormat::WriteString(output, bi.Name()); WireFormat::WriteString(output, bi.Type()->GetName()); + if (server_info_.revision >= DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION) { + // TODO: custom serialization + WireFormat::WriteFixed(output, 0); + } + // Empty columns are not serialized and occupy exactly 0 bytes. // ref https://github.com/ClickHouse/ClickHouse/blob/39b37a3240f74f4871c8c1679910e065af6bea19/src/Formats/NativeWriter.cpp#L163 const bool containsData = block.GetRowCount() > 0; diff --git a/clickhouse/query.h b/clickhouse/query.h index b6551803..acd5bf4c 100644 --- a/clickhouse/query.h +++ b/clickhouse/query.h @@ -26,6 +26,7 @@ struct QuerySettingsField { }; using QuerySettings = std::unordered_map; +using QueryParams = std::unordered_map; struct Profile { uint64_t rows = 0; @@ -115,6 +116,18 @@ class Query : public QueryEvents { return *this; } + inline const QueryParams& GetParams() const { return query_params_; } + + inline Query& SetParams(QueryParams query_params) { + query_params_ = std::move(query_params); + return *this; + } + + inline Query& SetParam(const std::string& name, const std::string& value) { + query_params_[name] = value; + return *this; + } + inline const std::optional& GetTracingContext() const { return tracing_context_; } @@ -219,6 +232,7 @@ class Query : public QueryEvents { const std::string query_id_; std::optional tracing_context_; QuerySettings query_settings_; + QueryParams query_params_; ExceptionCallback exception_cb_; ProgressCallback progress_cb_; SelectCallback select_cb_; diff --git a/tests/simple/main.cpp b/tests/simple/main.cpp index 2911e559..725170fc 100644 --- a/tests/simple/main.cpp +++ b/tests/simple/main.cpp @@ -234,6 +234,50 @@ inline void GenericExample(Client& client) { client.Execute("DROP TEMPORARY TABLE test_client"); } +inline void ParamExample(Client& client) { + /// Create a table. + client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS test_client (id UInt64, name String)"); + + { + Query query("insert into test_client values ({id: UInt64}, {name: String})"); + + query.SetParam("id", "1").SetParam("name", "NAME"); + client.Execute(query); + + query.SetParam("id", "123").SetParam("name", "FromParam"); + client.Execute(query); + + query.SetParam("id", "333") + .SetParam("name", + std::string("A\000A\001A\002A\003A\004A\005A\006A\007A\010A\011A\012A\013A\014A\015A\016A\017A\020A\021A\022A\023A\024A\025A\026A\027A\030A\031A\032A\033A\034" + "A\035A\036A\037A", 65)); + client.Execute(query); + + unsigned char big_string[128 - 32]; + for (unsigned int i = 0; i < sizeof(big_string); i++) big_string[i] = i + 32; + query.SetParam("id", "444") + .SetParam("name", + std::string((char*)big_string, sizeof(big_string))); + client.Execute(query); + + query.SetParam("id", "555") + .SetParam("name", "utf8Русский"); + client.Execute(query); + } + + /// Select values inserted in the previous step. + Query query ("SELECT id, name, length(name) FROM test_client where id > {a: Int32}"); + query.SetParam("a", "4"); + SelectCallback cb([](const Block& block) + { + std::cout << PrettyPrintBlock{block} << std::endl; + }); + query.OnData(cb); + client.Select(query); + /// Delete table. + client.Execute("DROP TEMPORARY TABLE test_client"); +} + inline void NullableExample(Client& client) { /// Create a table. client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS test_client (id Nullable(UInt64), date Nullable(Date))"); @@ -478,6 +522,7 @@ inline void IPExample(Client &client) { } static void RunTests(Client& client) { + ParamExample(client); ArrayExample(client); CancelableExample(client); DateExample(client); From 7642eaae8a2dbf522f1d1c7d1911ee098f59af10 Mon Sep 17 00:00:00 2001 From: Oleg Galizin Date: Fri, 20 Sep 2024 15:46:39 +0300 Subject: [PATCH 2/9] Nullable params --- clickhouse/base/wire_format.cpp | 9 ++++++++- clickhouse/base/wire_format.h | 3 ++- clickhouse/query.h | 5 +++-- tests/simple/main.cpp | 28 ++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/clickhouse/base/wire_format.cpp b/clickhouse/base/wire_format.cpp index 5a9e8ff2..0e4ace39 100644 --- a/clickhouse/base/wire_format.cpp +++ b/clickhouse/base/wire_format.cpp @@ -113,7 +113,14 @@ inline const char* find_quoted_chars(const char* start, const char* end) { return nullptr; } -void WireFormat::WriteQuotedString(OutputStream& output, std::string_view value) { +void WireFormat::WriteQuotedString(OutputStream& output, std::optional opt) { + if (!opt) //NULL + { + WriteVarint64(output, 5); + WriteAll(output, "'\\\\N'", 5); + return; + } + std::string_view value = *opt; auto size = value.size(); const char* start = value.data(); const char* end = start + size; diff --git a/clickhouse/base/wire_format.h b/clickhouse/base/wire_format.h index 8707b2d6..3c58c6d0 100644 --- a/clickhouse/base/wire_format.h +++ b/clickhouse/base/wire_format.h @@ -2,6 +2,7 @@ #include #include +#include namespace clickhouse { @@ -22,7 +23,7 @@ class WireFormat { static void WriteFixed(OutputStream& output, const T& value); static void WriteBytes(OutputStream& output, const void* buf, size_t len); static void WriteString(OutputStream& output, std::string_view value); - static void WriteQuotedString(OutputStream& output, std::string_view value); + static void WriteQuotedString(OutputStream& output, std::optional value); static void WriteUInt64(OutputStream& output, const uint64_t value); static void WriteVarint64(OutputStream& output, uint64_t value); diff --git a/clickhouse/query.h b/clickhouse/query.h index acd5bf4c..0dc82dd1 100644 --- a/clickhouse/query.h +++ b/clickhouse/query.h @@ -26,7 +26,8 @@ struct QuerySettingsField { }; using QuerySettings = std::unordered_map; -using QueryParams = std::unordered_map; +using QueryParamValue = std::optional; +using QueryParams = std::unordered_map; struct Profile { uint64_t rows = 0; @@ -123,7 +124,7 @@ class Query : public QueryEvents { return *this; } - inline Query& SetParam(const std::string& name, const std::string& value) { + inline Query& SetParam(const std::string& name, const QueryParamValue& value) { query_params_[name] = value; return *this; } diff --git a/tests/simple/main.cpp b/tests/simple/main.cpp index 725170fc..36f89624 100644 --- a/tests/simple/main.cpp +++ b/tests/simple/main.cpp @@ -278,6 +278,33 @@ inline void ParamExample(Client& client) { client.Execute("DROP TEMPORARY TABLE test_client"); } +inline void ParamNullExample(Client& client) { + client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS test_client (id UInt64, name Nullable(String))"); + + Query query("insert into test_client values ({id: UInt64}, {name: Nullable(String)})"); + + query.SetParam("id", "123").SetParam("name", QueryParamValue()); + client.Execute(query); + + query.SetParam("id", "456").SetParam("name", "String Value"); + client.Execute(query); + + client.Select("SELECT id, name FROM test_client", [](const Block& block) { + for (size_t c = 0; c < block.GetRowCount(); ++c) { + std::cerr << block[0]->As()->At(c) << " "; + + auto col_string = block[1]->As(); + if (col_string->IsNull(c)) { + std::cerr << "\\N\n"; + } else { + std::cerr << col_string->Nested()->As()->At(c) << "\n"; + } + } + }); + + client.Execute("DROP TEMPORARY TABLE test_client"); +} + inline void NullableExample(Client& client) { /// Create a table. client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS test_client (id Nullable(UInt64), date Nullable(Date))"); @@ -523,6 +550,7 @@ inline void IPExample(Client &client) { static void RunTests(Client& client) { ParamExample(client); + ParamNullExample(client); ArrayExample(client); CancelableExample(client); DateExample(client); From 786b5d4b9a4dc513a9eac9a7c291c6c2f4d0e60b Mon Sep 17 00:00:00 2001 From: Oleg Galizin Date: Mon, 21 Oct 2024 20:34:28 +0300 Subject: [PATCH 3/9] Query parameters unit test --- ut/client_ut.cpp | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 3e08b17d..4fc8c7aa 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1488,3 +1488,45 @@ TEST(SimpleClientTest, issue_335_reconnects_count) { << "\tThere was no attempt to connect to endpoint " << endpoint; } } + +TEST_P(ClientCase, QueryParameters) { + const std::string table_name = "test_clickhouse_cpp_query_parameter"; + client_->Execute("CREATE TEMPORARY TABLE IF NOT EXISTS " + table_name + " (id UInt64, name String)"); + { + Query query("insert into " + table_name + " values ({id: UInt64}, {name: String})"); + + query.SetParam("id", "1").SetParam("name", "NAME"); + client_->Execute(query); + + query.SetParam("id", "123").SetParam("name", "FromParam"); + client_->Execute(query); + + query.SetParam("id", "333") + .SetParam("name", std::string("A\000A\001A\002A\003A\004A\005A\006A\007A\010A\011A\012A\013A\014A\015A\016A\017A\020A\021A\022A" + "\023A\024A\025A\026A\027A\030A\031A\032A\033A\034" + "A\035A\036A\037A", + 65)); + client_->Execute(query); + + unsigned char big_string[128 - 32]; + for (unsigned int i = 0; i < sizeof(big_string); i++) big_string[i] = i + 32; + query.SetParam("id", "444").SetParam("name", std::string((char*)big_string, sizeof(big_string))); + client_->Execute(query); + + query.SetParam("id", "555").SetParam("name", "utf8Русский"); + client_->Execute(query); + } + + Query query("SELECT id, name, length(name) FROM " + table_name + " where id > {a: Int32}"); + query.SetParam("a", "4"); + size_t total_count = 0; + SelectCallback cb([&total_count](const Block& block) { + total_count += block.GetRowCount(); + //std::cout << PrettyPrintBlock{block} << std::endl; + }); + query.OnData(cb); + client_->Select(query); + EXPECT_EQ(4u, total_count); + + client_->Execute("DROP TEMPORARY TABLE " + table_name); +} From 8cdd4d8eb5c48031e1956ccd98d725fdab15db4e Mon Sep 17 00:00:00 2001 From: Oleg Galizin Date: Wed, 30 Oct 2024 13:43:21 +0300 Subject: [PATCH 4/9] Changes by review --- clickhouse/base/wire_format.cpp | 29 ++++++++++++++--------------- clickhouse/base/wire_format.h | 4 ++-- clickhouse/client.cpp | 9 +++++++-- tests/simple/main.cpp | 24 ++++++++++++++---------- ut/client_ut.cpp | 19 ++++++++++++------- 5 files changed, 49 insertions(+), 36 deletions(-) diff --git a/clickhouse/base/wire_format.cpp b/clickhouse/base/wire_format.cpp index 0e4ace39..46364db2 100644 --- a/clickhouse/base/wire_format.cpp +++ b/clickhouse/base/wire_format.cpp @@ -100,12 +100,12 @@ bool WireFormat::SkipString(InputStream& input) { return false; } -const char quoted_chars[] = {'\0', '\b', '\t', '\n', '\'', '\\'}; +const std::vector quoted_chars = {'\0', '\b', '\t', '\n', '\'', '\\'}; inline const char* find_quoted_chars(const char* start, const char* end) { while (start < end) { char c = *start; - for (unsigned i = 0; i < sizeof(quoted_chars); i++) { + for (unsigned i = 0; i < quoted_chars.size(); i++) { if (quoted_chars[i] == c) return start; } start++; @@ -113,14 +113,7 @@ inline const char* find_quoted_chars(const char* start, const char* end) { return nullptr; } -void WireFormat::WriteQuotedString(OutputStream& output, std::optional opt) { - if (!opt) //NULL - { - WriteVarint64(output, 5); - WriteAll(output, "'\\\\N'", 5); - return; - } - std::string_view value = *opt; +void WireFormat::WriteQuotedString(OutputStream& output, std::string_view value) { auto size = value.size(); const char* start = value.data(); const char* end = start + size; @@ -157,20 +150,19 @@ void WireFormat::WriteQuotedString(OutputStream& output, std::optional #include -#include namespace clickhouse { @@ -23,7 +22,8 @@ class WireFormat { static void WriteFixed(OutputStream& output, const T& value); static void WriteBytes(OutputStream& output, const void* buf, size_t len); static void WriteString(OutputStream& output, std::string_view value); - static void WriteQuotedString(OutputStream& output, std::optional value); + static void WriteQuotedString(OutputStream& output, std::string_view value); + static void WriteParamNullRepresentation(OutputStream& output); static void WriteUInt64(OutputStream& output, const uint64_t value); static void WriteVarint64(OutputStream& output, uint64_t value); diff --git a/clickhouse/client.cpp b/clickhouse/client.cpp index ce63e3c0..a8ad3400 100644 --- a/clickhouse/client.cpp +++ b/clickhouse/client.cpp @@ -814,6 +814,7 @@ void Client::Impl::SendQuery(const Query& query) { } } if (server_info_.revision >= DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS) { + // replica dont supported by client WireFormat::WriteUInt64(*output_, 0); WireFormat::WriteUInt64(*output_, 0); WireFormat::WriteUInt64(*output_, 0); @@ -848,8 +849,12 @@ void Client::Impl::SendQuery(const Query& query) { for(const auto& [name, value] : query.GetParams()) { // params is like query settings WireFormat::WriteString(*output_, name); - WireFormat::WriteVarint64(*output_, 2); // Custom - WireFormat::WriteQuotedString(*output_, value); + const uint64_t Custom = 2; + WireFormat::WriteVarint64(*output_, Custom); + if (value) + WireFormat::WriteQuotedString(*output_, *value); + else + WireFormat::WriteParamNullRepresentation(*output_); } WireFormat::WriteString(*output_, std::string()); // empty string after last param } diff --git a/tests/simple/main.cpp b/tests/simple/main.cpp index 36f89624..aa45bb11 100644 --- a/tests/simple/main.cpp +++ b/tests/simple/main.cpp @@ -245,19 +245,23 @@ inline void ParamExample(Client& client) { client.Execute(query); query.SetParam("id", "123").SetParam("name", "FromParam"); - client.Execute(query); + client.Execute(query); + + const char FirstPrintable = ' '; + char test_str1[FirstPrintable * 2 + 1]; + for (unsigned int i = 0; i < FirstPrintable; i++) { + test_str1[i * 2] = 'A'; + test_str1[i * 2 + 1] = i; + } + test_str1[int(FirstPrintable * 2)] = 'A'; - query.SetParam("id", "333") - .SetParam("name", - std::string("A\000A\001A\002A\003A\004A\005A\006A\007A\010A\011A\012A\013A\014A\015A\016A\017A\020A\021A\022A\023A\024A\025A\026A\027A\030A\031A\032A\033A\034" - "A\035A\036A\037A", 65)); + query.SetParam("id", "333").SetParam("name", std::string(test_str1, FirstPrintable * 2 + 1)); client.Execute(query); - unsigned char big_string[128 - 32]; - for (unsigned int i = 0; i < sizeof(big_string); i++) big_string[i] = i + 32; - query.SetParam("id", "444") - .SetParam("name", - std::string((char*)big_string, sizeof(big_string))); + const char LastPrintable = 127; + unsigned char big_string[LastPrintable - FirstPrintable]; + for (unsigned int i = 0; i < sizeof(big_string); i++) big_string[i] = i + FirstPrintable; + query.SetParam("id", "444").SetParam("name", std::string((char*)big_string, sizeof(big_string))); client.Execute(query); query.SetParam("id", "555") diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 4fc8c7aa..b4f58acf 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1501,15 +1501,20 @@ TEST_P(ClientCase, QueryParameters) { query.SetParam("id", "123").SetParam("name", "FromParam"); client_->Execute(query); - query.SetParam("id", "333") - .SetParam("name", std::string("A\000A\001A\002A\003A\004A\005A\006A\007A\010A\011A\012A\013A\014A\015A\016A\017A\020A\021A\022A" - "\023A\024A\025A\026A\027A\030A\031A\032A\033A\034" - "A\035A\036A\037A", - 65)); + const char FirstPrintable = ' '; + char test_str1[FirstPrintable * 2 + 1]; + for (unsigned int i = 0; i < FirstPrintable; i++) { + test_str1[i * 2] = 'A'; + test_str1[i * 2 + 1] = i; + } + test_str1[int(FirstPrintable * 2)] = 'A'; + + query.SetParam("id", "333").SetParam("name", std::string(test_str1, FirstPrintable * 2 + 1)); client_->Execute(query); - unsigned char big_string[128 - 32]; - for (unsigned int i = 0; i < sizeof(big_string); i++) big_string[i] = i + 32; + const char LastPrintable = 127; + unsigned char big_string[LastPrintable - FirstPrintable]; + for (unsigned int i = 0; i < sizeof(big_string); i++) big_string[i] = i + FirstPrintable; query.SetParam("id", "444").SetParam("name", std::string((char*)big_string, sizeof(big_string))); client_->Execute(query); From bf1fefd544f18b51fb0a2d3609abe771b8c5ffca Mon Sep 17 00:00:00 2001 From: Oleg Galizin Date: Thu, 31 Oct 2024 17:36:56 +0300 Subject: [PATCH 5/9] Additional changes by review --- clickhouse/base/wire_format.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/base/wire_format.cpp b/clickhouse/base/wire_format.cpp index 46364db2..61a56106 100644 --- a/clickhouse/base/wire_format.cpp +++ b/clickhouse/base/wire_format.cpp @@ -173,7 +173,7 @@ void WireFormat::WriteQuotedString(OutputStream& output, std::string_view value) } void WireFormat::WriteParamNullRepresentation(OutputStream& output) { - const std::string NULL_REPRESENTATION("'\\\\N'"); + const std::string NULL_REPRESENTATION(R"('\\N')"); WriteVarint64(output, NULL_REPRESENTATION.size()); WriteAll(output, NULL_REPRESENTATION.data(), NULL_REPRESENTATION.size()); } From 86d0c843f4f1ecea555cdef6f5013575923dd417 Mon Sep 17 00:00:00 2001 From: Oleg Galizin Date: Tue, 5 Nov 2024 09:01:16 +0300 Subject: [PATCH 6/9] Chenges by review --- clickhouse/base/wire_format.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/clickhouse/base/wire_format.cpp b/clickhouse/base/wire_format.cpp index 61a56106..b3ca7c98 100644 --- a/clickhouse/base/wire_format.cpp +++ b/clickhouse/base/wire_format.cpp @@ -100,12 +100,11 @@ bool WireFormat::SkipString(InputStream& input) { return false; } -const std::vector quoted_chars = {'\0', '\b', '\t', '\n', '\'', '\\'}; - inline const char* find_quoted_chars(const char* start, const char* end) { + static const char quoted_chars[] = {'\0', '\b', '\t', '\n', '\'', '\\'}; while (start < end) { char c = *start; - for (unsigned i = 0; i < quoted_chars.size(); i++) { + for (unsigned i = 0; i < sizeof(quoted_chars); i++) { if (quoted_chars[i] == c) return start; } start++; From 2edfc17e41a7563b96c053896fb04f38645ac31e Mon Sep 17 00:00:00 2001 From: Oleg Galizin Date: Fri, 22 Nov 2024 09:20:23 +0300 Subject: [PATCH 7/9] Skip test query parameters for CH < 24.7 --- ut/client_ut.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index b4f58acf..43b8a104 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -1490,6 +1490,10 @@ TEST(SimpleClientTest, issue_335_reconnects_count) { } TEST_P(ClientCase, QueryParameters) { + const auto & server_info = client_->GetServerInfo(); + if (versionNumber(server_info) < versionNumber(24, 7)) { + GTEST_SKIP() << "Test is skipped since server '" << server_info << "' does not support query parameters" << std::endl; + } const std::string table_name = "test_clickhouse_cpp_query_parameter"; client_->Execute("CREATE TEMPORARY TABLE IF NOT EXISTS " + table_name + " (id UInt64, name String)"); { From 335ff07c340aaf5cdfaaca76d0f2f9eaa5ac49e2 Mon Sep 17 00:00:00 2001 From: Oleg Galizin Date: Thu, 23 Jan 2025 12:53:26 +0300 Subject: [PATCH 8/9] use find_first_of --- clickhouse/base/wire_format.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/clickhouse/base/wire_format.cpp b/clickhouse/base/wire_format.cpp index b3ca7c98..1ded6b56 100644 --- a/clickhouse/base/wire_format.cpp +++ b/clickhouse/base/wire_format.cpp @@ -7,6 +7,7 @@ #include "../exceptions.h" #include +#include namespace { constexpr int MAX_VARINT_BYTES = 10; @@ -100,16 +101,11 @@ bool WireFormat::SkipString(InputStream& input) { return false; } -inline const char* find_quoted_chars(const char* start, const char* end) { - static const char quoted_chars[] = {'\0', '\b', '\t', '\n', '\'', '\\'}; - while (start < end) { - char c = *start; - for (unsigned i = 0; i < sizeof(quoted_chars); i++) { - if (quoted_chars[i] == c) return start; - } - start++; - } - return nullptr; +inline const char* find_quoted_chars(const char* start, const char* end) +{ + static constexpr std::array quoted_chars {'\0', '\b', '\t', '\n', '\'', '\\'}; + const auto first = std::find_first_of(start, end, quoted_chars.begin(), quoted_chars.end()); + return (first == end) ? nullptr : first; } void WireFormat::WriteQuotedString(OutputStream& output, std::string_view value) { From 4f836833f0210b8548158419db0bc94d1ab39263 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 17 Feb 2025 09:48:32 +0100 Subject: [PATCH 9/9] Fixed compilation error on windows --- clickhouse/base/wire_format.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clickhouse/base/wire_format.cpp b/clickhouse/base/wire_format.cpp index 1ded6b56..55d4fb8c 100644 --- a/clickhouse/base/wire_format.cpp +++ b/clickhouse/base/wire_format.cpp @@ -103,8 +103,9 @@ bool WireFormat::SkipString(InputStream& input) { inline const char* find_quoted_chars(const char* start, const char* end) { - static constexpr std::array quoted_chars {'\0', '\b', '\t', '\n', '\'', '\\'}; - const auto first = std::find_first_of(start, end, quoted_chars.begin(), quoted_chars.end()); + static constexpr char quoted_chars[] = {'\0', '\b', '\t', '\n', '\'', '\\'}; + const auto first = std::find_first_of(start, end, std::begin(quoted_chars), std::end(quoted_chars)); + return (first == end) ? nullptr : first; }