Merge branch 'master' into openssl_support

.github/workflows/tags_stable.yml

@@ -61,6 +61,7 @@ jobs:
         committer: "robot-clickhouse <[email protected]>"
         commit-message: Update version_date.tsv and changelogs after ${{ env.GITHUB_TAG }}
         branch: auto/${{ env.GITHUB_TAG }}
+        assignees: ${{ github.event.sender.login }}  # assign the PR to the tag pusher
         delete-branch: true
         title: Update version_date.tsv and changelogs after ${{ env.GITHUB_TAG }}
         labels: do not test

README.md

@@ -16,8 +16,6 @@ ClickHouse® is an open-source column-oriented database management system that a
 * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
 
 ## Upcoming events
-* [**v22.11 Release Webinar**](https://clickhouse.com/company/events/v22-11-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
-* [**ClickHosue Meetup at the RELEX Solutions office in Stockholm**](https://www.meetup.com/clickhouse-stockholm-user-group/events/289492084/) - Dec 1 - Formulate by RELEX is a Swedish promotion planning and analytics company. They will share why they chose ClickHouse for their real time analytics and forecasting solution. The ClickHouse team will then present how ClickHouse is used for real time financial data analytics, including tick data, trade analytics and risk management.
-* [**ClickHouse Meetup at the Deutsche Bank office in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/289311596/) - Dec 5 - Hear from Deutsche Bank on why they chose ClickHouse for big sensitive data in a regulated environment. The ClickHouse team will then present how ClickHouse is used for real time financial data analytics, including tick data, trade analytics and risk management.
-* [**ClickHouse Meetup at the Rokt offices in Manhattan**](https://www.meetup.com/clickhouse-new-york-user-group/events/289403909/) - Dec 6 - We are very excited to be holding our next in-person ClickHouse meetup at the Rokt offices in Manhattan. Featuring talks from Bloomberg, Disney Streaming, Prequel, Rokt, and ClickHouse
-
+* [**v22.12 Release Webinar**](https://clickhouse.com/company/events/v22-12-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
+* [**ClickHouse Meetup at the CHEQ office in Tel Aviv**](https://www.meetup.com/clickhouse-tel-aviv-user-group/events/289599423/) - Jan 16 - We are very excited to be holding our next in-person ClickHouse meetup at the CHEQ office in Tel Aviv! Hear from CHEQ, ServiceNow and Contentsquare, as well as a deep dive presentation from ClickHouse CTO Alexey Milovidov. Join us for a fun evening of talks, food and discussion!
+* **ClickHouse Meetup in Seattle* - Keep an eye on this space as we will be announcing a January meetup in Seattle soon!

base/base/ReplxxLineReader.cpp

@@ -17,6 +17,7 @@
 #include <filesystem>
 #include <fmt/format.h>
 #include <boost/algorithm/string/split.hpp>
+#include <boost/algorithm/string/replace.hpp>
 #include <boost/algorithm/string/classification.hpp> /// is_any_of
 
 namespace
@@ -38,7 +39,7 @@ std::string getEditor()
     return editor;
 }
 
-std::string getFuzzyFinder()
+std::pair<std::string, FuzzyFinderType> getFuzzyFinder()
 {
     const char * env_path = std::getenv("PATH"); // NOLINT(concurrency-mt-unsafe)
 
@@ -52,14 +53,20 @@ std::string getFuzzyFinder()
         std::filesystem::path path(path_str);
         std::filesystem::path sk_bin_path = path / "sk";
         if (!access(sk_bin_path.c_str(), X_OK))
-            return sk_bin_path;
+            return {sk_bin_path, FUZZY_FINDER_SKIM};
 
         std::filesystem::path fzf_bin_path = path / "fzf";
         if (!access(fzf_bin_path.c_str(), X_OK))
-            return fzf_bin_path;
+            return {fzf_bin_path, FUZZY_FINDER_FZF};
     }
 
-    return {};
+    return {"", FUZZY_FINDER_NONE};
+}
+
+String escapeShellArgument(std::string arg)
+{
+    boost::replace_all(arg, "'", "'\\''");
+    return fmt::format("'{}'", arg);
 }
 
 /// See comments in ShellCommand::executeImpl()
@@ -305,11 +312,12 @@ ReplxxLineReader::ReplxxLineReader(
     replxx::Replxx::highlighter_callback_t highlighter_)
     : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_)), highlighter(std::move(highlighter_))
     , editor(getEditor())
-    , fuzzy_finder(getFuzzyFinder())
 {
     using namespace std::placeholders;
     using Replxx = replxx::Replxx;
 
+    std::tie(fuzzy_finder, fuzzy_finder_type) = getFuzzyFinder();
+
     if (!history_file_path.empty())
     {
         history_file_fd = open(history_file_path.c_str(), O_RDWR);
@@ -415,11 +423,12 @@ ReplxxLineReader::ReplxxLineReader(
     rx.bind_key(Replxx::KEY::meta('#'), insert_comment_action);
 
     /// interactive search in history (requires fzf/sk)
-    if (!fuzzy_finder.empty())
+    if (fuzzy_finder_type != FUZZY_FINDER_NONE)
     {
         auto interactive_history_search = [this](char32_t code)
         {
             openInteractiveHistorySearch();
+            rx.invoke(Replxx::ACTION::CLEAR_SELF, code);
             return rx.invoke(Replxx::ACTION::REPAINT, code);
         };
         rx.bind_key(Replxx::KEY::control('R'), interactive_history_search);
@@ -515,9 +524,22 @@ void ReplxxLineReader::openInteractiveHistorySearch()
     ///
     /// And also note, that fzf and skim is 95% compatible (at least option
     /// that is used here)
-    std::string fuzzy_finder_command = fmt::format(
-        "{} --read0 --tac --no-sort --tiebreak=index --bind=ctrl-r:toggle-sort --height=30% < {} > {}",
-        fuzzy_finder, history_file.getPath(), output_file.getPath());
+    std::string fuzzy_finder_command = fmt::format("{} --read0 --height=30%", fuzzy_finder);
+    switch (fuzzy_finder_type)
+    {
+        case FUZZY_FINDER_SKIM:
+            fuzzy_finder_command += " --tac --tiebreak=-score";
+            break;
+        case FUZZY_FINDER_FZF:
+            fuzzy_finder_command += " --tac --tiebreak=index";
+            break;
+        case FUZZY_FINDER_NONE:
+            /// assertion for !fuzzy_finder.empty() is enough
+            break;
+    }
+    fuzzy_finder_command += fmt::format(" < {} > {}",
+        escapeShellArgument(history_file.getPath()),
+        escapeShellArgument(output_file.getPath()));
     char * const argv[] = {sh, sh_c, fuzzy_finder_command.data(), nullptr};
 
     try

base/base/ReplxxLineReader.h

@@ -4,6 +4,14 @@
 
 #include <replxx.hxx>
 
+enum FuzzyFinderType
+{
+    FUZZY_FINDER_NONE,
+    /// Use https://github.com/junegunn/fzf
+    FUZZY_FINDER_FZF,
+    /// Use https://github.com/lotabout/skim
+    FUZZY_FINDER_SKIM,
+};
 
 class ReplxxLineReader : public LineReader
 {
@@ -38,4 +46,5 @@ class ReplxxLineReader : public LineReader
 
     std::string editor;
     std::string fuzzy_finder;
+    FuzzyFinderType fuzzy_finder_type = FUZZY_FINDER_NONE;
 };

docs/en/interfaces/formats.md

@@ -1202,6 +1202,7 @@ SELECT * FROM json_each_row_nested
 - [input_format_import_nested_json](../operations/settings/settings.md#input_format_import_nested_json) - map nested JSON data to nested tables (it works for JSONEachRow format). Default value - `false`.
 - [input_format_json_read_bools_as_numbers](../operations/settings/settings.md#input_format_json_read_bools_as_numbers) - allow to parse bools as numbers in JSON input formats. Default value - `true`.
 - [input_format_json_read_numbers_as_strings](../operations/settings/settings.md#input_format_json_read_numbers_as_strings) - allow to parse numbers as strings in JSON input formats. Default value - `false`.
+- [input_format_json_read_objects_as_strings](../operations/settings/settings.md#input_format_json_read_objects_as_strings) - allow to parse JSON objects as strings in JSON input formats. Default value - `false`.
 - [output_format_json_quote_64bit_integers](../operations/settings/settings.md#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`.
 - [output_format_json_quote_64bit_floats](../operations/settings/settings.md#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`.
 - [output_format_json_quote_denormals](../operations/settings/settings.md#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`.

docs/en/interfaces/http.md

@@ -244,15 +244,15 @@ The username and password can be indicated in one of three ways:
 $ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @-
 ```
 
-1.  In the ‘user’ and ‘password’ URL parameters. Example:
+2.  In the ‘user’ and ‘password’ URL parameters (*We do not recommend using this method as the parameter might be logged by web proxy and cached in the browser*). Example:
 
 <!-- -->
 
 ``` bash
 $ echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d @-
 ```
 
-1.  Using ‘X-ClickHouse-User’ and ‘X-ClickHouse-Key’ headers. Example:
+3.  Using ‘X-ClickHouse-User’ and ‘X-ClickHouse-Key’ headers. Example:
 
 <!-- -->
 

docs/en/operations/settings/constraints-on-settings.md

@@ -91,4 +91,21 @@ Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should n
 
 **Note:** the `default` profile has special handling: all the constraints defined for the `default` profile become the default constraints, so they restrict all the users until they’re overridden explicitly for these users.
 
+## Constraints on Merge Tree Settings
+It is possible to set constraints for [merge tree settings](merge-tree-settings.md). There constraints are applied when table with merge tree engine is created or its storage settings are altered. Name of merge tree setting must be prepended by `merge_tree_` prefix when referenced in `<constraint>` section.
+
+**Example:** Forbid to create new tables with explicitly specified `storage_policy`
+
+``` xml
+<profiles>
+  <default>
+    <constraints>
+      <merge_tree_storage_policy>
+        <const/>
+      </merge_tree_storage_policy>
+    </constraints>
+  </default>
+</profiles>
+```
+
 [Original article](https://clickhouse.com/docs/en/operations/settings/constraints_on_settings/) <!--hide-->

docs/en/operations/settings/settings.md

@@ -3753,6 +3753,29 @@ Allow parsing numbers as strings in JSON input formats.
 
 Disabled by default.
 
+### input_format_json_read_objects_as_strings {#input_format_json_read_objects_as_strings}
+
+Allow parsing JSON objects as strings in JSON input formats.
+
+Example:
+
+```sql
+SET input_format_json_read_objects_as_strings = 1;
+CREATE TABLE test (id UInt64, obj String, date Date) ENGINE=Memory();
+INSERT INTO test FORMAT JSONEachRow {"id" : 1, "obj" : {"a" : 1, "b" : "Hello"}, "date" : "2020-01-01"};
+SELECT * FROM test;
+```
+
+Result:
+
+```
+┌─id─┬─obj──────────────────────┬───────date─┐
+│  1 │ {"a" : 1, "b" : "Hello"} │ 2020-01-01 │
+└────┴──────────────────────────┴────────────┘
+```
+
+Disabled by default.
+
 ### input_format_json_validate_types_from_metadata {#input_format_json_validate_types_from_metadata}
 
 For JSON/JSONCompact/JSONColumnsWithMetadata input formats, if this setting is set to 1,

docs/en/operations/update.md

@@ -1,10 +1,82 @@
 ---
 slug: /en/operations/update
+sidebar_title: Self-managed Upgrade
+title: Self-managed Upgrade
 ---
 
-# Update
+## ClickHouse upgrade overview
 
-## Self-managed ClickHouse Upgrade
+This document contains:
+- general guidelines
+- a recommended plan
+- specifics for upgrading the binaries on your systems
+
+## General guidelines
+
+These notes should help you with planning, and to understand why we make the recommendations that we do later in the document.
+
+### Upgrade ClickHouse server separately from ClickHouse Keeper or ZooKeeper
+Unless there is a security fix needed for ClickHouse Keeper or Apache ZooKeeper it is not necessary to upgrade Keeper when you upgrade ClickHouse server.  Keeper stability is required during the upgrade process, so complete the ClickHouse server upgrades before considering an upgrade of Keeper.
+
+### Minor version upgrades should be adopted often
+It is highly recommended to always upgrade to the newest minor version as soon as it is released. Minor releases do not have breaking changes but do have important bug fixes (and may have security fixes).
+
+
+### Test experimental features on a separate ClickHouse server running the target version
+
+The compatibility of experimental features can be broken at any moment in any way.  If you are using experimental features, then check the changelogs and consider setting up a separate ClickHouse server with the target version installed and test your use of the experimental features there.
+
+### Downgrades
+If you upgrade and then realize that the new version is not compatible with some feature that you depend on you may be able to downgrade to a recent (less than one year old) version if you have not started to use any of the new features.  Once the new features are used the downgrade will not work.
+
+### Multiple ClickHouse server versions in a cluster
+
+We make an effort to maintain a one-year compatibility window (which includes 2 LTS versions). This means that any two versions should be able to work together in a cluster if the difference between them is less than one year (or if there are less than two LTS versions between them). However, it is recommended to upgrade all members of a cluster to the same version as quickly as possible, as some minor issues are possible (like slowdown of distributed queries, retriable errors in some background operations in ReplicatedMergeTree, etc).
+
+We never recommend running different versions in the same cluster when the release dates are more than one year. While we do not expect that you will have data loss, the cluster may become unusable. The issues that you should expect if you have more than one year difference in versions include:
+
+- the cluster may not work
+- some (or even all) queries may fail with arbitrary errors
+- arbitrary errors/warnings may appear in the logs
+- it may be impossible to downgrade
+
+### Incremental upgrades
+
+If the difference between the current version and the target version is more than one year, then it is recommended to either:
+- Upgrade with downtime (stop all servers, upgrade all servers, run all servers).
+- Or to upgrade through an intermediate version (a version less than one year more recent than the current version).
+
+
+
+## Recommended plan
+
+These are the recommended steps for a zero-downtime ClickHouse upgrade:
+
+1. Make sure that your configuration changes are not in the default `/etc/clickhouse-server/config.xml` file and that they are instead in `/etc/clickhouse-server/config.d/`, as `/etc/clickhouse-server/config.xml` could be overwritten during an upgrade.
+2. Read through the [changelogs](/docs/en/whats-new/changelog/index.md) for breaking changes (going back from the target release to the release you are currently on).
+3. Make any updates identified in the breaking changes that can be made before upgrading, and a list of the changes that will need to be made after the upgrade.
+4. Identify one or more replicas for each shard to keep up while the rest of the replicas for each shard are upgraded.
+5. On the replicas that will be upgraded, one at a time:
+   - shutdown ClickHouse server
+   - upgrade the server to the target version
+   - bring ClickHouse server up
+   - wait for the Keeper messages to indicate that the system is stable
+   - continue to the next replica
+6. Check for errors in the Keeper log and the ClickHouse log
+7. Upgrade the replicas identified in step 4 to the new version
+8. Refer to the list of changes made in steps 1 through 3 and make the changes that need to be made after the upgrade.
+
+:::note
+This error message is expected when there are multiple versions of ClickHouse running in a replicated environment.  You will stop seeing these when all replicas are upgraded to the same version.
+```
+MergeFromLogEntryTask: Code: 40. DB::Exception: Checksums of parts don't match:
+hash of uncompressed files doesn't match. (CHECKSUM_DOESNT_MATCH)  Data after merge is not
+byte-identical to data on another replicas.
+```
+:::
+
+
+## ClickHouse server binary upgrade process
 
 If ClickHouse was installed from `deb` packages, execute the following commands on the server:
 

docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md

@@ -1,6 +1,7 @@
 ---
 slug: /en/sql-reference/aggregate-functions/reference/exponentialmovingaverage
 sidebar_position: 108
+sidebar_title: exponentialMovingAverage
 ---
 
 ## exponentialMovingAverage

docs/en/sql-reference/statements/alter/update.md

@@ -7,7 +7,7 @@ sidebar_label: UPDATE
 # ALTER TABLE … UPDATE Statements
 
 ``` sql
-ALTER TABLE [db.]table [ON CLUSTER cluster] UPDATE column1 = expr1 [, ...] WHERE filter_expr
+ALTER TABLE [db.]table [ON CLUSTER cluster] UPDATE column1 = expr1 [, ...] [IN PARTITION partition_id] WHERE filter_expr
 ```
 
 Manipulates data matching the specified filtering expression. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).

docs/en/sql-reference/statements/create/settings-profile.md

@@ -10,7 +10,7 @@ Creates [settings profiles](../../../operations/access-rights.md#settings-profil
 Syntax:
 
 ``` sql
-CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1]
+CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
         [, name2 [ON CLUSTER cluster_name2] ...]
     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...]
 ```

docs/ru/sql-reference/statements/create/settings-profile.md

@@ -11,7 +11,7 @@ sidebar_label: "Профиль настроек"
 Синтаксис:
 
 ``` sql
-CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1]
+CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
         [, name2 [ON CLUSTER cluster_name2] ...]
     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...]
 ```
@@ -26,4 +26,4 @@ CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluste
 CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin
 ```
 
-<!--hide-->
+<!--hide-->

programs/disks/CMakeLists.txt

@@ -1,4 +1,15 @@
-set (CLICKHOUSE_DISKS_SOURCES DisksApp.cpp ICommand.cpp)
+set (CLICKHOUSE_DISKS_SOURCES
+    DisksApp.cpp
+    ICommand.cpp
+    CommandCopy.cpp
+    CommandLink.cpp
+    CommandList.cpp
+    CommandListDisks.cpp
+    CommandMkDir.cpp
+    CommandMove.cpp
+    CommandRead.cpp
+    CommandRemove.cpp
+    CommandWrite.cpp)
 
 set (CLICKHOUSE_DISKS_LINK
     PRIVATE

programs/disks/CommandCopy.cpp

@@ -1,7 +1,6 @@
-#pragma once
-
 #include "ICommand.h"
 #include <Interpreters/Context.h>
+#include <Common/TerminalSize.h>
 
 namespace DB
 {
@@ -11,7 +10,7 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
-class CommandCopy : public ICommand
+class CommandCopy final : public ICommand
 {
 public:
     CommandCopy()
@@ -51,16 +50,16 @@ class CommandCopy : public ICommand
         String disk_name_from = config.getString("diskFrom", config.getString("disk", "default"));
         String disk_name_to = config.getString("diskTo", config.getString("disk", "default"));
 
-        String path_from = command_arguments[0];
-        String path_to =  command_arguments[1];
+        const String & path_from = command_arguments[0];
+        const String & path_to =  command_arguments[1];
 
         DiskPtr disk_from = global_context->getDisk(disk_name_from);
         DiskPtr disk_to = global_context->getDisk(disk_name_to);
 
-        String full_path_from = fullPathWithValidate(disk_from, path_from);
-        String full_path_to = fullPathWithValidate(disk_to, path_to);
+        String relative_path_from = validatePathAndGetAsRelative(path_from);
+        String relative_path_to = validatePathAndGetAsRelative(path_to);
 
-        disk_from->copy(full_path_from, disk_to, full_path_to);
+        disk_from->copy(relative_path_from, disk_to, relative_path_to);
     }
 };
 }