diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3557610 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +.* export-ignore +/examples export-ignore +/tests export-ignore +*.dist export-ignore +include.php export-ignore diff --git a/.gitignore b/.gitignore index 1bbc0b8..80d8ca3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ composer.lock vendor/ var/ +phpunit +temp/ \ No newline at end of file diff --git a/.scrutinizer.yml b/.scrutinizer.yml index d0c415a..efb246b 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -3,7 +3,7 @@ build: analysis: environment: php: - version: 7.1 + version: 7.3 cache: disabled: false directories: diff --git a/.travis.yml b/.travis.yml index 3c8fd89..a011414 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: bionic language: php sudo: false @@ -7,8 +7,9 @@ cache: - $HOME/.composer/cache php: - - 7.1 - - 7.2 + - 7.3 + - 7.4 + - 8.0 - nightly services: @@ -36,7 +37,19 @@ jobs: - stage: Test env: LOWEST_DEPENDENCIES - php: 7.2 + php: 7.3 + install: + - travis_retry composer update -n --prefer-dist --prefer-lowest + + - stage: Test + env: LOWEST_DEPENDENCIES + php: 7.4 + install: + - travis_retry composer update -n --prefer-dist --prefer-lowest + + - stage: Test + env: LOWEST_DEPENDENCIES + php: 8.0 install: - travis_retry composer update -n --prefer-dist --prefer-lowest @@ -55,7 +68,7 @@ jobs: - stage: Test env: COVERAGE - php: 7.1 + php: 7.3 before_script: - mv ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/xdebug.ini{.disabled,} - if [[ ! $(php -m | grep -si xdebug) ]]; then echo "xdebug required for coverage"; exit 1; fi @@ -68,7 +81,7 @@ jobs: - stage: Code Quality if: type = pull_request env: PULL_REQUEST_CODING_STANDARD - php: 7.1 + php: 7.3 install: travis_retry composer install --prefer-dist script: - | diff --git a/CHANGELOG.md b/CHANGELOG.md index eb32220..8d9558f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,110 @@ PHP ClickHouse wrapper - Changelog + + + ====================== + +### 2025-01-14 [Release 1.6.0] +* Support PHP 8.4 + +### 2024-01-18 [Release 1.5.3] +* Fix release 1.5.2 +* Support php 7 +* Update Statement.php #204 +* fix(#202): Fix converting boolean when inserting into int and fix(#194): Fix unexpected readonly mode with specific string in query #203 +* Update README.md #199 +* remove dev files for --prefer-dist #192 + + + +### 2024-01-16 [Release 1.5.2] +* Update Statement.php #204 +* fix(#202): Fix converting boolean when inserting into int and fix(#194): Fix unexpected readonly mode with specific string in query #203 +* Update README.md #199 +* remove dev files for --prefer-dist #192 + +### May 25, 2023 [ 1.5.1 ] +* BREAKING CHANGES Post type bindings support + +### 2022-12-20 [Release 1.5.0] + +* Change exceptionCode in Clickhouse version 22.8.3.13 (official build) #180 +* Fix Docker for tests, Change the correct Docker image name #177 +* Some type fix +* Fix types: max_execution_time & setConnectTimeOut, undo: Support floats in timeout and connect_timeout #173 +* add mbstring to composer require #183 +* fixed progressFunction #182 +* Add allow_plugins setting since Composer 2.2.x #178 + + +### 2022-04-23 [Release 1.4.4] +* Fix ping() for windows users +* ping(true) throw TransportException if can`t connect/ping + +### 2022-04-20 [Release 1.4.3] +* Fix: prevent enable_http_compression parameter from being overridden #164 +* For correct work with utf-8 . I am working on server with PHP 5.6.40 Update CurlerRequest.php #158 +* Add curl setStdErrOut, for custom StdOutErr. +* Fix some test for check exceptions + +### 2022-02-11 [Release 1.4.2] +* Fixed issue with non-empty raw data processing during init() on every fetchRow() and fetchOne() call - PR #161 + +### 2021-01-19 [Release 1.4.1] +* Add support php 7.3 & php 8 + + +### 2019-09-29 [Release 1.3.10] +* Add two new types of authentication #139 +* Fixed typo in streamRead exception text #140 +* fix the exception(multi-statement not allow) when sql end with ';' #138 +* Added more debug info for empty response with error #135 + + + +### 2020-02-03 [Release 1.3.9] +* #134 Enhancement: Add a new exception to be able to distinguish that ClickHouse is not available. + + +### 2020-01-17 [Release 1.3.8] +* #131 Fix: async loop breaks after 20 seconds +* #129 Add client certificate support to able to work with Yandex ClickHouse cloud hosting +* Delete `dropOldPartitions` +* Fix error : The error of array saving #127 +* More test + +### 2019-09-20 [Release 1.3.7] +* #125 WriteToFile: support for JSONEachRow format + +### 2019-08-24 [Release 1.3.6] +* #122 Add function fetchRow() +* Use X-ClickHouse-User by headers +* Add setDirtyCurler() in HTTP +* Add more tests + + + +### 2019-04-29 [Release 1.3.5] +* Reupload 1.3.4 + +### 2019-04-29 [Release 1.3.4] +* #118 Fix Error in Conditions & more ConditionsTest +* Fix phpStan warnings in getConnectTimeOut() & max_execution_time() + + + +### 2019-04-25 [Release 1.3.3] +* fix clickhouse release 19.5.2.6 error parsing #117 +* chore(Travis CI): Enable PHP 7.3 testing #116 + + +### 2019-03-18 [Release 1.3.2] +* fix: add CSVWithNames to supported formats #107 +* Upgraded Expression proposal #106 -> UUIDStringToNum +* Correct query format parsing #108 +* Can not use numbers() function in read requests #110 +* #109 Do not use keep-alive or reuse them across requests + ### 2018-09-25 [Release 1.3.1] * Pull request #94 from simPod: Uint64 values * Pull request #95 from simPod: Bump to php 7.1 diff --git a/README.md b/README.md index c204d2b..0a15fb3 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,9 @@ PHP ClickHouse wrapper ====================== -[![Build Status](https://travis-ci.org/smi2/phpClickHouse.svg)](https://travis-ci.org/smi2/phpClickHouse) [![Downloads](https://poser.pugx.org/smi2/phpClickHouse/d/total.svg)](https://packagist.org/packages/smi2/phpClickHouse) [![Packagist](https://poser.pugx.org/smi2/phpClickHouse/v/stable.svg)](https://packagist.org/packages/smi2/phpClickHouse) [![Licence](https://poser.pugx.org/smi2/phpClickHouse/license.svg)](https://packagist.org/packages/smi2/phpClickHouse) -[![Quality Score](https://scrutinizer-ci.com/g/smi2/phpClickHouse/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/smi2/phpClickHouse) -[![Code Coverage](https://scrutinizer-ci.com/g/smi2/phpClickHouse/badges/coverage.png?b=master)](https://scrutinizer-ci.com/g/smi2/phpClickHouse) ## Features @@ -37,14 +34,18 @@ composer require smi2/phpclickhouse In php ```php + // vendor autoload $db = new ClickHouseDB\Client(['config_array']); -$db->ping(); + +if (!$db->ping()) echo 'Error connect'; ``` -Last stable version for: -- php 5.6 = `1.1.2` -- php 7.0 = `1.2.4` +Last stable version for +* php 5.6 <= `1.1.2` +* php 7.2 <= `1.3.10` +* php 7.3 >= `1.4.x ... 1.5.X` +* php 8.4 >= `1.6.0` [Packagist](https://packagist.org/packages/smi2/phpclickhouse) @@ -57,14 +58,15 @@ $config = [ 'host' => '192.168.1.1', 'port' => '8123', 'username' => 'default', - 'password' => '' + 'password' => '', + 'https' => true ]; $db = new ClickHouseDB\Client($config); $db->database('default'); -$db->setTimeout(1.5); // 1500 ms +$db->setTimeout(1.5); // 1 second , support only Int value $db->setTimeout(10); // 10 seconds $db->setConnectTimeOut(5); // 5 seconds - +$db->ping(true); // if can`t connect throw exception ``` Show tables: @@ -156,6 +158,14 @@ print_r($statement->info()); // if clickhouse-server version >= 54011 $db->settings()->set('output_format_write_statistics',true); print_r($statement->statistics()); + + +// Statement Iterator +$state=$this->client->select('SELECT (number+1) as nnums FROM system.numbers LIMIT 5'); +foreach ($state as $key=>$value) { + echo $value['nnums']; +} + ``` Select result as tree: @@ -311,16 +321,6 @@ $count_result = 2; print_r($db->partitions('summing_partions_views', $count_result)); ``` -Drop partitions ( pre production ) - -```php -$count_old_days = 10; -print_r($db->dropOldPartitions('summing_partions_views', $count_old_days)); - -// by `partition_id` -print_r($db->dropPartition('summing_partions_views', '201512')); -``` - ### Select WHERE IN ( _local csv file_ ) ```php @@ -347,7 +347,7 @@ $Bindings = [ 'from_table' => 'table' ]; -$statement = $db->selectAsync("SELECT FROM {table} WHERE datetime=:datetime limit {limit}", $Bindings); +$statement = $db->selectAsync("SELECT FROM {from_table} WHERE datetime=:datetime limit {limit}", $Bindings); // Double bind in {KEY} $keys=[ @@ -521,82 +521,20 @@ Class for FormatLine array ```php var_dump( - \ClickHouseDB\FormatLine::CSV( + ClickHouseDB\Quote\FormatLine::CSV( ['HASH1', ["a", "dddd", "xxx"]] ) ); var_dump( - \ClickHouseDB\FormatLine::TSV( + ClickHouseDB\Quote\FormatLine::TSV( ['HASH1', ["a", "dddd", "xxx"]] ) ); // example write to file $row=['event_time'=>date('Y-m-d H:i:s'),'arr1'=>[1,2,3],'arrs'=>["A","B\nD\nC"]]; -file_put_contents($fileName,\ClickHouseDB\FormatLine::TSV($row)."\n",FILE_APPEND); -``` - -### Cluster drop old Partitions - -Example code : - -```php -class my -{ - /** - * @return \ClickHouseDB\Cluster - */ - public function getClickHouseCluster() - { - return $this->_cluster; - } - - public function msg($text) - { - echo $text."\n"; - } - - private function cleanTable($dbt) - { - - $sizes=$this->getClickHouseCluster()->getSizeTable($dbt); - $this->msg("Clean table : $dbt,size = ".$this->humanFileSize($sizes)); - - // split string "DB.TABLE" - list($db,$table)=explode('.',$dbt); - - // Get Master node for table - $nodes=$this->getClickHouseCluster()->getMasterNodeForTable($dbt); - foreach ($nodes as $node) - { - $client=$this->getClickHouseCluster()->client($node); - - $size=$client->database($db)->tableSize($table); - - $this->msg("$node \t {$size['size']} \t {$size['min_date']} \t {$size['max_date']}"); - - $client->dropOldPartitions($table,30,30); - } - } - - public function clean() - { - $this->msg("clean"); - - $this->getClickHouseCluster()->setScanTimeOut(2.5); // 2500 ms - $this->getClickHouseCluster()->setSoftCheck(true); - if (!$this->getClickHouseCluster()->isReplicasIsOk()) - { - throw new Exception('Replica state is bad , error='.$this->getClickHouseCluster()->getError()); - } - - $this->cleanTable('model.history_full_model_sharded'); - - $this->cleanTable('model.history_model_result_sharded'); - } -} - +file_put_contents($fileName,ClickHouseDB\Quote\FormatLine::TSV($row)."\n",FILE_APPEND); ``` ### HTTPS @@ -741,6 +679,26 @@ $r=$client->streamRead($streamRead,'SELECT sin(number) as sin,cos(number) as cos $db->insertAssocBulk([$oneRow, $oneRow, $failRow]) ``` +### Auth methods + +``` + AUTH_METHOD_HEADER = 1; + AUTH_METHOD_QUERY_STRING = 2; + AUTH_METHOD_BASIC_AUTH = 3; +``` + +In config set `auth_method` + +```php +$config=[ + 'host'=>'host.com', + //... + 'auth_method'=>1, +]; + +``` + + ### progressFunction ```php @@ -760,6 +718,14 @@ $st=$db->select('SELECT number,sleep(0.2) FROM system.numbers limit 5'); ``` +### ssl CA +```php +$config = [ + 'host' => 'cluster.clickhouse.dns.com', // any node name in cluster + 'port' => '8123', + 'sslCA' => '...', +]; +``` ### Cluster @@ -822,7 +788,7 @@ $cli->ping(); $result=$cl->truncateTable('dbNane.TableName_sharded'); // get one active node ( random ) -$cl->activeClient()->setTimeout(0.01); +$cl->activeClient()->setTimeout(500); $cl->activeClient()->write("DROP TABLE IF EXISTS default.asdasdasd ON CLUSTER cluster2"); @@ -868,6 +834,26 @@ $db->verbose(); ``` +Verbose to file|steam: + +```php + // init client + $cli = new Client($config); + $cli->verbose(); + // temp stream + $stream = fopen('php://memory', 'r+'); + // set stream to curl + $cli->transport()->setStdErrOut($stream); + // exec curl + $st=$cli->select('SElect 1 as ppp'); + $st->rows(); + // rewind + fseek($stream,0,SEEK_SET); + + // output + echo stream_get_contents($stream); +``` + ### Dev & PHPUnit Test @@ -892,6 +878,11 @@ Edit in phpunit.xml constants: ``` +Run docker ClickHouse server +``` +cd ./tests +docker-compose up +``` Run test ```bash @@ -900,6 +891,10 @@ Run test ./vendor/bin/phpunit --group ClientTest +./vendor/bin/phpunit --group ClientTest --filter testInsertNestedArray + +./vendor/bin/phpunit --group ConditionsTest + ``` diff --git a/composer.json b/composer.json index 155b7fd..edd3a4c 100644 --- a/composer.json +++ b/composer.json @@ -13,14 +13,15 @@ } ], "require": { - "php": "^7.1", - "ext-curl": "*" + "php": "^8.0", + "ext-curl": "*", + "ext-json": "*" }, "require-dev": { - "doctrine/coding-standard": "^5.0", - "phpstan/phpstan": "^0.10.3", - "phpunit/phpunit": "^7", - "sebastian/comparator": "~3.0" + "doctrine/coding-standard": "^8.2", + "phpstan/phpstan": "^0.12", + "phpunit/phpunit": "^9.5", + "sebastian/comparator": "^4.0" }, "autoload": { "psr-4": { @@ -32,5 +33,10 @@ "ClickHouseDB\\Tests\\": "tests/", "ClickHouseDB\\Example\\": "example/" } + }, + "config": { + "allow-plugins": { + "dealerdirect/phpcodesniffer-composer-installer": true + } } } diff --git a/example/00_config_connect.php b/example/00_config_connect.php index 8c1acff..cd4d637 100644 --- a/example/00_config_connect.php +++ b/example/00_config_connect.php @@ -1,7 +1,8 @@ 'tabix.dev7', // you hot name + 'host' => '127.0.0.1', // you hot name 'port' => '8123', 'username' => 'default', - 'password' => '' + 'password' => '', + 'auth_method' => 1, // On of HTTP::AUTH_METHODS_LIST ]; \ No newline at end of file diff --git a/example/exam04_sql_conditions.php b/example/exam04_sql_conditions.php index 6a53e2a..2804361 100644 --- a/example/exam04_sql_conditions.php +++ b/example/exam04_sql_conditions.php @@ -16,6 +16,22 @@ $db->enableQueryConditions(); +$select='SELECT {ifint lastdays} + + event_date>=today()-{lastdays} + + {else} + + event_date=today() + + {/if}'; + + +$statement = $db->selectAsync($select, $input_params); +echo $statement->sql(); +echo "\n"; + + $select = ' SELECT * FROM {from_table} @@ -49,6 +65,100 @@ echo $statement->sql(); echo "\n"; + + + + + + +$select=" +1: {if ll}NOT_SHOW{else}OK{/if}{if ll}NOT_SHOW{else}OK{/if} +2: {if null}NOT_SHOW{else}OK{/if} +3: {if qwert}NOT_SHOW{/if} +4: {ifset zero} NOT_SHOW {else}OK{/if} +5: {ifset false} NOT_SHOW {/if} +6: {ifset s_false} OK {/if} +7: {ifint zero} NOT_SHOW {/if} +8: {if zero}OK{/if} +9: {ifint s_empty}NOT_SHOW{/if} +0: {ifint s_null}NOT_SHOW{/if} +1: {ifset null} NOT_SHOW {/if} + + +INT: +0: {ifint zero} NOT_SHOW {/if} +1: {ifint int1} OK {/if} +2: {ifint int30} OK {/if} +3: {ifint int30}OK {else} NOT_SHOW {/if} +4: {ifint str0} NOT_SHOW {else}OK{/if} +5: {ifint str1} OK {else} NOT_SHOW {/if} +6: {ifint int30} OK {else} NOT_SHOW {/if} +7: {ifint s_empty} NOT_SHOW {else} OK {/if} +8: {ifint true} OK {else} NOT_SHOW {/if} + +STRING: +0: {ifstring s_empty}NOT_SHOW{else}OK{/if} +1: {ifstring s_null}OK{else}NOT_SHOW{/if} + +BOOL: +1: {ifbool int1}NOT_SHOW{else}OK{/if} +2: {ifbool int30}NOT_SHOW{else}OK{/if} +3: {ifbool zero}NOT_SHOW{else}OK{/if} +4: {ifbool false}NOT_SHOW{else}OK{/if} +5: {ifbool true}OK{else}NOT_SHOW{/if} +5: {ifbool true}OK{/if} +6: {ifbool false}OK{/if} + + +0: {if s_empty} + + +SHOW + + +{/if} + +{ifint lastdays} + + + event_date>=today()-{lastdays}-{lastdays}-{lastdays} + + +{else} + + + event_date>=today() + + +{/if} +"; + +// +//$select='{ifint s_empty}NOT_SHOW{/if} +//1: {ifbool int1}NOT_SHOW{else}OK{/if} +//2: {ifbool int30}NOT_SHOW{else}OK{/if} +// +//'; + +$input_params=[ + 'lastdays'=>3, + 'null'=>null, + 'false'=>false, + 'true'=>true, + 'zero'=>0, + 's_false'=>'false', + 's_null'=>'null', + 's_empty'=>'', + 'int30'=>30, + 'int1'=>1, + 'str0'=>'0', + 'str1'=>'1' +]; +$statement = $db->selectAsync($select, $input_params); +echo "\n------------------------------------\n"; +echo $statement->sql(); +echo "\n"; + /* SELECT * FROM table WHERE diff --git a/example/exam12_array.php b/example/exam12_array.php index 3db971f..8655d67 100644 --- a/example/exam12_array.php +++ b/example/exam12_array.php @@ -149,4 +149,26 @@ $st=$db->select('SELECT round(sum(flos),5),sum(ints) FROM testTABWrite'); print_r($st->rows()); -// \ No newline at end of file +// +$db->write("DROP TABLE IF EXISTS NestedNested_arr"); + +$res = $db->write(' + CREATE TABLE IF NOT EXISTS NestedNested_arr ( + s_key String, + s_arr Array(String) + ) ENGINE = Memory +'); + +//------------------------------------------------------------------------------ + +$XXX=['AAA'."'".'A',"BBBBB".'\\']; + +print_r($XXX); + +echo "Insert\n"; +$stat = $db->insert('NestedNested_arr', [ + ['HASH\1', $XXX], +], ['s_key','s_arr']); +echo "Insert Done\n"; + +print_r($db->select('SELECT * FROM NestedNested_arr WHERE s_key=\'HASH\1\'')->rows()); diff --git a/example/exam13_nested.php b/example/exam13_nested.php new file mode 100644 index 0000000..c3c17f4 --- /dev/null +++ b/example/exam13_nested.php @@ -0,0 +1,35 @@ +write("DROP TABLE IF EXISTS NestedNested_test"); + +$res = $db->write(' + CREATE TABLE IF NOT EXISTS NestedNested_test ( + s_key String, + topics Nested( id UInt8 , ww Float32 ), + s_arr Array(String) + ) ENGINE = Memory +'); + +//------------------------------------------------------------------------------ + +$XXX=['AAA'."'".'A',"BBBBB".'\\']; + +print_r($XXX); + +echo "Insert\n"; +$stat = $db->insert('NestedNested_test', [ + ['HASH\1', [11,33],[3.2,2.1],$XXX], +], ['s_key', 'topics.id','topics.ww','s_arr']); +echo "Insert Done\n"; + +print_r($db->select('SELECT * FROM NestedNested_test')->rows()); +print_r($db->select('SELECT * FROM NestedNested_test ARRAY JOIN topics')->rows()); + diff --git a/include.php b/include.php index f731363..eb79fd0 100644 --- a/include.php +++ b/include.php @@ -9,6 +9,7 @@ include_once __DIR__ . '/src/Exception/QueryException.php'; include_once __DIR__ . '/src/Exception/DatabaseException.php'; include_once __DIR__ . '/src/Exception/TransportException.php'; +include_once __DIR__ . '/src/Exception/ClickHouseUnavailableException.php'; // Client include_once __DIR__ . '/src/Statement.php'; include_once __DIR__ . '/src/Client.php'; @@ -21,7 +22,6 @@ include_once __DIR__ . '/src/Query/WriteToFile.php'; include_once __DIR__ . '/src/Query/WhereInFile.php'; include_once __DIR__ . '/src/Query/Query.php'; -include_once __DIR__ . '/src/Query/Expression.php'; // Transport include_once __DIR__ . '/src/Transport/Http.php'; include_once __DIR__ . '/src/Transport/CurlerRolling.php'; diff --git a/src/Client.php b/src/Client.php index 2fbdcdc..9cb841e 100644 --- a/src/Client.php +++ b/src/Client.php @@ -5,6 +5,7 @@ namespace ClickHouseDB; use ClickHouseDB\Exception\QueryException; +use ClickHouseDB\Exception\TransportException; use ClickHouseDB\Query\Degeneration; use ClickHouseDB\Query\Degeneration\Bindings; use ClickHouseDB\Query\Degeneration\Conditions; @@ -31,9 +32,13 @@ use function strtotime; use function trim; +/** + * Class Client + * @package ClickHouseDB + */ class Client { - const SUPPORTED_FORMATS = ['TabSeparated', 'TabSeparatedWithNames', 'CSV', 'CSVWithNames', 'JSONEachRow']; + const SUPPORTED_FORMATS = ['TabSeparated', 'TabSeparatedWithNames', 'CSV', 'CSVWithNames', 'JSONEachRow','CSVWithNamesAndTypes','TSVWithNamesAndTypes']; /** @var Http */ private $transport; @@ -47,9 +52,12 @@ class Client /** @var string */ private $connectHost; - /** @var string */ + /** @var int */ private $connectPort; + /** @var int */ + private $authMethod; + /** @var bool */ private $connectUserReadonly = false; @@ -59,40 +67,53 @@ class Client */ public function __construct(array $connectParams, array $settings = []) { - if (! isset($connectParams['username'])) { - throw new \InvalidArgumentException('not set username'); + if (!isset($connectParams['username'])) { + throw new \InvalidArgumentException('not set username'); } - if (! isset($connectParams['password'])) { - throw new \InvalidArgumentException('not set password'); + if (!isset($connectParams['password'])) { + throw new \InvalidArgumentException('not set password'); } - if (! isset($connectParams['port'])) { - throw new \InvalidArgumentException('not set port'); + if (!isset($connectParams['port'])) { + throw new \InvalidArgumentException('not set port'); } - if (! isset($connectParams['host'])) { - throw new \InvalidArgumentException('not set host'); + if (!isset($connectParams['host'])) { + throw new \InvalidArgumentException('not set host'); + } + + if (array_key_exists('auth_method', $connectParams)) { + if (false === in_array($connectParams['auth_method'], Http::AUTH_METHODS_LIST)) { + $errorMessage = sprintf( + 'Invalid value for "auth_method" param. Should be one of [%s].', + json_encode(Http::AUTH_METHODS_LIST) + ); + throw new \InvalidArgumentException($errorMessage); + } + + $this->authMethod = $connectParams['auth_method']; } $this->connectUsername = $connectParams['username']; $this->connectPassword = $connectParams['password']; - $this->connectPort = $connectParams['port']; - $this->connectHost = $connectParams['host']; + $this->connectPort = intval($connectParams['port']); + $this->connectHost = $connectParams['host']; // init transport class $this->transport = new Http( $this->connectHost, $this->connectPort, $this->connectUsername, - $this->connectPassword + $this->connectPassword, + $this->authMethod ); $this->transport->addQueryDegeneration(new Bindings()); // apply settings to transport class $this->settings()->database('default'); - if (! empty($settings)) { + if (!empty($settings)) { $this->settings()->apply($settings); } @@ -104,7 +125,9 @@ public function __construct(array $connectParams, array $settings = []) $this->https($connectParams['https']); } - $this->enableHttpCompression(); + if (isset($connectParams['sslCA'])) { + $this->transport->setSslCa($connectParams['sslCA']); + } } /** @@ -141,7 +164,7 @@ public function addQueryDegeneration(Degeneration $degeneration) * * @return bool */ - public function enableQueryConditions() + public function enableQueryConditions(): bool { return $this->transport->addQueryDegeneration(new Conditions()); } @@ -151,24 +174,24 @@ public function enableQueryConditions() * * @param string $host */ - public function setHost($host) + public function setHost($host): void { $this->connectHost = $host; $this->transport()->setHost($host); } /** - * @return Settings + * max_execution_time , in int value (seconds) */ - public function setTimeout(float $timeout) + public function setTimeout($timeout): Settings { - return $this->settings()->max_execution_time($timeout); + return $this->settings()->max_execution_time(intval($timeout)); } /** - * @return mixed + * @return int */ - public function getTimeout() + public function getTimeout(): int { return $this->settings()->getTimeOut(); } @@ -176,15 +199,15 @@ public function getTimeout() /** * ConnectTimeOut in seconds ( support 1.5 = 1500ms ) */ - public function setConnectTimeOut(float $connectTimeOut) + public function setConnectTimeOut(float $connectTimeOut): void { $this->transport()->setConnectTimeOut($connectTimeOut); } /** - * @return int + * @return float */ - public function getConnectTimeOut() + public function getConnectTimeOut(): float { return $this->transport()->getConnectTimeOut(); } @@ -192,10 +215,10 @@ public function getConnectTimeOut() /** * @return Http */ - public function transport() + public function transport(): Http { - if (! $this->transport) { - throw new \InvalidArgumentException('Empty transport class'); + if (!$this->transport) { + throw new \InvalidArgumentException('Empty transport class'); } return $this->transport; @@ -204,7 +227,7 @@ public function transport() /** * @return string */ - public function getConnectHost() + public function getConnectHost(): string { return $this->connectHost; } @@ -212,7 +235,7 @@ public function getConnectHost() /** * @return string */ - public function getConnectPassword() + public function getConnectPassword(): string { return $this->connectPassword; } @@ -220,9 +243,9 @@ public function getConnectPassword() /** * @return string */ - public function getConnectPort() + public function getConnectPort(): string { - return $this->connectPort; + return strval($this->connectPort); } /** @@ -233,6 +256,14 @@ public function getConnectUsername() return $this->connectUsername; } + /** + * @return int + */ + public function getAuthMethod(): int + { + return $this->authMethod; + } + /** * @return Http */ @@ -242,9 +273,9 @@ public function getTransport() } /** - * @return mixed + * @return bool */ - public function verbose() + public function verbose(bool $flag = true):bool { return $this->transport()->verbose(true); } @@ -258,18 +289,18 @@ public function settings() } /** - * @return static + * @param string|null $useSessionId + * @return $this */ - public function useSession(bool $useSessionId = false) + public function useSession(string $useSessionId = '') { - if (! $this->settings()->getSessionId()) { - if (! $useSessionId) { + if (!$this->settings()->getSessionId()) { + if (!$useSessionId) { $this->settings()->makeSessionId(); } else { $this->settings()->session_id($useSessionId); } } - return $this; } @@ -310,7 +341,7 @@ public function database(string $db) */ public function enableLogQueries(bool $flag = true) { - $this->settings()->set('log_queries', (int) $flag); + $this->settings()->set('log_queries', (int)$flag); return $this; } @@ -346,21 +377,23 @@ public function https(bool $flag = true) */ public function enableExtremes(bool $flag = true) { - $this->settings()->set('extremes', (int) $flag); + $this->settings()->set('extremes', (int)$flag); return $this; } /** - * @param mixed[] $bindings + * @param string $sql + * @param array $bindings * @return Statement */ public function select( string $sql, array $bindings = [], - WhereInFile $whereInFile = null, - WriteToFile $writeToFile = null - ) { + ?WhereInFile $whereInFile = null, + ?WriteToFile $writeToFile = null + ) + { return $this->transport()->select($sql, $bindings, $whereInFile, $writeToFile); } @@ -382,14 +415,14 @@ public function maxTimeExecutionAllAsync() */ public function progressFunction(callable $callback) { - if (! is_callable($callback)) { + if (!is_callable($callback)) { throw new \InvalidArgumentException('Not is_callable progressFunction'); } - if (! $this->settings()->is('send_progress_in_http_headers')) { + if (!$this->settings()->is('send_progress_in_http_headers')) { $this->settings()->set('send_progress_in_http_headers', 1); } - if (! $this->settings()->is('http_headers_progress_interval_ms')) { + if (!$this->settings()->is('http_headers_progress_interval_ms')) { $this->settings()->set('http_headers_progress_interval_ms', 100); } @@ -405,9 +438,10 @@ public function progressFunction(callable $callback) public function selectAsync( string $sql, array $bindings = [], - WhereInFile $whereInFile = null, - WriteToFile $writeToFile = null - ) { + ?WhereInFile $whereInFile = null, + ?WriteToFile $writeToFile = null + ) + { return $this->transport()->selectAsync($sql, $bindings, $whereInFile, $writeToFile); } @@ -463,11 +497,11 @@ public function getCountPendingQueue() /** * @param mixed[][] $values - * @param string[] $columns + * @param string[] $columns * @return Statement * @throws Exception\TransportException */ - public function insert(string $table, array $values, array $columns = []) : Statement + public function insert(string $table, array $values, array $columns = []): Statement { if (empty($values)) { throw QueryException::cannotInsertEmptyValues(); @@ -493,12 +527,12 @@ public function insert(string $table, array $values, array $columns = []) : Stat } /** - *       * Prepares the values to insert from the associative array. - *       * There may be one or more lines inserted, but then the keys inside the array list must match (including in the sequence) - *       * - *       * @param mixed[] $values - array column_name => value (if we insert one row) or array list column_name => value if we insert many lines - *       * @return mixed[][] - list of arrays - 0 => fields, 1 => list of value arrays for insertion - *       */ + * Prepares the values to insert from the associative array. + * There may be one or more lines inserted, but then the keys inside the array list must match (including in the sequence) + * + * @param mixed[] $values - array column_name => value (if we insert one row) or array list column_name => value if we insert many lines + * @return mixed[][] - list of arrays - 0 => fields, 1 => list of value arrays for insertion + **/ public function prepareInsertAssocBulk(array $values) { if (isset($values[0]) && is_array($values[0])) { //случай, когда много строк вставляется @@ -530,6 +564,7 @@ public function prepareInsertAssocBulk(array $values) * Inserts one or more rows from an associative array. * If there is a discrepancy between the keys of the value arrays (or their order) - throws an exception. * + * @param string $tableName - name table * @param mixed[] $values - array column_name => value (if we insert one row) or array list column_name => value if we insert many lines * @return Statement */ @@ -544,7 +579,7 @@ public function insertAssocBulk(string $tableName, array $values) * insert TabSeparated files * * @param string|string[] $fileNames - * @param string[] $columns + * @param string[] $columns * @return mixed */ public function insertBatchTSVFiles(string $tableName, $fileNames, array $columns = []) @@ -556,8 +591,8 @@ public function insertBatchTSVFiles(string $tableName, $fileNames, array $column * insert Batch Files * * @param string|string[] $fileNames - * @param string[] $columns - * @param string $format ['TabSeparated','TabSeparatedWithNames','CSV','CSVWithNames'] + * @param string[] $columns + * @param string $format ['TabSeparated','TabSeparatedWithNames','CSV','CSVWithNames'] * @return Statement[] * @throws Exception\TransportException */ @@ -570,15 +605,15 @@ public function insertBatchFiles(string $tableName, $fileNames, array $columns = throw new QueryException('Queue must be empty, before insertBatch, need executeAsync'); } - if (! in_array($format, self::SUPPORTED_FORMATS, true)) { + if (!in_array($format, self::SUPPORTED_FORMATS, true)) { throw new QueryException('Format not support in insertBatchFiles'); } $result = []; foreach ($fileNames as $fileName) { - if (! is_file($fileName) || ! is_readable($fileName)) { - throw new QueryException('Cant read file: ' . $fileName . ' ' . (is_file($fileName) ? '' : ' is not file')); + if (!is_file($fileName) || !is_readable($fileName)) { + throw new QueryException('Cant read file: ' . $fileName . ' ' . (is_file($fileName) ? '' : ' is not file')); } if (empty($columns)) { @@ -594,7 +629,7 @@ public function insertBatchFiles(string $tableName, $fileNames, array $columns = // fetch resutl foreach ($fileNames as $fileName) { - if (! $result[$fileName]->isError()) { + if (!$result[$fileName]->isError()) { continue; } @@ -608,7 +643,7 @@ public function insertBatchFiles(string $tableName, $fileNames, array $columns = * insert Batch Stream * * @param string[] $columns - * @param string $format ['TabSeparated','TabSeparatedWithNames','CSV','CSVWithNames'] + * @param string $format ['TabSeparated','TabSeparatedWithNames','CSV','CSVWithNames'] * @return Transport\CurlerRequest */ public function insertBatchStream(string $tableName, array $columns = [], string $format = 'CSV') @@ -617,7 +652,7 @@ public function insertBatchStream(string $tableName, array $columns = [], string throw new QueryException('Queue must be empty, before insertBatch, need executeAsync'); } - if (! in_array($format, self::SUPPORTED_FORMATS, true)) { + if (!in_array($format, self::SUPPORTED_FORMATS, true)) { throw new QueryException('Format not support in insertBatchFiles'); } @@ -655,7 +690,7 @@ public function streamWrite(Stream $stream, string $sql, array $bind = []) public function streamRead(Stream $streamRead, string $sql, array $bind = []) { if ($this->getCountPendingQueue() > 0) { - throw new QueryException('Queue must be empty, before streamWrite'); + throw new QueryException('Queue must be empty, before streamRead'); } return $this->transport()->streamRead($streamRead, $sql, $bind); @@ -665,6 +700,7 @@ public function streamRead(Stream $streamRead, string $sql, array $bind = []) * Size of database * * @return mixed|null + * @throws \Exception */ public function databaseSize() { @@ -685,6 +721,7 @@ public function databaseSize() * Size of tables * * @return mixed + * @throws \Exception */ public function tableSize(string $tableName) { @@ -700,11 +737,15 @@ public function tableSize(string $tableName) /** * Ping server * + * @param bool $throwException * @return bool + * @throws TransportException */ - public function ping() + public function ping(bool $throwException=false): bool { - return $this->transport()->ping(); + $result=$this->transport()->ping(); + if ($throwException && !$result) throw new TransportException('Can`t ping server'); + return $result; } /** @@ -712,6 +753,7 @@ public function ping() * * @param bool $flatList * @return mixed[][] + * @throws \Exception */ public function tablesSize($flatList = false) { @@ -732,7 +774,7 @@ public function tablesSize($flatList = false) FROM system.parts WHERE active AND database=:database GROUP BY table,database - ) USING ( table,database ) + ) as s USING ( table,database ) WHERE database=:database GROUP BY table,database ', @@ -749,6 +791,7 @@ public function tablesSize($flatList = false) * isExists * * @return array + * @throws \Exception */ public function isExists(string $database, string $table) { @@ -763,27 +806,33 @@ public function isExists(string $database, string $table) /** * List of partitions * - * @return mixed[][] + * @return array + * @throws \Exception */ - public function partitions(string $table, int $limit = null, bool $active = null) + public function partitions(string $table, int $limit = 0, ?bool $active = null) { - $database = $this->settings()->getDatabase(); - $whereActiveClause = $active === null ? '' : sprintf(' AND active = %s', (int) $active); - $limitClause = $limit !== null ? ' LIMIT ' . $limit : ''; + $database = $this->settings()->getDatabase(); + $whereActiveClause = $active === null ? '' : sprintf(' AND active = %s', (int)$active); + $limitClause = $limit > 0 ? ' LIMIT ' . $limit : ''; return $this->select(<<$database, + 'tbl'=>$table + ] )->rowsAsTree('name'); } /** * dropPartition - * @deprecated * @return Statement + * @deprecated */ public function dropPartition(string $dataBaseTableName, string $partition_id) { @@ -793,7 +842,7 @@ public function dropPartition(string $dataBaseTableName, string $partition_id) $state = $this->write('ALTER TABLE {dataBaseTableName} DROP PARTITION :partion_id', [ 'dataBaseTableName' => $dataBaseTableName, - 'partion_id' => $partition_id, + 'partion_id' => $partition_id, ]); return $state; @@ -801,15 +850,16 @@ public function dropPartition(string $dataBaseTableName, string $partition_id) /** * Truncate ( drop all partitions ) - * @deprecated * @return array + * @throws \Exception + * @deprecated */ public function truncateTable(string $tableName) { $partions = $this->partitions($tableName); - $out = []; + $out = []; foreach ($partions as $part_key => $part) { - $part_id = $part['partition']; + $part_id = $part['partition']; $out[$part_id] = $this->dropPartition($tableName, $part_id); } @@ -821,6 +871,7 @@ public function truncateTable(string $tableName) * * @return int * @throws Exception\TransportException + * @throws \Exception */ public function getServerUptime() { @@ -830,19 +881,20 @@ public function getServerUptime() /** * Returns string with the server version. */ - public function getServerVersion() : string + public function getServerVersion(): string { - return (string) $this->select('SELECT version() as version')->fetchOne('version'); + return (string)$this->select('SELECT version() as version')->fetchOne('version'); } /** * Read system.settings table * * @return mixed[][] + * @throws \Exception */ public function getServerSystemSettings(string $like = '') { - $l = []; + $l = []; $list = $this->select('SELECT * FROM system.settings' . ($like ? ' WHERE name LIKE :like' : ''), ['like' => '%' . $like . '%'])->rows(); foreach ($list as $row) { @@ -856,39 +908,4 @@ public function getServerSystemSettings(string $like = '') return $l; } - /** - * dropOldPartitions by day_ago - * @deprecated - * - * @return array - * @throws Exception\TransportException - * @throws \Exception - */ - public function dropOldPartitions(string $table_name, int $days_ago, int $count_partitons_per_one = 100) - { - $days_ago = strtotime(date('Y-m-d 00:00:00', strtotime('-' . $days_ago . ' day'))); - - $drop = []; - $list_patitions = $this->partitions($table_name, $count_partitons_per_one); - - foreach ($list_patitions as $partion_id => $partition) { - if (stripos($partition['engine'], 'mergetree') === false) { - continue; - } - - // $min_date = strtotime($partition['min_date']); - $max_date = strtotime($partition['max_date']); - - if ($max_date < $days_ago) { - $drop[] = $partition['partition']; - } - } - - $result = []; - foreach ($drop as $partition_id) { - $result[$partition_id] = $this->dropPartition($table_name, $partition_id); - } - - return $result; - } } diff --git a/src/Exception/ClickHouseUnavailableException.php b/src/Exception/ClickHouseUnavailableException.php new file mode 100644 index 0000000..a17b8ff --- /dev/null +++ b/src/Exception/ClickHouseUnavailableException.php @@ -0,0 +1,8 @@ +requestDetails = $requestDetails; + } + + public function getRequestDetails(): array + { + return $this->requestDetails; + } + + public function setResponseDetails(array $responseDetails) + { + $this->responseDetails = $responseDetails; + } + + public function getResponseDetails(): array + { + return $this->responseDetails; + } } diff --git a/src/Query/Degeneration.php b/src/Query/Degeneration.php index 08d144d..8ba02d0 100644 --- a/src/Query/Degeneration.php +++ b/src/Query/Degeneration.php @@ -5,4 +5,5 @@ interface Degeneration { public function process($sql); public function bindParams(array $bindings); + public function getBind():array; } \ No newline at end of file diff --git a/src/Query/Degeneration/Bindings.php b/src/Query/Degeneration/Bindings.php index c95be14..8baeda2 100644 --- a/src/Query/Degeneration/Bindings.php +++ b/src/Query/Degeneration/Bindings.php @@ -28,6 +28,11 @@ public function bindParams(array $bindings) } } + public function getBind(): array + { + return $this->bindings; + } + /** * @param string $column * @param mixed $value diff --git a/src/Query/Degeneration/Conditions.php b/src/Query/Degeneration/Conditions.php index b2c5c83..348f555 100644 --- a/src/Query/Degeneration/Conditions.php +++ b/src/Query/Degeneration/Conditions.php @@ -2,6 +2,7 @@ namespace ClickHouseDB\Query\Degeneration; +use ClickHouseDB\Exception\QueryException; use ClickHouseDB\Query\Degeneration; class Conditions implements Degeneration @@ -21,38 +22,46 @@ public function bindParams(array $bindings) } } + public function getBind(): array + { + return $this->bindings; + } - static function __ifsets($matches, $markers, $else = false) + static function __ifsets($matches, $markers) { $content_false = ''; - - if ($else) - { - list($condition, $preset, $variable, $content_true, $content_false) = $matches; - } else - { + $condition = ''; + $flag_else = ''; +//print_r($matches); + if (sizeof($matches) == 4) { list($condition, $preset, $variable, $content_true) = $matches; + } elseif (sizeof($matches) == 6) { + list($condition, $preset, $variable, $content_true, $flag_else, $content_false) = $matches; + } else { + throw new QueryException('Error in parse Conditions' . json_encode($matches)); } - $preset = strtolower($preset); + $variable = trim($variable); + $preset = strtolower(trim($preset)); - if ($preset == 'set') - { + if ($preset == '') { + return (isset($markers[$variable]) && ($markers[$variable] || is_numeric($markers[$variable]))) + ? $content_true + : $content_false; + } + if ($preset == 'set') { return (isset($markers[$variable]) && !empty($markers[$variable])) ? $content_true : $content_false; } - if ($preset == 'bool') - { + if ($preset == 'bool') { return (isset($markers[$variable]) && is_bool($markers[$variable]) && $markers[$variable] == true) ? $content_true : $content_false; } - if ($preset == 'string') - { + if ($preset == 'string') { return (isset($markers[$variable]) && is_string($markers[$variable]) && strlen($markers[$variable])) ? $content_true : $content_false; } - if ($preset == 'int') - { + if ($preset == 'int') { return (isset($markers[$variable]) && intval($markers[$variable]) <> 0) ? $content_true : $content_false; @@ -69,29 +78,41 @@ public function process($sql) { $markers = $this->bindings; - // 2. process if/else conditions - $sql = preg_replace_callback('#\{if\s(.+?)}(.+?)\{else}(.+?)\{/if}#sui', function($matches) use ($markers) { - list($condition, $variable, $content_true, $content_false) = $matches; - - return (isset($markers[$variable]) && ($markers[$variable] || is_numeric($markers[$variable]))) - ? $content_true - : $content_false; - }, $sql); + // ------ if/else conditions & if[set|int]/else conditions ----- + $sql = preg_replace_callback('#\{if(.{0,}?)\s+([^\}]+?)\}(.+?)(\{else\}([^\{]+?)?)?\s*\{\/if}#sui', function ($matches) use ($markers) { + return self::__ifsets($matches, $markers); + } + , $sql); - // 3. process if conditions - $sql = preg_replace_callback('#\{if\s(.+?)}(.+?)\{/if}#sui', function($matches) use ($markers) { - list($condition, $variable, $content) = $matches; + return $sql; - if (isset($markers[$variable]) && ($markers[$variable] || is_numeric($markers[$variable]))) { - return $content; - } - }, $sql); + /* + * $ifint var ELSE {ENDIF} + * + */ + + // stackoverflow + // if(whatever) { } else { adsffdsa } else if() { } + // /^if\s*\((.*?)\)\s*{(.*?)}(\s*(else|else\s+if\s*\((.*?)\))\s*{(.*?)})*/ + // if (condition_function(params)) { + // statements; + //} + // if\s*\(((?:(?:(?:"(?:(?:\\")|[^"])*")|(?:'(?:(?:\\')|[^'])*'))|[^\(\)]|\((?1)\))*+)\)\s*{((?:(?:(?:"(?:(?:\\")|[^"])*")|(?:'(?:(?:\\')|[^'])*'))|[^{}]|{(?2)})*+)}\s*(?:(?:else\s*{((?:(?:(?:"(?:(?:\\")|[^"])*")|(?:'(?:(?:\\')|[^'])*'))|[^{}]|{(?3)})*+)}\s*)|(?:else\s*if\s*\(((?:(?:(?:"(?:(?:\\")|[^"])*")|(?:'(?:(?:\\')|[^'])*'))|[^\(\)]|\((?4)\))*+)\)\s*{((?:(?:(?:"(?:(?:\\")|[^"])*")|(?:'(?:(?:\\')|[^'])*'))|[^{}]|{(?5)})*+)}\s*))*; + // @if\s*\(\s*([^)]*)\s*\)\s*(((?!@if|@endif).)*)\s*(?:@else\s*(((?!@if|@endif).)*))?\s*@endif + // @if \s* \( \s* ([^)]*)\s*\)\s*(((?!@if|@endif).)*)\s*(?:@else\s*(((?!@if|@endif).)*))?\s*@endif + // [^}] + + // // 3. process if conditions + // $sql = preg_replace_callback('#\{if\s(.+?)}(.+?)\{/if}#sui', function($matches) use ($markers) { + // list($condition, $variable, $content) = $matches; + // if (isset($markers[$variable]) && ($markers[$variable] || is_numeric($markers[$variable]))) { + // return $content; + // } + // }, $sql); // 1. process if[set|int]/else conditions - $sql = preg_replace_callback('#\{if(.{1,}?)\s(.+?)}(.+?)\{else}(.+?)\{/if}#sui', function($matches) use ($markers) {return self::__ifsets($matches, $markers, true); }, $sql); - $sql = preg_replace_callback('#\{if(.{1,}?)\s(.+?)}(.+?)\{/if}#sui', function($matches) use ($markers) { return self::__ifsets($matches, $markers, false); }, $sql); - - return $sql; + // $sql = preg_replace_callback('#\{if(.{1,}?)\s(.+?)}(.+?)\{else}(.+?)\{/if}#sui', function($matches) use ($markers) {return self::__ifsets($matches, $markers, true); }, $sql); + // $sql = preg_replace_callback('#\{if(.{1,}?)\s(.+?)}(.+?)\{/if}#sui', function($matches) use ($markers) { return self::__ifsets($matches, $markers, false); }, $sql); } } diff --git a/src/Query/Expression/Expression.php b/src/Query/Expression/Expression.php new file mode 100644 index 0000000..07354a3 --- /dev/null +++ b/src/Query/Expression/Expression.php @@ -0,0 +1,11 @@ +uuid = $uuid; + } + + public function needsEncoding() : bool + { + return false; + } + + public function getValue() : string + { + return sprintf("UUIDStringToNum('%s')", $this->uuid); + } +} diff --git a/src/Query/Expression.php b/src/Query/Expression/Raw.php similarity index 52% rename from src/Query/Expression.php rename to src/Query/Expression/Raw.php index a30f64d..0dd5126 100644 --- a/src/Query/Expression.php +++ b/src/Query/Expression/Raw.php @@ -2,13 +2,13 @@ declare(strict_types=1); -namespace ClickHouseDB\Query; +namespace ClickHouseDB\Query\Expression; /** * Pass expression "as is" to be sent and executed at server. - * P.ex.: `new Expression("UUIDStringToNum('0f372656-6a5b-4727-a4c4-f6357775d926')");` + * P.ex.: `new Expression\Raw("UUIDStringToNum('0f372656-6a5b-4727-a4c4-f6357775d926')");` */ -class Expression +class Raw implements Expression { /** @var string */ private $expression; @@ -18,7 +18,12 @@ public function __construct(string $expression) $this->expression = $expression; } - public function __toString() : string + public function needsEncoding() : bool + { + return false; + } + + public function getValue() : string { return $this->expression; } diff --git a/src/Query/Query.php b/src/Query/Query.php index 943e5d4..0c4b979 100644 --- a/src/Query/Query.php +++ b/src/Query/Query.php @@ -22,6 +22,25 @@ class Query */ private $degenerations = []; + private $supportFormats=[ + "FORMAT\\s+TSVRaw", + "FORMAT\\s+TSVWithNamesAndTypes", + "FORMAT\\s+TSVWithNames", + "FORMAT\\s+TSV", + "FORMAT\\s+Vertical", + "FORMAT\\s+JSONCompact", + "FORMAT\\s+JSONEachRow", + "FORMAT\\s+TSKV", + "FORMAT\\s+TabSeparatedWithNames", + "FORMAT\\s+TabSeparatedWithNamesAndTypes", + "FORMAT\\s+TabSeparatedRaw", + "FORMAT\\s+BlockTabSeparated", + "FORMAT\\s+CSVWithNames", + "FORMAT\\s+CSV", + "FORMAT\\s+JSON", + "FORMAT\\s+TabSeparated" + ]; + /** * Query constructor. * @param string $sql @@ -52,8 +71,12 @@ private function applyFormatQuery() if (null === $this->format) { return false; } - $supportFormats = - "FORMAT\\s+TSV|FORMAT\\s+TSVRaw|FORMAT\\s+TSVWithNames|FORMAT\\s+TSVWithNamesAndTypes|FORMAT\\s+Vertical|FORMAT\\s+JSONCompact|FORMAT\\s+JSONEachRow|FORMAT\\s+TSKV|FORMAT\\s+TabSeparatedWithNames|FORMAT\\s+TabSeparatedWithNamesAndTypes|FORMAT\\s+TabSeparatedRaw|FORMAT\\s+BlockTabSeparated|FORMAT\\s+CSVWithNames|FORMAT\\s+CSV|FORMAT\\s+JSON|FORMAT\\s+TabSeparated"; + $supportFormats = implode("|",$this->supportFormats); + + $this->sql = trim($this->sql); + if (substr($this->sql, -1) == ';') { + $this->sql = substr($this->sql, 0, -1); + } $matches = []; if (preg_match_all('%(' . $supportFormats . ')%ius', $this->sql, $matches)) { @@ -81,10 +104,36 @@ private function applyFormatQuery() */ public function getFormat() { - return $this->format; } + public function isUseInUrlBindingsParams():bool + { + // 'query=select {p1:UInt8} + {p2:UInt8}' -F "param_p1=3" -F "param_p2=4" + return preg_match('#{[\w+]+:[\w+()]+}#',$this->sql); + + } + public function getUrlBindingsParams():array + { + $out=[]; + if (sizeof($this->degenerations)) { + foreach ($this->degenerations as $degeneration) { + if ($degeneration instanceof Degeneration) { + $params=$degeneration->getBind(); + break; + // need first response + } + } + } + if (sizeof($params)) { + foreach ($params as $key=>$value) + { + $out['param_'.$key]=$value; + } + } + return $out; + } + public function toSql() { if ($this->format !== null) { diff --git a/src/Query/WriteToFile.php b/src/Query/WriteToFile.php index c83e303..4f50b66 100644 --- a/src/Query/WriteToFile.php +++ b/src/Query/WriteToFile.php @@ -12,8 +12,10 @@ class WriteToFile const FORMAT_TabSeparated = 'TabSeparated'; const FORMAT_TabSeparatedWithNames = 'TabSeparatedWithNames'; const FORMAT_CSV = 'CSV'; + const FORMAT_CSVWithNames = 'CSVWithNames'; + const FORMAT_JSONEACHROW = 'JSONEachRow'; - private $support_format = ['TabSeparated', 'TabSeparatedWithNames', 'CSV']; + private $support_format = ['TabSeparated', 'TabSeparatedWithNames', 'CSV', 'CSVWithNames', 'JSONEachRow']; /** * @var string */ diff --git a/src/Quote/FormatLine.php b/src/Quote/FormatLine.php index eb119dc..562c80a 100644 --- a/src/Quote/FormatLine.php +++ b/src/Quote/FormatLine.php @@ -29,11 +29,12 @@ public static function strictQuote($format) * Array in a string for a query Insert * * @param mixed[] $row + * @param bool $skipEncode * @return string */ - public static function Insert(array $row) + public static function Insert(array $row,bool $skipEncode=false) { - return self::strictQuote('Insert')->quoteRow($row); + return self::strictQuote('Insert')->quoteRow($row,$skipEncode); } /** diff --git a/src/Quote/StrictQuoteLine.php b/src/Quote/StrictQuoteLine.php index 0f03f38..8d71252 100644 --- a/src/Quote/StrictQuoteLine.php +++ b/src/Quote/StrictQuoteLine.php @@ -2,6 +2,7 @@ namespace ClickHouseDB\Quote; use ClickHouseDB\Exception\QueryException; +use ClickHouseDB\Query\Expression\Expression; use ClickHouseDB\Type\NumericType; use function array_map; use function is_array; @@ -50,44 +51,59 @@ public function __construct($format) $this->settings = $this->preset[$format]; } - public function quoteRow($row) + + /** + * @param $row + * @param bool $skipEncode + * @return string + */ + public function quoteRow($row,bool $skipEncode=false ) { - return implode($this->settings['Delimiter'], $this->quoteValue($row)); + return implode($this->settings['Delimiter'], $this->quoteValue($row,$skipEncode)); } - public function quoteValue($row) + + /** + * @param $row + * @param bool $skipEncode + * @return array + */ + public function quoteValue($row,bool $skipEncode=false) { $enclosure = $this->settings['Enclosure']; $delimiter = $this->settings['Delimiter']; - $encode = $this->settings['EncodeEnclosure']; + $encodeEnclosure = $this->settings['EncodeEnclosure']; $encodeArray = $this->settings['EnclosureArray']; $null = $this->settings['Null']; $tabEncode = $this->settings['TabEncode']; - $quote = function($value) use ($enclosure, $delimiter, $encode, $encodeArray, $null, $tabEncode) { + $quote = function($value) use ($enclosure, $delimiter, $encodeEnclosure, $encodeArray, $null, $tabEncode, $skipEncode) { $delimiter_esc = preg_quote($delimiter, '/'); $enclosure_esc = preg_quote($enclosure, '/'); - $encode_esc = preg_quote($encode, '/'); + $encode_esc = preg_quote($encodeEnclosure, '/'); $encode = true; if ($value instanceof NumericType) { $encode = false; } + if ($value instanceof Expression) { + $encode = $value->needsEncoding(); + } if (is_array($value)) { // Arrays are formatted as a list of values separated by commas in square brackets. // Elements of the array - the numbers are formatted as usual, and the dates, dates-with-time, and lines are in // single quotation marks with the same screening rules as above. // as in the TabSeparated format, and then the resulting string is output in InsertRow in double quotes. + $value = array_map( function ($v) use ($enclosure_esc, $encode_esc) { return is_string($v) ? $this->encodeString($v, $enclosure_esc, $encode_esc) : $v; }, $value ); - $resultArray = FormatLine::Insert($value); - + $resultArray = FormatLine::Insert($value,($encodeEnclosure==='\\'?true:false)); return $encodeArray . '[' . $resultArray . ']' . $encodeArray; } @@ -97,11 +113,14 @@ function ($v) use ($enclosure_esc, $encode_esc) { return (string) $value; } + $value = is_bool($value) ? ($value ? 'true' : 'false') : $value; + if (is_string($value) && $encode) { if ($tabEncode) { return str_replace(["\t", "\n"], ['\\t', '\\n'], $value); } + if (!$skipEncode) $value = $this->encodeString($value, $enclosure_esc, $encode_esc); return $enclosure . $value . $enclosure; diff --git a/src/Quote/ValueFormatter.php b/src/Quote/ValueFormatter.php index 43840cd..c597022 100644 --- a/src/Quote/ValueFormatter.php +++ b/src/Quote/ValueFormatter.php @@ -5,7 +5,7 @@ namespace ClickHouseDB\Quote; use ClickHouseDB\Exception\UnsupportedValueType; -use ClickHouseDB\Query\Expression; +use ClickHouseDB\Query\Expression\Expression; use ClickHouseDB\Type\Type; use DateTimeInterface; use function addslashes; @@ -21,6 +21,7 @@ class ValueFormatter { /** * @param mixed $value + * @param bool $addQuotes * @return mixed */ public static function formatValue($value, bool $addQuotes = true) @@ -38,7 +39,7 @@ public static function formatValue($value, bool $addQuotes = true) } if ($value instanceof Expression) { - return $value; + return $value->getValue(); } if (is_object($value) && is_callable([$value, '__toString'])) { diff --git a/src/Settings.php b/src/Settings.php index 9109394..2ff547c 100644 --- a/src/Settings.php +++ b/src/Settings.php @@ -2,15 +2,8 @@ namespace ClickHouseDB; -use ClickHouseDB\Transport\Http; - class Settings { - /** - * @var Http - */ - private $client = null; - /** * @var array */ @@ -18,27 +11,20 @@ class Settings private $_ReadOnlyUser = false; - /** - * @var bool - */ - private $_isHttps = false; - /** * Settings constructor. - * @param Http $client */ - public function __construct(Http $client) + public function __construct() { $default = [ - 'extremes' => false, - 'readonly' => true, - 'max_execution_time' => 20, - 'enable_http_compression' => 0, - 'https' => false + 'extremes' => false, + 'readonly' => true, + 'max_execution_time' => 20.0, + 'enable_http_compression' => 1, + 'https' => false, ]; $this->settings = $default; - $this->client = $client; } /** @@ -93,11 +79,11 @@ public function database($db) } /** - * @return mixed + * @return int */ - public function getTimeOut() + public function getTimeOut(): int { - return $this->get('max_execution_time'); + return intval($this->get('max_execution_time')); } /** @@ -150,6 +136,7 @@ public function session_id($session_id) $this->set('session_id', $session_id); return $this; } + /** * @return mixed|bool */ @@ -171,12 +158,15 @@ public function makeSessionId() } /** + * + * max_execution_time - is integer in Seconds clickhouse source + * * @param int $time * @return $this */ - public function max_execution_time($time) + public function max_execution_time(int $time) { - $this->set('max_execution_time', $time); + $this->set('max_execution_time',$time); return $this; } @@ -192,7 +182,7 @@ public function getSettings() * @param array $settings_array * @return $this */ - public function apply($settings_array) + public function apply(array $settings_array) { foreach ($settings_array as $key => $value) { $this->set($key, $value); @@ -204,7 +194,7 @@ public function apply($settings_array) /** * @param int|bool $flag */ - public function setReadOnlyUser($flag) + public function setReadOnlyUser($flag):void { $this->_ReadOnlyUser = $flag; } @@ -212,7 +202,7 @@ public function setReadOnlyUser($flag) /** * @return bool */ - public function isReadOnlyUser() + public function isReadOnlyUser():bool { return $this->_ReadOnlyUser; } @@ -221,7 +211,7 @@ public function isReadOnlyUser() * @param string $name * @return mixed|null */ - public function getSetting($name) + public function getSetting(string $name) { if (!isset($this->settings[$name])) { return null; @@ -229,4 +219,9 @@ public function getSetting($name) return $this->get($name); } + + public function clear():void + { + $this->settings = []; + } } diff --git a/src/Statement.php b/src/Statement.php index 8a2663e..9701d51 100644 --- a/src/Statement.php +++ b/src/Statement.php @@ -2,14 +2,17 @@ namespace ClickHouseDB; +use ClickHouseDB\Exception\ClickHouseUnavailableException; use ClickHouseDB\Exception\DatabaseException; use ClickHouseDB\Exception\QueryException; use ClickHouseDB\Query\Query; use ClickHouseDB\Transport\CurlerRequest; use ClickHouseDB\Transport\CurlerResponse; -class Statement +class Statement implements \Iterator { + private const CLICKHOUSE_ERROR_REGEX = "%Code:\s(\d+)\.\s*DB::Exception\s*:\s*(.*)(?:,\s*e\.what|\(version).*%ius"; + /** * @var string|mixed */ @@ -80,6 +83,11 @@ class Statement */ private $statistics = null; + /** + * @var int + */ + public $iterator = 0; + public function __construct(CurlerRequest $request) { @@ -127,17 +135,32 @@ public function sql() * @param string $body * @return array|bool */ - private function parseErrorClickHouse($body) + private function parseErrorClickHouse(string $body) { $body = trim($body); - $mathes = []; + $matches = []; - // Code: 115, e.displayText() = DB::Exception: Unknown setting readonly[0], e.what() = DB::Exception - // Code: 192, e.displayText() = DB::Exception: Unknown user x, e.what() = DB::Exception - // Code: 60, e.displayText() = DB::Exception: Table default.ZZZZZ doesn't exist., e.what() = DB::Exception + // Code: 115. DB::Exception: Unknown setting readonly[0], e.what() = DB::Exception + // Code: 192. DB::Exception: Unknown user x, e.what() = DB::Exception + // Code: 60. DB::Exception: Table default.ZZZZZ doesn't exist., e.what() = DB::Exception + // Code: 516. DB::Exception: test_username: Authentication failed: password is incorrect or there is no user with such name. (AUTHENTICATION_FAILED) (version 22.8.3.13 (official build)) - if (preg_match("%Code: (\d+),\se\.displayText\(\) \=\s*DB\:\:Exception\s*:\s*(.*)\,\s*e\.what.*%ius", $body, $mathes)) { - return ['code' => $mathes[1], 'message' => $mathes[2]]; + if (preg_match(self::CLICKHOUSE_ERROR_REGEX, $body, $matches)) { + return ['code' => $matches[1], 'message' => $matches[2]]; + } + + return false; + } + + private function hasErrorClickhouse(string $body, string $contentType): bool { + if (false === stripos($contentType, 'application/json')) { + return preg_match(self::CLICKHOUSE_ERROR_REGEX, $body) === 1; + } + + try { + json_decode($body, true, 512, JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + return true; } return false; @@ -157,6 +180,7 @@ public function error() $error_no = $this->response()->error_no(); $error = $this->response()->error(); + $dumpStatement = false; if (!$error_no && !$error) { $parse = $this->parseErrorClickHouse($body); @@ -165,25 +189,48 @@ public function error() } else { $code = $this->response()->http_code(); $message = "HttpCode:" . $this->response()->http_code() . " ; " . $this->response()->error() . " ;" . $body; + $dumpStatement = true; } } else { $code = $error_no; $message = $this->response()->error(); } - throw new QueryException($message, $code); + $exception = new QueryException($message, $code); + if ($code === CURLE_COULDNT_CONNECT) { + $exception = new ClickHouseUnavailableException($message, $code); + } + + if ($dumpStatement) { + $exception->setRequestDetails($this->_request->getDetails()); + $exception->setResponseDetails($this->response()->getDetails()); + } + + throw $exception; } /** * @return bool * @throws Exception\TransportException */ - public function isError() + public function isError(): bool { - return ($this->response()->http_code() !== 200 || $this->response()->error_no()); + if ($this->response()->http_code() !== 200) { + return true; + } + + if ($this->response()->error_no()) { + return true; + } + + if ($this->hasErrorClickhouse($this->response()->body(), $this->response()->content_type())) { + return true; + } + + return false; } - private function check() : bool + private function check(): bool { if (!$this->_request->isResponseExists()) { throw QueryException::noResponse(); @@ -196,6 +243,14 @@ private function check() : bool return true; } + /** + * @return bool + */ + public function isInited() + { + return $this->_init; + } + /** * @return bool * @throws Exception\TransportException @@ -208,30 +263,27 @@ private function init() $this->check(); - $this->_rawData = $this->response()->rawDataOrJson($this->format); if (!$this->_rawData) { $this->_init = true; return false; } - $data=[]; + + $data = []; foreach (['meta', 'data', 'totals', 'extremes', 'rows', 'rows_before_limit_at_least', 'statistics'] as $key) { if (isset($this->_rawData[$key])) { - if ($key=='data') - { + if ($key=='data') { $data=$this->_rawData[$key]; - } - else{ + } else { $this->{$key} = $this->_rawData[$key]; } - } } if (empty($this->meta)) { - throw new QueryException('Can`t find meta'); + throw new QueryException('Can`t find meta'); } $isJSONCompact=(stripos($this->format,'JSONCompact')!==false?true:false); @@ -239,12 +291,9 @@ private function init() foreach ($data as $rows) { $r = []; - - if ($isJSONCompact) - { - $r[]=$rows; - } - else { + if ($isJSONCompact) { + $r[] = $rows; + } else { foreach ($this->meta as $meta) { $r[$meta['name']] = $rows[$meta['name']]; } @@ -253,6 +302,8 @@ private function init() $this->array_data[] = $r; } + $this->_init = true; + return true; } @@ -317,14 +368,6 @@ public function totals() return $this->totals; } - /** - * - */ - public function dumpRaw() - { - print_r($this->_rawData); - } - /** * */ @@ -352,17 +395,19 @@ public function countAll() public function statistics($key = false) { $this->init(); - if ($key) - { - if (!is_array($this->statistics)) { - return null; - } - if (!isset($this->statistics[$key])) { - return null; - } - return $this->statistics[$key]; + + if (!is_array($this->statistics)) { + return null; } - return $this->statistics; + + if (!$key) return $this->statistics; + + if (!isset($this->statistics[$key])) { + return null; + } + + return $this->statistics[$key]; + } /** @@ -390,28 +435,57 @@ public function rawData() return $this->response()->rawDataOrJson($this->format); } + /** + * + */ + public function resetIterator() + { + $this->iterator=0; + } + + public function fetchRow($key = null) + { + $this->init(); + + $position=$this->iterator; + + if (!isset($this->array_data[$position])) { + return null; + } + + $this->iterator++; + + if (!$key) { + return $this->array_data[$position]; + } + if (!isset($this->array_data[$position][$key])) { + return null; + } + + return $this->array_data[$position][$key]; + + } /** * @param string $key * @return mixed|null * @throws Exception\TransportException */ - public function fetchOne($key = '') + public function fetchOne($key = null) { $this->init(); + if (!isset($this->array_data[0])) { + return null; + } - if (isset($this->array_data[0])) { - if ($key) { - if (isset($this->array_data[0][$key])) { - return $this->array_data[0][$key]; - } else { - return null; - } - } - + if (!$key) { return $this->array_data[0]; } - return null; + if (!isset($this->array_data[0][$key])) { + return null; + } + + return $this->array_data[0][$key]; } /** @@ -488,6 +562,14 @@ public function rows() return $this->array_data; } + /** + * @return false|string + */ + public function jsonRows() + { + return json_encode($this->rows(), JSON_PRETTY_PRINT); + } + /** * @param array|string $arr * @param null|string|array $path @@ -527,4 +609,33 @@ private function array_to_tree($arr, $path = null) } return $tree; } + + + public function rewind(): void { + $this->iterator = 0; + } + + /** + * @return mixed + */ + #[\ReturnTypeWillChange] + public function current() { + if (!isset($this->array_data[$this->iterator])) { + return null; + } + return $this->array_data[$this->iterator]; + } + + public function key(): int { + return $this->iterator; + } + + public function next(): void { + ++$this->iterator; + } + + public function valid(): bool { + $this->init(); + return isset($this->array_data[$this->iterator]); + } } diff --git a/src/Transport/CurlerRequest.php b/src/Transport/CurlerRequest.php index 421bd8c..e62b9a2 100644 --- a/src/Transport/CurlerRequest.php +++ b/src/Transport/CurlerRequest.php @@ -10,7 +10,7 @@ class CurlerRequest /** * @var array */ - public $extendinfo = array(); + public $extendinfo = []; /** * @var string|array @@ -93,6 +93,16 @@ class CurlerRequest */ private $resultFileHandle = null; + /** + * @var string + */ + private $sslCa = null; + + + /** + * @var null|resource + */ + private $stdErrOut = null; /** * @param bool $id */ @@ -114,7 +124,6 @@ public function __construct($id = false) CURLOPT_HEADER => TRUE, CURLOPT_FOLLOWLOCATION => TRUE, CURLOPT_AUTOREFERER => 1, // при редиректе подставлять в «Referer:» значение из «Location:» - CURLOPT_BINARYTRANSFER => 1, // передавать в binary-safe CURLOPT_RETURNTRANSFER => TRUE, CURLOPT_USERAGENT => 'smi2/PHPClickHouse/client', ); @@ -131,8 +140,7 @@ public function __destruct() public function close() { - if ($this->handle) - { + if ($this->handle) { curl_close($this->handle); } $this->handle = null; @@ -207,8 +215,7 @@ public function setInfile($file_name) { $this->header('Expect', ''); $this->infile_handle = fopen($file_name, 'r'); - if (is_resource($this->infile_handle)) - { + if (is_resource($this->infile_handle)) { if ($this->_httpCompression) { $this->header('Content-Encoding', 'gzip'); @@ -240,8 +247,9 @@ public function setCallbackFunction($callback) */ public function setWriteFunction($callback) { - $this->options[CURLOPT_WRITEFUNCTION]=$callback; + $this->options[CURLOPT_WRITEFUNCTION] = $callback; } + /** * @param callable $callback */ @@ -281,6 +289,28 @@ public function onCallback() } } + public function getDetails(): array + { + return [ + 'url' => $this->url, + 'method' => $this->method, + 'parameters' => $this->parameters, + 'headers' => $this->headers, + ]; + } + + /** + * @param resource $stream + * @return void + */ + public function setStdErrOut($stream) + { + if (is_resource($stream)) { + $this->stdErrOut=$stream; + } + + } + /** * @param bool $result * @return string @@ -342,7 +372,7 @@ public function isPersistent() * @param int $sec * @return $this */ - public function keepAlive($sec = 60) + public function keepAlive(int $sec = 60) { $this->options[CURLOPT_FORBID_REUSE] = TRUE; $this->headers['Connection'] = 'Keep-Alive'; @@ -355,7 +385,7 @@ public function keepAlive($sec = 60) * @param bool $flag * @return $this */ - public function verbose($flag = true) + public function verbose(bool $flag = true) { $this->options[CURLOPT_VERBOSE] = $flag; return $this; @@ -366,7 +396,7 @@ public function verbose($flag = true) * @param string $value * @return $this */ - public function header($key, $value) + public function header(string $key, string $value) { $this->headers[$key] = $value; return $this; @@ -375,11 +405,11 @@ public function header($key, $value) /** * @return array */ - public function getHeaders() + public function getHeaders():array { $head = []; - foreach ($this->headers as $key=>$value) { - $head[] = sprintf("%s: %s", $key, $value); + foreach ($this->headers as $key => $value) { + $head[] = sprintf("%s: %s", $key, $value); } return $head; } @@ -388,16 +418,16 @@ public function getHeaders() * @param string $url * @return $this */ - public function url($url) + public function url(string $url) { $this->url = $url; return $this; } /** - * @return mixed + * @return string */ - public function getUrl() + public function getUrl():string { return $this->url; } @@ -407,7 +437,7 @@ public function getUrl() * @param string $id * @return string */ - public function getUniqHash($id) + public function getUniqHash(string $id):string { return $id . '.' . microtime() . mt_rand(0, 1000000); } @@ -415,13 +445,12 @@ public function getUniqHash($id) /** * @param bool $flag */ - public function httpCompression($flag) + public function httpCompression(bool $flag):void { if ($flag) { $this->_httpCompression = $flag; $this->options[CURLOPT_ENCODING] = 'gzip'; - } else - { + } else { $this->_httpCompression = false; unset($this->options[CURLOPT_ENCODING]); } @@ -432,12 +461,19 @@ public function httpCompression($flag) * @param string $password * @return $this */ - public function auth($username, $password) + public function authByBasicAuth($username, $password) { $this->options[CURLOPT_USERPWD] = sprintf("%s:%s", $username, $password); return $this; } + public function authByHeaders($username, $password) + { + $this->headers['X-ClickHouse-User'] = $username; + $this->headers['X-ClickHouse-Key'] = $password; + return $this; + } + /** * @param array|string $data * @return $this @@ -451,12 +487,12 @@ public function parameters($data) /** * The number of seconds to wait when trying to connect. Use 0 for infinite waiting. * - * @param int $seconds + * @param float $seconds * @return $this */ - public function connectTimeOut($seconds = 1) + public function connectTimeOut(float $seconds = 1.0) { - $this->options[CURLOPT_CONNECTTIMEOUT] = $seconds; + $this->options[CURLOPT_CONNECTTIMEOUT_MS] = (int) ($seconds*1000.0); return $this; } @@ -466,9 +502,9 @@ public function connectTimeOut($seconds = 1) * @param float $seconds * @return $this */ - public function timeOut($seconds = 10) + public function timeOut(float $seconds = 10) { - return $this->timeOutMs(intval($seconds * 1000)); + return $this->timeOutMs((int) ($seconds * 1000.0)); } /** @@ -477,7 +513,7 @@ public function timeOut($seconds = 10) * @param int $ms millisecond * @return $this */ - protected function timeOutMs($ms = 10000) + protected function timeOutMs(int $ms = 10000) { $this->options[CURLOPT_TIMEOUT_MS] = $ms; return $this; @@ -599,6 +635,17 @@ public function getDnsCache() return $this->_dns_cache; } + /** + * Sets client certificate + * + * @param string $filePath + */ + public function setSslCa($filePath) + { + $this->option(CURLOPT_SSL_VERIFYPEER, true); + $this->option(CURLOPT_CAINFO, $filePath); + } + /** * @param string $method * @return $this @@ -615,19 +662,19 @@ private function execute($method) */ public function response() { - if (! $this->response) { + if (!$this->response) { throw new \ClickHouseDB\Exception\TransportException('Can`t fetch response - is empty'); } return $this->response; } - public function isResponseExists() : bool + public function isResponseExists(): bool { return $this->response !== null; } - public function setResponse(CurlerResponse $response) : void + public function setResponse(CurlerResponse $response): void { $this->response = $response; } @@ -674,7 +721,7 @@ private function prepareRequest() if (strtoupper($method) == 'GET') { - $curl_opt[CURLOPT_HTTPGET] = true; + $curl_opt[CURLOPT_HTTPGET] = true; $curl_opt[CURLOPT_CUSTOMREQUEST] = strtoupper($method); $curl_opt[CURLOPT_POSTFIELDS] = false; } else { @@ -688,7 +735,7 @@ private function prepareRequest() $curl_opt[CURLOPT_POSTFIELDS] = $this->parameters; if (!is_array($this->parameters)) { - $this->header('Content-Length', strlen($this->parameters)); + $this->header('Content-Length', mb_strlen($this->parameters, '8bit')); } } } @@ -697,7 +744,7 @@ private function prepareRequest() $curl_opt[CURLOPT_URL] = $this->url; if (!empty($this->headers) && sizeof($this->headers)) { - $curl_opt[CURLOPT_HTTPHEADER] = array(); + $curl_opt[CURLOPT_HTTPHEADER] = []; foreach ($this->headers as $key => $value) { $curl_opt[CURLOPT_HTTPHEADER][] = sprintf("%s: %s", $key, $value); @@ -709,9 +756,8 @@ private function prepareRequest() $curl_opt[CURLOPT_PUT] = true; } - if (!empty($curl_opt[CURLOPT_WRITEFUNCTION])) - { - $curl_opt[CURLOPT_HEADER]=false; + if (!empty($curl_opt[CURLOPT_WRITEFUNCTION])) { + $curl_opt[CURLOPT_HEADER] = false; } if ($this->resultFileHandle) { @@ -720,8 +766,20 @@ private function prepareRequest() } if ($this->options[CURLOPT_VERBOSE]) { - echo "\n-----------BODY REQUEST----------\n" . $curl_opt[CURLOPT_POSTFIELDS] . "\n------END--------\n"; + $msg="\n-----------BODY REQUEST----------\n" . $curl_opt[CURLOPT_POSTFIELDS] . "\n------END--------\n"; + if ($this->stdErrOut && is_resource($this->stdErrOut)) { + fwrite($this->stdErrOut,$msg); + } else { + echo $msg; + } } + + if ($this->stdErrOut) { + if (is_resource($this->stdErrOut)) { + $curl_opt[CURLOPT_STDERR]=$this->stdErrOut; + } + } + curl_setopt_array($this->handle, $curl_opt); return true; } diff --git a/src/Transport/CurlerResponse.php b/src/Transport/CurlerResponse.php index a900c4b..f2afef6 100644 --- a/src/Transport/CurlerResponse.php +++ b/src/Transport/CurlerResponse.php @@ -158,6 +158,16 @@ public function dump_json() print_r($this->json()); } + public function getDetails(): array + { + return [ + 'body' => $this->_body, + 'headers' => $this->_headers, + 'error' => $this->error(), + 'info' => $this->_info, + ]; + } + /** * @param bool $result * @return string diff --git a/src/Transport/CurlerRolling.php b/src/Transport/CurlerRolling.php index cabb4ab..e21c0b5 100644 --- a/src/Transport/CurlerRolling.php +++ b/src/Transport/CurlerRolling.php @@ -6,6 +6,8 @@ class CurlerRolling { + const SLEEP_DELAY = 1000; // 1ms + /** * @var int * @@ -55,7 +57,7 @@ class CurlerRolling /** * */ - public function __destructor() + public function __destruct() { $this->close(); } @@ -123,8 +125,8 @@ private function makeResponse($oneHandle) { $response = curl_multi_getcontent($oneHandle); $header_size = curl_getinfo($oneHandle, CURLINFO_HEADER_SIZE); - $header = substr($response, 0, $header_size); - $body = substr($response, $header_size); + $header = substr($response ?? '', 0, $header_size); + $body = substr($response ?? '', $header_size); $n = new CurlerResponse(); $n->_headers = $this->parse_headers_from_curl_response($header); @@ -143,35 +145,10 @@ private function makeResponse($oneHandle) */ public function execLoopWait() { - $c = 0; - $timeStart=time(); - $uSleep=1000; // Timer: check response from server, and add new Task/Que to loop - $PendingAllConnections=$this->countPending(); - - echo "$PendingAllConnections\n"; - // Max loop check - - $count=0; - // add all tasks do { $this->exec(); - $timeWork=time()-$timeStart; - // - $ActiveNowConnections = $this->countActive(); - $PendingNowConnections = $this->countPending(); - - echo "$ActiveNowConnections\t[ $PendingNowConnections // $PendingAllConnections ] \t\t\t[ $c ]\t$timeWork\n"; - - $count=$ActiveNowConnections+$PendingNowConnections; - $c++; - - if ($c > 20000) { - break; - } - - usleep($uSleep); - // usleep(2000000) == 2 seconds - } while ($count); + usleep(self::SLEEP_DELAY); + } while (($this->countActive() + $this->countPending()) > 0); return true; } @@ -311,7 +288,13 @@ public function exec() // send the return values to the callback function. - $key = (string) $done['handle']; + + if (is_object($done['handle'])) { + $key = spl_object_id( $done['handle'] ); + } else { + $key = (string) $done['handle'] ; + } + $task_id = $this->handleMapTasks[$key]; $request = $this->pendingRequests[$this->handleMapTasks[$key]]; @@ -342,13 +325,11 @@ public function exec() public function makePendingRequestsQue() { - $max = $this->getSimultaneousLimit(); $active = $this->countActive(); if ($active < $max) { - $canAdd = $max - $active; // $pending = sizeof($this->pendingRequests); @@ -375,7 +356,6 @@ public function makePendingRequestsQue() foreach ($ll as $task_id) { $this->_prepareLoopQue($task_id); } - }// if add }// if can add } @@ -393,7 +373,12 @@ private function _prepareLoopQue($task_id) // pool curl_multi_add_handle($this->handlerMulti(), $h); - $key = (string) $h; + if (is_object($h)) { + $key = spl_object_id( $h ); + } else { + $key = (string) $h ; + } + $this->handleMapTasks[$key] = $task_id; } } diff --git a/src/Transport/Http.php b/src/Transport/Http.php index da35c40..cd1d286 100644 --- a/src/Transport/Http.php +++ b/src/Transport/Http.php @@ -13,6 +13,16 @@ class Http { + const AUTH_METHOD_HEADER = 1; + const AUTH_METHOD_QUERY_STRING = 2; + const AUTH_METHOD_BASIC_AUTH = 3; + + const AUTH_METHODS_LIST = [ + self::AUTH_METHOD_HEADER, + self::AUTH_METHOD_QUERY_STRING, + self::AUTH_METHOD_BASIC_AUTH, + ]; + /** * @var string */ @@ -23,6 +33,17 @@ class Http */ private $_password = null; + /** + * The username and password can be indicated in one of three ways: + * - Using HTTP Basic Authentication. + * - In the ‘user’ and ‘password’ URL parameters. + * - Using ‘X-ClickHouse-User’ and ‘X-ClickHouse-Key’ headers (by default) + * + * @see https://clickhouse.tech/docs/en/interfaces/http/ + * @var int + */ + private $_authMethod = self::AUTH_METHOD_HEADER; + /** * @var string */ @@ -56,43 +77,73 @@ class Http /** * Count seconds (int) * - * @var int + * @var float */ - private $_connectTimeOut = 5; + private $_connectTimeOut = 5.0; /** * @var callable */ private $xClickHouseProgress = null; + /** + * @var null|string + */ + private $sslCA = null; + + /** + * @var null|resource + */ + private $stdErrOut = null; + + /** + * @var null|resource + */ + private $handle = null; + /** * Http constructor. * @param string $host * @param int $port * @param string $username * @param string $password + * @param int $authMethod */ - public function __construct($host, $port, $username, $password) + public function __construct($host, $port, $username, $password, $authMethod = null) { $this->setHost($host, $port); $this->_username = $username; $this->_password = $password; - $this->_settings = new Settings($this); + if ($authMethod) { + $this->_authMethod = $authMethod; + } + + $this->_settings = new Settings(); $this->setCurler(); } - public function setCurler() + public function setCurler() : void { $this->_curler = new CurlerRolling(); } + /** + * @param CurlerRolling $curler + */ + public function setDirtyCurler(CurlerRolling $curler) : void + { + if ($curler instanceof CurlerRolling) { + $this->_curler = $curler; + } + } + /** * @return CurlerRolling */ - public function getCurler() + public function getCurler(): ?CurlerRolling { return $this->_curler; } @@ -101,7 +152,7 @@ public function getCurler() * @param string $host * @param int $port */ - public function setHost($host, $port = -1) + public function setHost(string $host, int $port = -1) : void { if ($port > 0) { $this->_port = $port; @@ -110,20 +161,30 @@ public function setHost($host, $port = -1) $this->_host = $host; } + /** + * Sets client SSL certificate for Yandex Cloud + * + * @param string $caPath + */ + public function setSslCa(string $caPath) : void + { + $this->sslCA = $caPath; + } + /** * @return string */ - public function getUri() + public function getUri(): string { $proto = 'http'; if ($this->settings()->isHttps()) { $proto = 'https'; } $uri = $proto . '://' . $this->_host; - if (stripos($this->_host,'/')!==false || stripos($this->_host,':')!==false) { + if (stripos($this->_host, '/') !== false || stripos($this->_host, ':') !== false) { return $uri; } - if (intval($this->_port)>0) { + if (intval($this->_port) > 0) { return $uri . ':' . $this->_port; } return $uri; @@ -132,16 +193,16 @@ public function getUri() /** * @return Settings */ - public function settings() + public function settings(): Settings { return $this->_settings; } /** - * @param bool|int $flag - * @return mixed + * @param bool $flag + * @return bool */ - public function verbose($flag) + public function verbose(bool $flag): bool { $this->_verbose = $flag; return $flag; @@ -151,7 +212,7 @@ public function verbose($flag) * @param array $params * @return string */ - private function getUrl($params = []) + private function getUrl($params = []): string { $settings = $this->settings()->getSettings(); @@ -160,8 +221,7 @@ private function getUrl($params = []) } - if ($this->settings()->isReadOnlyUser()) - { + if ($this->settings()->isReadOnlyUser()) { unset($settings['extremes']); unset($settings['readonly']); unset($settings['enable_http_compression']); @@ -179,23 +239,41 @@ private function getUrl($params = []) * @param array $extendinfo * @return CurlerRequest */ - private function newRequest($extendinfo) + private function newRequest($extendinfo): CurlerRequest { $new = new CurlerRequest(); - $new->auth($this->_username, $this->_password) - ->POST() - ->setRequestExtendedInfo($extendinfo); - if ($this->settings()->isEnableHttpCompression()) { - $new->httpCompression(true); + switch ($this->_authMethod) { + case self::AUTH_METHOD_QUERY_STRING: + /* @todo: Move this implementation to CurlerRequest class. Possible options: the authentication method + * should be applied in method `CurlerRequest:prepareRequest()`. + */ + $this->settings()->set('user', $this->_username); + $this->settings()->set('password', $this->_password); + break; + case self::AUTH_METHOD_BASIC_AUTH: + $new->authByBasicAuth($this->_username, $this->_password); + break; + default: + // Auth with headers by default + $new->authByHeaders($this->_username, $this->_password); + break; } - if ($this->settings()->getSessionId()) - { + + $new->POST()->setRequestExtendedInfo($extendinfo); + + $new->httpCompression($this->settings()->isEnableHttpCompression()); + + if ($this->settings()->getSessionId()) { $new->persistent(); } + if ($this->sslCA) { + $new->setSslCa($this->sslCA); + } $new->timeOut($this->settings()->getTimeOut()); - $new->connectTimeOut($this->_connectTimeOut)->keepAlive(); // one sec + $new->connectTimeOut($this->_connectTimeOut); + $new->keepAlive(); $new->verbose(boolval($this->_verbose)); return $new; @@ -208,7 +286,7 @@ private function newRequest($extendinfo) * @return CurlerRequest * @throws \ClickHouseDB\Exception\TransportException */ - private function makeRequest(Query $query, $urlParams = [], $query_as_string = false) + private function makeRequest(Query $query, array $urlParams = [], bool $query_as_string = false): CurlerRequest { $sql = $query->toSql(); @@ -216,54 +294,76 @@ private function makeRequest(Query $query, $urlParams = [], $query_as_string = f $urlParams['query'] = $sql; } - $url = $this->getUrl($urlParams); - - $extendinfo = [ + $extendInfo = [ 'sql' => $sql, 'query' => $query, - 'format'=> $query->getFormat() + 'format' => $query->getFormat() ]; - $new = $this->newRequest($extendinfo); - $new->url($url); + $new = $this->newRequest($extendInfo); + /* + * Build URL after request making, since URL may contain auth data. This will not matter after the + * implantation of the todo in the `HTTP:newRequest()` method. + */ + if ($query->isUseInUrlBindingsParams()) { + $urlParams = array_replace_recursive($urlParams, $query->getUrlBindingsParams()); + } + $url = $this->getUrl($urlParams); + $new->url($url); if (!$query_as_string) { $new->parameters_json($sql); } - if ($this->settings()->isEnableHttpCompression()) { - $new->httpCompression(true); - } + $new->httpCompression($this->settings()->isEnableHttpCompression()); return $new; } + /** + * @param resource $stream + * @return void + */ + public function setStdErrOut($stream) + { + if (is_resource($stream)) { + $this->stdErrOut=$stream; + } + + } + /** * @param string|Query $sql * @return CurlerRequest */ - public function writeStreamData($sql) + public function writeStreamData($sql): CurlerRequest { if ($sql instanceof Query) { - $query=$sql; + $query = $sql; } else { $query = new Query($sql); } + $extendInfo = [ + 'sql' => $sql, + 'query' => $query, + 'format' => $query->getFormat() + ]; + + $request = $this->newRequest($extendInfo); + + /* + * Build URL after request making, since URL may contain auth data. This will not matter after the + * implantation of the todo in the `HTTP:newRequest()` method. + */ $url = $this->getUrl([ 'readonly' => 0, 'query' => $query->toSql() ]); - $extendinfo = [ - 'sql' => $sql, - 'query' => $query, - 'format'=> $query->getFormat() - ]; - $request = $this->newRequest($extendinfo); $request->url($url); return $request; } @@ -275,25 +375,30 @@ public function writeStreamData($sql) * @return Statement * @throws \ClickHouseDB\Exception\TransportException */ - public function writeAsyncCSV($sql, $file_name) + public function writeAsyncCSV($sql, $file_name): Statement { $query = new Query($sql); - $url = $this->getUrl([ - 'readonly' => 0, - 'query' => $query->toSql() - ]); - $extendinfo = [ 'sql' => $sql, 'query' => $query, - 'format'=> $query->getFormat() + 'format' => $query->getFormat() ]; $request = $this->newRequest($extendinfo); + + /* + * Build URL after request making, since URL may contain auth data. This will not matter after the + * implantation of the todo in the `HTTP:newRequest()` method. + */ + $url = $this->getUrl([ + 'readonly' => 0, + 'query' => $query->toSql() + ]); + $request->url($url); - $request->setCallbackFunction(function(CurlerRequest $request) { + $request->setCallbackFunction(function (CurlerRequest $request) { $handle = $request->getInfileHandle(); if (is_resource($handle)) { fclose($handle); @@ -311,7 +416,7 @@ public function writeAsyncCSV($sql, $file_name) * * @return int */ - public function getCountPendingQueue() + public function getCountPendingQueue(): int { return $this->_curler->countPending(); } @@ -319,9 +424,9 @@ public function getCountPendingQueue() /** * set Connect TimeOut in seconds [CURLOPT_CONNECTTIMEOUT] ( int ) * - * @param int $connectTimeOut + * @param float $connectTimeOut */ - public function setConnectTimeOut($connectTimeOut) + public function setConnectTimeOut(float $connectTimeOut) { $this->_connectTimeOut = $connectTimeOut; } @@ -329,15 +434,15 @@ public function setConnectTimeOut($connectTimeOut) /** * get ConnectTimeOut in seconds * - * @return int + * @return float */ - public function getConnectTimeOut() + public function getConnectTimeOut(): float { return $this->_connectTimeOut; } - public function __findXClickHouseProgress($handle) + public function __findXClickHouseProgress($handle): bool { $code = curl_getinfo($handle, CURLINFO_HTTP_CODE); @@ -350,31 +455,25 @@ public function __findXClickHouseProgress($handle) } $header = substr($response, 0, $header_size); - if (!$header_size) { - return false; - } - $pos = strrpos($header, 'X-ClickHouse-Progress'); - - if (!$pos) { + if (!$header) { return false; } - $last = substr($header, $pos); - $data = @json_decode(str_ireplace('X-ClickHouse-Progress:', '', $last), true); + $match = []; + if (preg_match_all('/^X-ClickHouse-(?:Progress|Summary):(.*?)$/im', $header, $match)) { + $data = @json_decode(end($match[1]), true); + if ($data && is_callable($this->xClickHouseProgress)) { - if ($data && is_callable($this->xClickHouseProgress)) { + if (is_array($this->xClickHouseProgress)) { + call_user_func_array($this->xClickHouseProgress, [$data]); + } else { + call_user_func($this->xClickHouseProgress, $data); + } - if (is_array($this->xClickHouseProgress)) { - call_user_func_array($this->xClickHouseProgress, [$data]); - } else { - call_user_func($this->xClickHouseProgress, $data); } - - } - } - + return false; } /** @@ -384,9 +483,9 @@ public function __findXClickHouseProgress($handle) * @return CurlerRequest * @throws \Exception */ - public function getRequestRead(Query $query, $whereInFile = null, $writeToFile = null) + public function getRequestRead(Query $query, $whereInFile = null, $writeToFile = null): CurlerRequest { - $urlParams = ['readonly' => 1]; + $urlParams = ['readonly' => 2]; $query_as_string = false; // --------------------------------------------------------------------------------- if ($whereInFile instanceof WhereInFile && $whereInFile->size()) { @@ -432,13 +531,16 @@ public function getRequestRead(Query $query, $whereInFile = null, $writeToFile = } - $request->setResultFileHandle($fout, $isGz)->setCallbackFunction(function(CurlerRequest $request) { + $request->setResultFileHandle($fout, $isGz)->setCallbackFunction(function (CurlerRequest $request) { fclose($request->getResultFileHandle()); }); } } - if ($this->xClickHouseProgress) - { + + if ($this->stdErrOut) { + $request->setStdErrOut($this->stdErrOut); + } + if ($this->xClickHouseProgress) { $request->setFunctionProgress([$this, '__findXClickHouseProgress']); } // --------------------------------------------------------------------------------- @@ -446,13 +548,13 @@ public function getRequestRead(Query $query, $whereInFile = null, $writeToFile = } - public function cleanQueryDegeneration() + public function cleanQueryDegeneration(): bool { $this->_query_degenerations = []; return true; } - public function addQueryDegeneration(Degeneration $degeneration) + public function addQueryDegeneration(Degeneration $degeneration): bool { $this->_query_degenerations[] = $degeneration; return true; @@ -463,7 +565,7 @@ public function addQueryDegeneration(Degeneration $degeneration) * @return CurlerRequest * @throws \ClickHouseDB\Exception\TransportException */ - public function getRequestWrite(Query $query) + public function getRequestWrite(Query $query): CurlerRequest { $urlParams = ['readonly' => 0]; return $this->makeRequest($query, $urlParams); @@ -472,13 +574,13 @@ public function getRequestWrite(Query $query) /** * @throws TransportException */ - public function ping() : bool + public function ping(): bool { $request = new CurlerRequest(); $request->url($this->getUri())->verbose(false)->GET()->connectTimeOut($this->getConnectTimeOut()); $this->_curler->execOne($request); - return $request->response()->body() === 'Ok.' . PHP_EOL; + return trim($request->response()->body()) === 'Ok.'; } /** @@ -486,7 +588,7 @@ public function ping() : bool * @param mixed[] $bindings * @return Query */ - private function prepareQuery($sql, $bindings) + private function prepareQuery($sql, $bindings): Query { // add Degeneration query @@ -506,7 +608,7 @@ private function prepareQuery($sql, $bindings) * @return CurlerRequest * @throws \Exception */ - private function prepareSelect($sql, $bindings, $whereInFile, $writeToFile = null) + private function prepareSelect($sql, $bindings, $whereInFile, $writeToFile = null): CurlerRequest { if ($sql instanceof Query) { return $this->getRequestWrite($sql); @@ -517,21 +619,28 @@ private function prepareSelect($sql, $bindings, $whereInFile, $writeToFile = nul } - - /** * @param Query|string $sql * @param mixed[] $bindings * @return CurlerRequest * @throws \ClickHouseDB\Exception\TransportException */ - private function prepareWrite($sql, $bindings = []) + private function prepareWrite($sql, $bindings = []): CurlerRequest { if ($sql instanceof Query) { return $this->getRequestWrite($sql); } $query = $this->prepareQuery($sql, $bindings); + + if (strpos($sql, 'ON CLUSTER') === false) { + return $this->getRequestWrite($query); + } + + if (strpos($sql, 'CREATE') === 0 || strpos($sql, 'DROP') === 0 || strpos($sql, 'ALTER') === 0) { + $query->setFormat('JSON'); + } + return $this->getRequestWrite($query); } @@ -539,7 +648,7 @@ private function prepareWrite($sql, $bindings = []) * @return bool * @throws \ClickHouseDB\Exception\TransportException */ - public function executeAsync() + public function executeAsync(): bool { return $this->_curler->execLoopWait(); } @@ -553,7 +662,7 @@ public function executeAsync() * @throws \ClickHouseDB\Exception\TransportException * @throws \Exception */ - public function select($sql, array $bindings = [], $whereInFile = null, $writeToFile = null) + public function select($sql, array $bindings = [], $whereInFile = null, $writeToFile = null): Statement { $request = $this->prepareSelect($sql, $bindings, $whereInFile, $writeToFile); $this->_curler->execOne($request); @@ -569,7 +678,7 @@ public function select($sql, array $bindings = [], $whereInFile = null, $writeTo * @throws \ClickHouseDB\Exception\TransportException * @throws \Exception */ - public function selectAsync($sql, array $bindings = [], $whereInFile = null, $writeToFile = null) + public function selectAsync($sql, array $bindings = [], $whereInFile = null, $writeToFile = null): Statement { $request = $this->prepareSelect($sql, $bindings, $whereInFile, $writeToFile); $this->_curler->addQueLoop($request); @@ -579,7 +688,7 @@ public function selectAsync($sql, array $bindings = [], $whereInFile = null, $wr /** * @param callable $callback */ - public function setProgressFunction(callable $callback) + public function setProgressFunction(callable $callback) : void { $this->xClickHouseProgress = $callback; } @@ -591,7 +700,7 @@ public function setProgressFunction(callable $callback) * @return Statement * @throws \ClickHouseDB\Exception\TransportException */ - public function write($sql, array $bindings = [], $exception = true) + public function write($sql, array $bindings = [], $exception = true): Statement { $request = $this->prepareWrite($sql, $bindings); $this->_curler->execOne($request); @@ -610,19 +719,17 @@ public function write($sql, array $bindings = [], $exception = true) * @return Statement * @throws \ClickHouseDB\Exception\TransportException */ - private function streaming(Stream $streamRW,CurlerRequest $request) + private function streaming(Stream $streamRW, CurlerRequest $request): Statement { - $callable=$streamRW->getClosure(); - $stream=$streamRW->getStream(); - + $callable = $streamRW->getClosure(); + $stream = $streamRW->getStream(); try { if (!is_callable($callable)) { - if ($streamRW->isWrite()) - { + if ($streamRW->isWrite()) { $callable = function ($ch, $fd, $length) use ($stream) { return ($line = fread($stream, $length)) ? $line : ''; @@ -636,8 +743,7 @@ private function streaming(Stream $streamRW,CurlerRequest $request) if ($streamRW->isGzipHeader()) { - if ($streamRW->isWrite()) - { + if ($streamRW->isWrite()) { $request->header('Content-Encoding', 'gzip'); $request->header('Content-Type', 'application/x-www-form-urlencoded'); } else { @@ -647,23 +753,18 @@ private function streaming(Stream $streamRW,CurlerRequest $request) } - $request->header('Transfer-Encoding', 'chunked'); - - if ($streamRW->isWrite()) - { + if ($streamRW->isWrite()) { $request->setReadFunction($callable); } else { $request->setWriteFunction($callable); - - // $request->setHeaderFunction($callableHead); } - $this->_curler->execOne($request,true); + $this->_curler->execOne($request, true); $response = new Statement($request); if ($response->isError()) { $response->error(); @@ -671,7 +772,7 @@ private function streaming(Stream $streamRW,CurlerRequest $request) return $response; } finally { if ($streamRW->isWrite()) - fclose($stream); + fclose($stream); } @@ -685,11 +786,11 @@ private function streaming(Stream $streamRW,CurlerRequest $request) * @return Statement * @throws \ClickHouseDB\Exception\TransportException */ - public function streamRead(Stream $streamRead,$sql,$bindings=[]) + public function streamRead(Stream $streamRead, $sql, $bindings = []): Statement { - $sql=$this->prepareQuery($sql,$bindings); - $request=$this->getRequestRead($sql); - return $this->streaming($streamRead,$request); + $sql = $this->prepareQuery($sql, $bindings); + $request = $this->getRequestRead($sql); + return $this->streaming($streamRead, $request); } @@ -700,10 +801,10 @@ public function streamRead(Stream $streamRead,$sql,$bindings=[]) * @return Statement * @throws \ClickHouseDB\Exception\TransportException */ - public function streamWrite(Stream $streamWrite,$sql,$bindings=[]) + public function streamWrite(Stream $streamWrite, $sql, $bindings = []): Statement { - $sql=$this->prepareQuery($sql,$bindings); + $sql = $this->prepareQuery($sql, $bindings); $request = $this->writeStreamData($sql); - return $this->streaming($streamWrite,$request); + return $this->streaming($streamWrite, $request); } } diff --git a/tests/BindingsPostTest.php b/tests/BindingsPostTest.php new file mode 100644 index 0000000..d48ee78 --- /dev/null +++ b/tests/BindingsPostTest.php @@ -0,0 +1,72 @@ +client->select( + 'SELECT number+{num_num:UInt8} as numbe_r, {xpx1:UInt32} as xpx1,{zoza:String} as zoza FROM system.numbers LIMIT 6', + [ + 'num_num'=>123, + 'xpx1'=>$xpx1, + 'zoza'=>'ziza' + ] + ); + $this->assertEquals(null,$result->fetchRow('x')); //0 + $this->assertEquals(null,$result->fetchRow('y')); //1 + $this->assertEquals($xpx1,$result->fetchRow('xpx1')); //2 + $this->assertEquals('ziza',$result->fetchRow('zoza'));//3 + $this->assertEquals(127,$result->fetchRow('numbe_r')); // 123+4 + $this->assertEquals(128,$result->fetchRow('numbe_r')); // 123+5 item + } + + public function testSelectAsKeys() + { + // chr(0....255); + $this->client->settings()->set('max_block_size', 100); + + $bind['k1']=1; + $bind['k2']=2; + + $select=[]; + for($z=0;$z<4;$z++) + { + $bind['k'.$z]=$z; + $select[]="{k{$z}:UInt16} as k{$z}"; + } + $rows=$this->client->select("SELECT ".implode(",\n",$select),$bind)->rows(); + + $this->assertNotEmpty($rows); + + $row=$rows[0]; + + for($z=10;$z<4;$z++) { + $this->assertArrayHasKey('k'.$z,$row); + $this->assertEquals($z,$row['k'.$z]); + + } + } + + public function testArrayAsPostParam() + { + $arr = [1,3,6]; + $result = $this->client->select( + 'SELECT {arr:Array(UInt8)} as arr', + [ + 'arr'=>json_encode($arr) + ] + ); + $this->assertEquals($arr, $result->fetchRow('arr')); + } + +} diff --git a/tests/BindingsTest.php b/tests/BindingsTest.php index 30197c3..16912b8 100644 --- a/tests/BindingsTest.php +++ b/tests/BindingsTest.php @@ -133,7 +133,7 @@ public function testBindselectAsync() ]; $a=$this->client->selectAsync(":a :b :c :aa :bb :cc ", $arr); - $this->assertEquals("'[A]' '[B]' '[C]' '[AA]' '[BB]' :cc FORMAT JSON",$a->sql()); + $this->assertEquals("'[A]' '[B]' '[C]' '[AA]' '[BB]' :cc FORMAT JSON",$a->sql()); $a=$this->client->selectAsync(":a1 :a2 :a3 :a11 :a23 :a5 :arra", $arr); $this->assertEquals("'[A1]' '[A2]' '[A3]' '[A11]' '[A23]' '[a5]' 1,2,3,4 FORMAT JSON",$a->sql()); diff --git a/tests/ClientAuthTest.php b/tests/ClientAuthTest.php new file mode 100644 index 0000000..641ec05 --- /dev/null +++ b/tests/ClientAuthTest.php @@ -0,0 +1,92 @@ + getenv('CLICKHOUSE_HOST'), + 'port' => getenv('CLICKHOUSE_PORT'), + 'username' => getenv('CLICKHOUSE_USER'), + 'password' => getenv('CLICKHOUSE_PASSWORD'), + + ]; + } + + private function execCommand(array $config):string + { + $cli = new Client($config); + $cli->verbose(); + $stream = fopen('php://memory', 'r+'); + // set stream to curl + $cli->transport()->setStdErrOut($stream); + // exec + $st=$cli->select('SElect 1 as ppp'); + $st->rows(); + fseek($stream,0,SEEK_SET); + return stream_get_contents($stream); + } + + public function testInsertDotTable() + { + $conf=$this->getConfig(); + + + // AUTH_METHOD_BASIC_AUTH + $conf['auth_method']=Transport\Http::AUTH_METHOD_BASIC_AUTH; + + $data=$this->execCommand($conf); + $this->assertIsString($data); + $this->assertStringContainsString('Authorization: Basic ZGVmYXVsdDo=',$data); + $this->assertStringNotContainsString('&user=default&password=',$data); + $this->assertStringNotContainsString('X-ClickHouse-User',$data); + + // AUTH_METHOD_QUERY_STRING + $conf['auth_method']=Transport\Http::AUTH_METHOD_QUERY_STRING; + + $data=$this->execCommand($conf); + $this->assertIsString($data); + $this->assertStringContainsString('&user=default&password=',$data); + $this->assertStringNotContainsString('Authorization: Basic ZGVmYXVsdDo=',$data); + $this->assertStringNotContainsString('X-ClickHouse-User',$data); + + + // AUTH_METHOD_HEADER + $conf['auth_method']=Transport\Http::AUTH_METHOD_HEADER; + + $data=$this->execCommand($conf); + $this->assertIsString($data); + $this->assertStringNotContainsString('&user=default&password=',$data); + $this->assertStringNotContainsString('Authorization: Basic ZGVmYXVsdDo=',$data); + $this->assertStringContainsString('X-ClickHouse-User',$data); + + } + + +} diff --git a/tests/ClientTest.php b/tests/ClientTest.php index c366418..13f7f50 100644 --- a/tests/ClientTest.php +++ b/tests/ClientTest.php @@ -5,6 +5,7 @@ use ClickHouseDB\Client; use ClickHouseDB\Exception\DatabaseException; use ClickHouseDB\Exception\QueryException; +use ClickHouseDB\Exception\TransportException; use ClickHouseDB\Query\WhereInFile; use ClickHouseDB\Query\WriteToFile; use ClickHouseDB\Quote\FormatLine; @@ -23,18 +24,19 @@ class ClientTest extends TestCase { use WithClient; - public function setUp() + public function setUp(): void { date_default_timezone_set('Europe/Moscow'); $this->client->enableHttpCompression(true); $this->client->ping(); + $this->client->setReadOnlyUser(false); } /** * */ - public function tearDown() + public function tearDown(): void { // } @@ -116,7 +118,7 @@ private function create_fake_file($file_name, $size = 1, $file_type = 'CSV') fwrite($handle, json_encode($j) . PHP_EOL); break; default: - fputcsv($handle, $j); + fputcsv($handle, $j,",",'"',"\\"); } $rows++; } @@ -149,82 +151,8 @@ private function create_table_summing_url_views() - /** - * - */ - public function testSqlConditions() - { - $input_params = [ - 'select_date' => ['2000-10-10', '2000-10-11', '2000-10-12'], - 'limit' => 5, - 'from_table' => 'table_x_y', - 'idid' => 0, - 'false' => false - ]; - $this->assertEquals( - 'SELECT * FROM table_x_y FORMAT JSON', - $this->client->selectAsync('SELECT * FROM {from_table}', $input_params)->sql() - ); - - $this->assertEquals( - 'SELECT * FROM table_x_y WHERE event_date IN (\'2000-10-10\',\'2000-10-11\',\'2000-10-12\') FORMAT JSON', - $this->client->selectAsync('SELECT * FROM {from_table} WHERE event_date IN (:select_date)', $input_params)->sql() - ); - - $this->client->enableQueryConditions(); - - $this->assertEquals( - 'SELECT * FROM ZZZ LIMIT 5 FORMAT JSON', - $this->client->selectAsync('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if}', $input_params)->sql() - ); - $this->assertEquals( - 'SELECT * FROM ZZZ NOOPE FORMAT JSON', - $this->client->selectAsync('SELECT * FROM ZZZ {if nope}LIMIT {limit}{else}NOOPE{/if}', $input_params)->sql() - ); - $this->assertEquals( - 'SELECT * FROM 0 FORMAT JSON', - $this->client->selectAsync('SELECT * FROM :idid', $input_params)->sql() - ); - - - $this->assertEquals( - 'SELECT * FROM FORMAT JSON', - $this->client->selectAsync('SELECT * FROM :false', $input_params)->sql() - ); - - - - $isset=[ - 'FALSE'=>false, - 'ZERO'=>0, - 'NULL'=>null - - ]; - - $this->assertEquals( - '|ZERO|| FORMAT JSON', - $this->client->selectAsync('{if FALSE}FALSE{/if}|{if ZERO}ZERO{/if}|{if NULL}NULL{/if}| ' ,$isset)->sql() - ); - - - - } - - - - public function testSqlDisableConditions() - { - $this->assertEquals('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if} FORMAT JSON', $this->client->selectAsync('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if}', [])->sql()); - $this->assertEquals('SELECT * FROM ZZZ {if limit}LIMIT 123{/if} FORMAT JSON', $this->client->selectAsync('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if}', ['limit'=>123])->sql()); - $this->client->cleanQueryDegeneration(); - $this->assertEquals('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if} FORMAT JSON', $this->client->selectAsync('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if}', ['limit'=>123])->sql()); - $this->restartClickHouseClient(); - $this->assertEquals('SELECT * FROM ZZZ {if limit}LIMIT 123{/if} FORMAT JSON', $this->client->selectAsync('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if}', ['limit'=>123])->sql()); - - - } public function testInsertNullable() { @@ -307,8 +235,62 @@ public function testSearchWithCyrillic() + public function testInsertNestedArray() + { + + $this->client->write("DROP TABLE IF EXISTS NestedNested_test"); + + $this->client->write(' + CREATE TABLE IF NOT EXISTS NestedNested_test ( + s_key String, + topics Nested( id UInt8 , ww Float32 ), + s_arr Array(String) + ) ENGINE = Memory +'); + + + // + $TestArrayPHP=['AAA'."'".'A',"BBBBB".'\\']; + $this->client->insert('NestedNested_test', [ + ['HASH\1', [11,33],[3.2,2.1],$TestArrayPHP], + ], ['s_key', 'topics.id','topics.ww','s_arr']); + + // wait read [0] => AAA'A [1] => BBBBB\ + + + + $st=$this->client->select('SELECT cityHash64(s_arr) as hash FROM NestedNested_test'); + $this->assertEquals('3072250716474788897', $st->fetchOne('hash')); + + + $row=$this->client->select('SELECT * FROM NestedNested_test ARRAY JOIN topics WHERE topics.id=11')->fetchOne(); + + $this->assertEquals(11, $row['topics.id']); + $this->assertEquals(3.2, $row['topics.ww']); + $this->assertEquals($TestArrayPHP, $row['s_arr']); + + + + + } + + + public function testStatementIterator() + { + $calc=0; + $state=$this->client->select('SELECT (number+1) as nnums FROM system.numbers LIMIT 5'); + foreach ($state as $key=>$value) { + $calc+=$value['nnums']; + } + $this->assertEquals(15,$calc); + } + + public function testRFCCSVAndTSVWrite() { + + $check_hash='5774439760453101066'; + $fileName=$this->tmpPath.'__testRFCCSVWrite'; $array_value_test="\n1\n2's'"; @@ -331,7 +313,7 @@ public function testRFCCSVAndTSVWrite() ['event_time'=>date('Y-m-d H:i:s'),'strs'=>'SOME STRING','flos'=>2.3,'ints'=>2,'arr1'=>[1,2,3],'arrs'=>["A","B"]], ['event_time'=>date('Y-m-d H:i:s'),'strs'=>'SOME\'STRING','flos'=>0,'ints'=>0,'arr1'=>[1,2,3],'arrs'=>["A","B"]], ['event_time'=>date('Y-m-d H:i:s'),'strs'=>"SOMET\nRI\n\"N\"G\\XX_ABCDEFG",'flos'=>0,'ints'=>0,'arr1'=>[1,2,3],'arrs'=>["A","B\nD\nC"]], - ['event_time'=>date('Y-m-d H:i:s'),'strs'=>"ID_ARRAY",'flos'=>0,'ints'=>0,'arr1'=>[1,2,3],'arrs'=>["A","B\nD\nC",$array_value_test]] + ['event_time'=>date('Y-m-d H:i:s'),'strs'=>"ID_ARRAY",'flos'=>0,'ints'=>0,'arr1'=>[1,2,4],'arrs'=>["A","B\nD\nC",$array_value_test]] ]; // 1.1 + 2.3 = 3.3999999761581 @@ -350,17 +332,17 @@ public function testRFCCSVAndTSVWrite() 'arrs', ]); - $st=$this->client->select('SELECT sipHash64(strs) as hash FROM testRFCCSVWrite WHERE like(strs,\'%ABCDEFG%\') '); - $this->assertEquals('5774439760453101066', $st->fetchOne('hash')); + $st=$this->client->select('SELECT sipHash64(strs) as hash FROM testRFCCSVWrite WHERE like(strs,\'%ABCDEFG%\') '); + + $this->assertEquals($check_hash, $st->fetchOne('hash')); $ID_ARRAY=$this->client->select('SELECT * FROM testRFCCSVWrite WHERE strs=\'ID_ARRAY\'')->fetchOne('arrs')[2]; $this->assertEquals($array_value_test, $ID_ARRAY); - $row=$this->client->select('SELECT round(sum(flos),1) as flos,round(sum(ints),1) as ints FROM testRFCCSVWrite')->fetchOne(); $this->assertEquals(3, $row['ints']); @@ -405,14 +387,13 @@ public function testRFCCSVAndTSVWrite() $st=$this->client->select('SELECT sipHash64(strs) as hash FROM testRFCCSVWrite WHERE like(strs,\'%ABCDEFG%\') '); - $this->assertEquals('17721988568158798984', $st->fetchOne('hash')); + $this->assertEquals($check_hash, $st->fetchOne('hash')); $ID_ARRAY=$this->client->select('SELECT * FROM testRFCCSVWrite WHERE strs=\'ID_ARRAY\'')->fetchOne('arrs')[2]; $this->assertEquals($array_value_test, $ID_ARRAY); - $row=$this->client->select('SELECT round(sum(flos),1) as flos,round(sum(ints),1) as ints FROM testRFCCSVWrite')->fetchOne(); $this->assertEquals(3, $row['ints']); @@ -517,12 +498,31 @@ public function testWriteToFileSelect() } } - +// /** +// * @expectedException \ClickHouseDB\Exception\QueryException +// */ +// public function testDBException() +// { +//// $this->expectException(\ClickHouseDB\Exception\QueryException::class); +// $this->client->settings()->set('max_threads',1); +// $this->client->settings()->set('max_memory_usage_for_user',1); +// $this->client->settings()->set('cast_keep_nullable',1); +// $this->client->settings()->set('network_zstd_compression_level',123); +//// $stat = $this->client->select("SELECT SHA1(toString(number)) as g FROM system.numbers GROUP BY g LIMIT 2 ")->rows(); +// $stat = $this->client->write("DROP TABLE IF EXISTS arrays_test_ints"); +// $stat = $this->client->write("CREATE TABLE IF NOT EXISTS arrays_test_ints ENGINE = Memory AS SELECT sqrt(-1)"); +//// $stat = $this->client->write("ALTER TABLE arrays_test_ints ON CLUSTER default_cluster UPDATE c2=0 WHERE 1;"); +// echo "-----\n"; +// var_dump($stat->response()->body()); +// echo "-----\n"; +// } /** - * @expectedException \ClickHouseDB\Exception\DatabaseException + * @expectedException \ClickHouseDB\Exception\QueryException */ public function testInsertCSVError() { + $this->expectException(\ClickHouseDB\Exception\QueryException::class); + $file_data_names = [ $this->tmpPath . '_testInsertCSV_clickHouseDB_test.1.data' ]; @@ -556,7 +556,7 @@ private function make_csv_SelectWhereIn($file_name, $array) $handle = fopen($file_name, 'w'); foreach ($array as $row) { - fputcsv($handle, $row); + fputcsv($handle, $row,",",'"',"\\"); } fclose($handle); @@ -675,7 +675,8 @@ public function testInsertCSV() $this->assertEquals(6408, $st->count()); $st = $this->client->select('SELECT * FROM summing_url_views LIMIT 4'); - $this->assertEquals(4, $st->countAll()); + + $this->assertGreaterThan(4, $st->countAll()); $stat = $this->client->insertBatchFiles('summing_url_views', $file_data_names, [ @@ -714,6 +715,12 @@ public function testSelectAsync() $this->assertEquals(2, $state2->fetchOne('ping')); } + public function testPartsInfo() + { + $this->create_table_summing_url_views(); + $this->insert_data_table_summing_url_views(); + $this->assertIsArray($this->client->partitions('summing_url_views')); + } /** * */ @@ -772,15 +779,15 @@ public function testTableExists() public function testExceptionWrite() { - $this->expectException(DatabaseException::class); + $this->expectException(QueryException::class); $this->client->write("DRAP TABLEX")->isError(); } public function testExceptionInsert() { - $this->expectException(DatabaseException::class); - $this->expectExceptionCode(60); + $this->expectException(QueryException::class); + $this->expectExceptionCode(60); // Table default.ZZZZZ doesn't exist $this->client->insert('bla_bla', [ ['HASH1', [11, 22, 33]], @@ -798,8 +805,8 @@ public function testExceptionInsertNoData() : void public function testExceptionSelect() { - $this->expectException(DatabaseException::class); - $this->expectExceptionCode(60); + $this->expectException(QueryException::class); + $this->expectExceptionCode(60); // Table not exists $this->client->select("SELECT * FROM XXXXX_SSS")->rows(); } @@ -816,6 +823,10 @@ public function testExceptionConnects() $db = new Client($config); $this->assertFalse($db->ping()); + + $this->expectException(TransportException::class); + $db->ping(true); + } public function testSettings() @@ -914,13 +925,13 @@ public function testInsertTableTimeout() $this->create_table_summing_url_views(); - $this->client->setTimeout(0.01); + $this->client->settings()->set('max_execution_time',0.1); $stat = $this->client->insertBatchFiles('summing_url_views', $file_data_names, [ 'event_time', 'url_hash', 'site_id', 'views', 'v_00', 'v_55' ]); - $this->client->ping(); + } /** * @@ -1046,7 +1057,7 @@ public function testUptime() public function testVersion() { $version = $this->client->getServerVersion(); - $this->assertRegExp('/(^[0-9]+.[0-9]+.[0-9]+.*$)/mi', $version); + $this->assertMatchesRegularExpression('/(^[0-9]+.[0-9]+.[0-9]+.*$)/mi', $version); } public function testServerSystemSettings() @@ -1081,4 +1092,5 @@ public function testStreamInsertFormatJSONEachRow() $statement = $this->client->select('SELECT * FROM summing_url_views'); $this->assertEquals(count(file($file_name)), $statement->count()); } + } diff --git a/tests/ConditionsTest.php b/tests/ConditionsTest.php new file mode 100644 index 0000000..bcf58d1 --- /dev/null +++ b/tests/ConditionsTest.php @@ -0,0 +1,240 @@ +30, + 'lastdays'=>3, + 'null'=>null, + 'false'=>false, + 'true'=>true, + 'zero'=>0, + 's_false'=>'false', + 's_null'=>'null', + 's_empty'=>'', + 'int30'=>30, + 'int1'=>1, + 'str0'=>'0', + 'str1'=>'1' + ]; + } + + private function condTest($sql,$equal) + { + $equal=$equal.' FORMAT JSON'; + $input_params=$this->getInputParams(); +// echo "-----\n".$this->client->selectAsync($sql, $input_params)->sql()."\n----\n"; + + $this->assertEquals($equal,$this->client->selectAsync($sql, $input_params)->sql()); + + + } + /** + * + */ + public function testSqlConditionsBig() + { + + + $select=" + 1: {if ll}NOT_SHOW{else}OK{/if}{if ll}NOT_SHOW{else}OK{/if} + 2: {if null}NOT_SHOW{else}OK{/if} + 3: {if qwert}NOT_SHOW{/if} + 4: {ifset zero} NOT_SHOW {else}OK{/if} + 5: {ifset false} NOT_SHOW {/if} + 6: {ifset s_false} OK {/if} + 7: {ifint zero} NOT_SHOW {/if} + 8: {if zero}OK{/if} + 9: {ifint s_empty}NOT_SHOW{/if} + 0: {ifint s_null}NOT_SHOW{/if} + 1: {ifset null} NOT_SHOW {/if} + + + CHECK_INT: + 0: {ifint zero} NOT_SHOW {/if} + 1: {ifint int1} OK {/if} + 2: {ifint int30} OK {/if} + 3: {ifint int30}OK {else} NOT_SHOW {/if} + 4: {ifint str0} NOT_SHOW {else}OK{/if} + 5: {ifint str1} OK_11 {else} NOT_SHOW {/if} + 6: {ifint int30} OK_22 {else} NOT_SHOW {/if} + 7: {ifint s_empty} NOT_SHOW {else} OK {/if} + 8: {ifint true} OK_33 {else} NOT_SHOW {/if} + + CHECK_STRING: + 0: {ifstring s_empty}NOT_SHOW{else}OK{/if} + 1: {ifstring s_null}OK{else}NOT_SHOW{/if} + LAST_LINE_1 + BOOL: + 1: {ifbool int1}NOT_SHOW{else}OK{/if} + 2: {ifbool int30}NOT_SHOW{else}OK_B11{/if} + 3: {ifbool zero}NOT_SHOW{else}OK_B22{/if} + 4: {ifbool false}NOT_SHOW{else}OK{/if} + 5: {ifbool true}OK{else}NOT_SHOW{/if} + 5: {ifbool true}OK{/if} + 6: {ifbool false}OK{/if} + 0: s_empty_check:{if s_empty} + + SHOW + + {/if} + CHECL_IFINT: + {ifint lastdays} + + + event_date>=today()-{lastdays} + + + {else} + + + event_date>=today() + + + {/if} LAST_LINE_2 + {ifint lastdays} + event_date>=today()-{lastdays} + {else} + event_date>=today() + {/if} + {ifset topSites} + AND site_id in ( {->Sites->Top(topSites)} ) + {/if} + {ifset topSites} + AND site_id in ( {->Sites->Top(topSites)} ) + {/if} + + + {if topSites} + AND site_id in ( {->Sites->Top(topSites)} ) + {/if} + + + "; + + $this->restartClickHouseClient(); + $this->client->enableQueryConditions(); + $input_params=$this->getInputParams(); + + $result=$this->client->selectAsync($select, $input_params)->sql(); + + $this->assertStringNotContainsString('NOT_SHOW',$result); + $this->assertStringContainsString('s_empty_check',$result); + $this->assertStringContainsString('LAST_LINE_1',$result); + $this->assertStringContainsString('LAST_LINE_2',$result); + $this->assertStringContainsString('CHECL_IFINT',$result); + $this->assertStringContainsString('CHECK_INT',$result); + $this->assertStringContainsString('CHECK_STRING',$result); + $this->assertStringContainsString('OK_11',$result); + $this->assertStringContainsString('OK_22',$result); + $this->assertStringContainsString('OK_33',$result); + $this->assertStringContainsString('OK_B11',$result); + $this->assertStringContainsString('OK_B22',$result); + $this->assertStringContainsString('=today()-3',$result); + +// echo "\n----\n$result\n----\n"; + + } + public function testSqlConditions1() + { + $this->restartClickHouseClient(); + $this->client->enableQueryConditions(); + + $this->condTest('{ifint s_empty}NOT_SHOW{/if}{ifbool int1}NOT_SHOW{else}OK{/if}{ifbool int30}NOT_SHOW{else}OK{/if}','OKOK'); + $this->condTest('{ifbool false}OK{/if}{ifbool true}OK{/if}{ifbool true}OK{else}NOT_SHOW{/if}','OKOK'); + $this->condTest('{ifstring s_empty}NOT_SHOW{else}OK{/if}{ifstring s_null}OK{else}NOT_SHOW{/if}','OKOK'); + $this->condTest('{ifint int1} OK {/if}',' OK'); + $this->condTest('{ifint s_empty}NOT_SHOW{/if}_1_','_1_'); + $this->condTest('1_{ifint str0} NOT_SHOW {else}OK{/if}_2','1_OK_2'); + $this->condTest('1_{if zero}OK{/if}_2','1_OK_2'); + $this->condTest('1_{if empty}OK{/if}_2','1__2'); + $this->condTest('1_{if s_false}OK{/if}_2','1_OK_2'); + $this->condTest('1_{if qwert}NOT_SHOW{/if}_2','1__2'); + $this->condTest('1_{ifset zero} NOT_SHOW {else}OK{/if}{ifset false} NOT_SHOW {/if}{ifset s_false} OK {/if}_2','1_OK OK_2'); + $this->condTest('1_{ifint zero} NOT_SHOW {/if}{if zero}OK{/if}{ifint s_empty}NOT_SHOW{/if}_2','1_OK_2'); + $this->condTest('1_{ifint s_null}NOT_SHOW{/if}{ifset null} NOT_SHOW {/if}_2','1__2'); + $this->condTest("{ifint lastdays}\n\n\nevent_date>=today()-{lastdays}-{lastdays}-{lastdays}\n\n\n{else}\n\n\nevent_date>=today()\n\n\n{/if}", "\n\n\nevent_date>=today()-3-3-3\n\n\n"); + $this->condTest("1_{ifint lastdays}\n2_{lastdays}_\t{int1}_{str0}_{str1}\n_6{else}\n\n{/if}", "1_\n2_3_\t1_0_1\n_6"); + $this->condTest("1_{ifint qwer}\n\n\n\n_6{else}\n{int1}{str0}{str1}\n{/if}\n_77", "1_\n101\n_77"); + + + } + public function testSqlConditions() + { + $input_params = [ + 'select_date' => ['2000-10-10', '2000-10-11', '2000-10-12'], + 'limit' => 5, + 'from_table' => 'table_x_y', + 'idid' => 0, + 'false' => false + ]; + + $this->assertEquals( + 'SELECT * FROM table_x_y FORMAT JSON', + $this->client->selectAsync('SELECT * FROM {from_table}', $input_params)->sql() + ); + + $this->assertEquals( + 'SELECT * FROM table_x_y WHERE event_date IN (\'2000-10-10\',\'2000-10-11\',\'2000-10-12\') FORMAT JSON', + $this->client->selectAsync('SELECT * FROM {from_table} WHERE event_date IN (:select_date)', $input_params)->sql() + ); + + $this->client->enableQueryConditions(); + + $this->assertEquals( + 'SELECT * FROM ZZZ LIMIT 5 FORMAT JSON', + $this->client->selectAsync('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if}', $input_params)->sql() + ); + + $this->assertEquals( + 'SELECT * FROM ZZZ NOOPE FORMAT JSON', + $this->client->selectAsync('SELECT * FROM ZZZ {if nope}LIMIT {limit}{else}NOOPE{/if}', $input_params)->sql() + ); + $this->assertEquals( + 'SELECT * FROM 0 FORMAT JSON', + $this->client->selectAsync('SELECT * FROM :idid', $input_params)->sql() + ); + + + $this->assertEquals( + 'SELECT * FROM FORMAT JSON', + $this->client->selectAsync('SELECT * FROM :false', $input_params)->sql() + ); + + + + $isset=[ + 'FALSE'=>false, + 'ZERO'=>0, + 'NULL'=>null + + ]; + + $this->assertEquals( + '|ZERO|| FORMAT JSON', + $this->client->selectAsync('{if FALSE}FALSE{/if}|{if ZERO}ZERO{/if}|{if NULL}NULL{/if}| ' ,$isset)->sql() + ); + } + + public function testSqlDisableConditions() + { + $this->assertEquals('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if} FORMAT JSON', $this->client->selectAsync('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if}', [])->sql()); + $this->assertEquals('SELECT * FROM ZZZ {if limit}LIMIT 123{/if} FORMAT JSON', $this->client->selectAsync('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if}', ['limit'=>123])->sql()); + $this->client->cleanQueryDegeneration(); + $this->assertEquals('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if} FORMAT JSON', $this->client->selectAsync('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if}', ['limit'=>123])->sql()); + $this->restartClickHouseClient(); + $this->assertEquals('SELECT * FROM ZZZ {if limit}LIMIT 123{/if} FORMAT JSON', $this->client->selectAsync('SELECT * FROM ZZZ {if limit}LIMIT {limit}{/if}', ['limit'=>123])->sql()); + + + } +} diff --git a/tests/FetchTest.php b/tests/FetchTest.php new file mode 100644 index 0000000..b7fb56d --- /dev/null +++ b/tests/FetchTest.php @@ -0,0 +1,64 @@ +client->select( + 'SELECT number FROM system.numbers LIMIT 5' + ); + $this->assertEquals(null,$result->fetchRow('x')); + $this->assertEquals(null,$result->fetchRow('y')); + $this->assertEquals(2,$result->fetchRow('number')); + $result->resetIterator(); + $this->assertEquals(null,$result->fetchRow('x')); + $this->assertEquals(1,$result->fetchRow('number')); + + + $this->assertEquals(null,$result->fetchOne('w')); + $this->assertEquals(null,$result->fetchOne('q')); + $this->assertEquals(0,$result->fetchOne('number')); + } + + public function testFetchOne() + { + $result = $this->client->select( + 'SELECT number FROM system.numbers LIMIT 5' + ); + // fetchOne + $this->assertEquals(0,$result->fetchOne('number')); + $this->assertEquals(0,$result->fetchOne('number')); + $this->assertEquals(0,$result->fetchOne('number')); + + // fetchRow + $this->assertEquals(0,$result->fetchRow('number')); + $this->assertEquals(1,$result->fetchRow('number')); + $this->assertEquals(2,$result->fetchRow('number')); + $result->resetIterator(); + $this->assertEquals(0,$result->fetchRow('number')); + $this->assertEquals(1,$result->fetchRow('number')); + $this->assertEquals(2,$result->fetchRow('number')); + } + + public function testCorrectInitOnFetchRow() + { + $result = $this->client->select( + 'SELECT number FROM system.numbers LIMIT 5' + ); + + $result->fetchRow(); + $this->assertTrue($result->isInited()); + } +} diff --git a/tests/FormatQueryTest.php b/tests/FormatQueryTest.php index 3bced11..fccf056 100644 --- a/tests/FormatQueryTest.php +++ b/tests/FormatQueryTest.php @@ -16,7 +16,7 @@ final class FormatQueryTest extends TestCase /** * @throws Exception */ - public function setUp() + public function setUp(): void { date_default_timezone_set('Europe/Moscow'); @@ -41,26 +41,52 @@ public function testCreateTableTEMPORARYNoSession() + $query="SELECT number as format_id FROM system.numbers LIMIT 1,1 FORMAT CSV"; $st = $this->client->select($query); $this->assertEquals($query, $st->sql()); $this->assertEquals('CSV', $st->getFormat()); + + $query="SELECT number as format_id FROM number(2) LIMIT 1,1 FORMAT TSVWithNamesAndTypes"; + $st = $this->client->select($query); + $this->assertEquals($query, $st->sql()); + $this->assertEquals('TSVWithNamesAndTypes', $st->getFormat()); + } public function testClientTimeoutSettings() { + // https://github.com/smi2/phpClickHouse/issues/168 + // Only setConnectTimeOut & getConnectTimeOut can be float + // max_execution_time - is integer in clickhouse source - Seconds $this->client->database('default'); - $timeout = 1.5; - $this->client->setTimeout($timeout); // 1500 ms - $this->assertSame($timeout, $this->client->getTimeout()); + $timeout = 1.515; // un support, "clickhouse source - Seconds" + $this->client->setTimeout($timeout); // 550 ms + $this->client->select('SELECT 123,123 as ping ')->rows(); + $this->assertSame(intval($timeout), intval($this->client->getTimeout())); + + $timeout = 2.55; // un support, "clickhouse source - Seconds" + $this->client->setTimeout($timeout); // 550 ms + $this->client->select('SELECT 123,123 as ping ')->rows(); + $this->assertSame(intval($timeout), intval($this->client->getTimeout())); $timeout = 10.0; $this->client->setTimeout($timeout); // 10 seconds - $this->assertSame($timeout, $this->client->getTimeout()); + $this->client->select('SELECT 123,123 as ping ')->rows(); + $this->assertSame(intval($timeout), $this->client->getTimeout()); - $timeout = 5.0; + // getConnectTimeOut is curl, can be float + $timeout = 5.14; $this->client->setConnectTimeOut($timeout); // 5 seconds - $this->assertSame($timeout, $this->client->getConnectTimeOut()); + $this->client->select('SELECT 123,123 as ping ')->rows(); + + + $this->assertSame(5.14, $this->client->getConnectTimeOut()); + } + + + + } diff --git a/tests/JsonTest.php b/tests/JsonTest.php index 67d32f8..19cf041 100644 --- a/tests/JsonTest.php +++ b/tests/JsonTest.php @@ -21,7 +21,7 @@ public function testJSONEachRow() $state=$this->client->select('SELECT sin(number) as sin,cos(number) as cos FROM {table_name} LIMIT 2 FORMAT JSONEachRow', ['table_name'=>'system.numbers']); $checkString='{"sin":0,"cos":1}'; - $this->assertContains($checkString,$state->rawData()); + $this->assertStringContainsString($checkString,$state->rawData()); $state=$this->client->select('SELECT round(4+sin(number),2) as sin,round(4+cos(number),2) as cos FROM {table_name} LIMIT 2 FORMAT JSONCompact', ['table_name'=>'system.numbers']); diff --git a/tests/ProgressAndEscapeTest.php b/tests/ProgressAndEscapeTest.php index abe30a9..39e967f 100644 --- a/tests/ProgressAndEscapeTest.php +++ b/tests/ProgressAndEscapeTest.php @@ -16,7 +16,7 @@ final class ProgressAndEscapeTest extends TestCase /** * @throws Exception */ - public function setUp() + public function setUp(): void { date_default_timezone_set('Europe/Moscow'); @@ -28,19 +28,26 @@ public function testProgressFunction() global $resultTest; $this->client->settings()->set('max_block_size', 1); - $this->client->progressFunction(function ($data) { global $resultTest; $resultTest=$data; }); - $st=$this->client->select('SELECT number,sleep(0.1) FROM system.numbers limit 4'); + $st=$this->client->select('SELECT number,sleep(0.1) FROM system.numbers limit 2 UNION ALL SELECT number,sleep(0.1) FROM system.numbers limit 12'); // read_rows + read_bytes + total_rows $this->assertArrayHasKey('read_rows',$resultTest); $this->assertArrayHasKey('read_bytes',$resultTest); - $this->assertArrayHasKey('total_rows',$resultTest); - - $this->assertGreaterThan(3,$resultTest['read_rows']); - $this->assertGreaterThan(3,$resultTest['read_bytes']); + $this->assertArrayHasKey('written_rows',$resultTest); + $this->assertArrayHasKey('written_bytes',$resultTest); + + if (isset($resultTest['total_rows'])) + { + $this->assertArrayHasKey('total_rows',$resultTest); + } else { + $this->assertArrayHasKey('total_rows_to_read',$resultTest); + } + // Disable test GreaterThan - travis-ci is fast + // $this->assertGreaterThan(1,$resultTest['read_rows']); + // $this->assertGreaterThan(1,$resultTest['read_bytes']); } } diff --git a/tests/Query/ExpressionTest.php b/tests/Query/Expression/RawTest.php similarity index 53% rename from tests/Query/ExpressionTest.php rename to tests/Query/Expression/RawTest.php index 22ad092..94e3533 100644 --- a/tests/Query/ExpressionTest.php +++ b/tests/Query/Expression/RawTest.php @@ -2,29 +2,36 @@ declare(strict_types=1); -namespace ClickHouseDB\Tests\Query; +namespace ClickHouseDB\Tests\Query\Expression; use ClickHouseDB\Query\Expression; use ClickHouseDB\Quote\FormatLine; use PHPUnit\Framework\TestCase; -final class ExpressionTest extends TestCase +final class RawTest extends TestCase { - public function testToString() : void + public function testNeedsEncoding() : void + { + self::assertEquals( + false, + (new Expression\Raw(''))->needsEncoding() + ); + } + public function testGetValue() : void { $expressionString = "UUIDStringToNum('0f372656-6a5b-4727-a4c4-f6357775d926')"; - $expressionObject = new Expression($expressionString); + $expressionObject = new Expression\Raw($expressionString); self::assertEquals( $expressionString, - (string) $expressionObject + $expressionObject->getValue() ); } public function testExpressionValueForInsert() : void { $expressionString = "UUIDStringToNum('0f372656-6a5b-4727-a4c4-f6357775d926')"; - $preparedValue = FormatLine::Insert([new Expression($expressionString)]); + $preparedValue = FormatLine::Insert([new Expression\Raw($expressionString)]); self::assertEquals( $expressionString, diff --git a/tests/SessionsTest.php b/tests/SessionsTest.php index 8a1c58b..aa40eb9 100644 --- a/tests/SessionsTest.php +++ b/tests/SessionsTest.php @@ -2,12 +2,12 @@ namespace ClickHouseDB\Tests; -use ClickHouseDB\Exception\DatabaseException; +use ClickHouseDB\Exception\QueryException; use PHPUnit\Framework\TestCase; /** * Class ClientTest - * @group ClientTest + * @group SessionsTest */ final class SessionsTest extends TestCase { @@ -16,7 +16,7 @@ final class SessionsTest extends TestCase /** * @throws Exception */ - public function setUp() + public function setUp(): void { date_default_timezone_set('Europe/Moscow'); @@ -25,7 +25,7 @@ public function setUp() public function testCreateTableTEMPORARYNoSession() { - $this->expectException(DatabaseException::class); + $this->expectException(QueryException::class); $this->client->write('DROP TABLE IF EXISTS phpunti_test_xxxx'); $this->client->write(' diff --git a/tests/StatementTest.php b/tests/StatementTest.php new file mode 100644 index 0000000..8b44b26 --- /dev/null +++ b/tests/StatementTest.php @@ -0,0 +1,129 @@ +client->select( + 'SELECT throwIf(1=1, \'Raised error\');' + ); + + $this->assertGreaterThanOrEqual(500, $result->getRequest()->response()->http_code()); + $this->assertTrue($result->isError()); + } + + /** + * @link https://github.com/smi2/phpClickHouse/issues/144 + * @link https://clickhouse.com/docs/en/interfaces/http#http_response_codes_caveats + * + * During execution of query it is possible to get ExceptionWhileProcessing in Clickhouse + * In that case HTTP status code of Clickhouse interface would be 200 + * and it is kind of "expected" behaviour of CH + */ + public function testIsErrorWithOkStatusCode() + { + // value of "number" in query must be greater than 100 thousand + // for part of CH response to be flushed to client with 200 status code + // and further ExceptionWhileProcessing occurrence + $result = $this->client->select( + 'SELECT number, throwIf(number=100100, \'Raised error\') FROM system.numbers;' + ); + + $this->assertEquals(200, $result->getRequest()->response()->http_code()); + $this->assertTrue($result->isError()); + } + + /** + * @link https://github.com/smi2/phpClickHouse/issues/223 + * @see src/Statement.php:14 + * + * The response data may legitimately contain text that matches the + * CLICKHOUSE_ERROR_REGEX pattern. This is particularly common when querying + * system tables like system.mutations, where error messages are stored as data + */ + public function testIsNotErrorWhenJsonBodyContainsDbExceptionMessage() + { + $result = $this->client->select( + "SELECT + 'mutation_123456' AS mutation_id, + 'Code: 243. DB::Exception: Cannot reserve 61.64 GiB, not enough space. (NOT_ENOUGH_SPACE) (version 24.3.2.23 (official build))' AS latest_fail_reason" + ); + + $this->assertEquals(200, $result->getRequest()->response()->http_code()); + $this->assertFalse($result->isError()); + } + + /** + * @dataProvider dataProvider + */ + public function testParseErrorClickHouse( + string $errorMessage, + string $exceptionMessage, + int $exceptionCode + ): void { + $requestMock = $this->createMock(CurlerRequest::class); + $responseMock = $this->createMock(CurlerResponse::class); + + $responseMock->expects($this->any())->method('body')->will($this->returnValue($errorMessage)); + $responseMock->expects($this->any())->method('error_no')->will($this->returnValue(0)); + $responseMock->expects($this->any())->method('error')->will($this->returnValue(false)); + + $requestMock->expects($this->any())->method('response')->will($this->returnValue($responseMock)); + + $statement = new Statement($requestMock); + $this->assertInstanceOf(Statement::class, $statement); + + $this->expectException(DatabaseException::class); + $this->expectExceptionMessage($exceptionMessage); + $this->expectExceptionCode($exceptionCode); + + $statement->error(); + } + + /** + * @return Generator + */ + public function dataProvider(): Generator + { + yield 'Unknown setting readonly' => [ + 'Code: 115. DB::Exception: Unknown setting readonly[0], e.what() = DB::Exception', + 'Unknown setting readonly[0]', + 115, + ]; + + yield 'Unknown user x' => [ + 'Code: 192. DB::Exception: Unknown user x, e.what() = DB::Exception', + 'Unknown user x', + 192, + ]; + + yield 'Table default.ZZZZZ doesn\'t exist.' => [ + 'Code: 60. DB::Exception: Table default.ZZZZZ doesn\'t exist., e.what() = DB::Exception', + 'Table default.ZZZZZ doesn\'t exist.', + 60, + ]; + + yield 'Authentication failed' => [ + 'Code: 516. DB::Exception: test_username: Authentication failed: password is incorrect or there is no user with such name. (AUTHENTICATION_FAILED) (version 22.8.3.13 (official build))', + 'test_username: Authentication failed: password is incorrect or there is no user with such name. (AUTHENTICATION_FAILED)', + 516 + ]; + } +} diff --git a/tests/StreamTest.php b/tests/StreamTest.php index fa7ee5c..abc0b4b 100644 --- a/tests/StreamTest.php +++ b/tests/StreamTest.php @@ -38,7 +38,7 @@ public function testStreamRead() $checkString='{"max":0,"cos":1}'; - $this->assertContains($checkString,$bufferCheck); + $this->assertStringContainsString($checkString,$bufferCheck); } public function testStreamInsert() diff --git a/tests/StrictQuoteLineTest.php b/tests/StrictQuoteLineTest.php index cd4c65d..9b00d8d 100644 --- a/tests/StrictQuoteLineTest.php +++ b/tests/StrictQuoteLineTest.php @@ -19,7 +19,7 @@ class StrictQuoteLineTest extends TestCase /** * @return void */ - public function setUp() + public function setUp(): void { $this->client->write('DROP TABLE IF EXISTS cities'); $this->client->write(' diff --git a/tests/Type/UInt64Test.php b/tests/Type/UInt64Test.php index 14bbe7e..0d62ffa 100644 --- a/tests/Type/UInt64Test.php +++ b/tests/Type/UInt64Test.php @@ -22,7 +22,7 @@ final class UInt64Test extends TestCase /** * @return void */ - public function setUp() + public function setUp(): void { $this->client->write('DROP TABLE IF EXISTS uint64_data'); $this->client->write(' diff --git a/tests/docker-clickhouse/.gitignore b/tests/docker-clickhouse/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index f3a4c7a..f883d6b 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -1,12 +1,18 @@ version: '3' services: + clickhouse-server: - image: yandex/clickhouse-server + image: clickhouse/clickhouse-server:21.9 hostname: clickhouse container_name: clickhouse ports: - - 9000:9000 + - 19000:9000 - 8123:8123 + sysctls: + net.core.somaxconn: 1024 + net.ipv4.tcp_syncookies: 0 + volumes: + - "./docker-clickhouse:/var/lib/clickhouse" ulimits: nofile: soft: 262144