diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3557610 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +.* export-ignore +/examples export-ignore +/tests export-ignore +*.dist export-ignore +include.php export-ignore diff --git a/CHANGELOG.md b/CHANGELOG.md index 16e7592..8d9558f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,28 @@ PHP ClickHouse wrapper - Changelog ====================== +### 2025-01-14 [Release 1.6.0] +* Support PHP 8.4 + +### 2024-01-18 [Release 1.5.3] +* Fix release 1.5.2 +* Support php 7 +* Update Statement.php #204 +* fix(#202): Fix converting boolean when inserting into int and fix(#194): Fix unexpected readonly mode with specific string in query #203 +* Update README.md #199 +* remove dev files for --prefer-dist #192 + + + +### 2024-01-16 [Release 1.5.2] +* Update Statement.php #204 +* fix(#202): Fix converting boolean when inserting into int and fix(#194): Fix unexpected readonly mode with specific string in query #203 +* Update README.md #199 +* remove dev files for --prefer-dist #192 + +### May 25, 2023 [ 1.5.1 ] +* BREAKING CHANGES Post type bindings support + ### 2022-12-20 [Release 1.5.0] * Change exceptionCode in Clickhouse version 22.8.3.13 (official build) #180 diff --git a/README.md b/README.md index ad5c0ae..0a15fb3 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,9 @@ PHP ClickHouse wrapper ====================== -[![Build Status](https://travis-ci.org/smi2/phpClickHouse.svg)](https://travis-ci.org/smi2/phpClickHouse) [![Downloads](https://poser.pugx.org/smi2/phpClickHouse/d/total.svg)](https://packagist.org/packages/smi2/phpClickHouse) [![Packagist](https://poser.pugx.org/smi2/phpClickHouse/v/stable.svg)](https://packagist.org/packages/smi2/phpClickHouse) [![Licence](https://poser.pugx.org/smi2/phpClickHouse/license.svg)](https://packagist.org/packages/smi2/phpClickHouse) -[![Quality Score](https://scrutinizer-ci.com/g/smi2/phpClickHouse/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/smi2/phpClickHouse) -[![Code Coverage](https://scrutinizer-ci.com/g/smi2/phpClickHouse/badges/coverage.png?b=master)](https://scrutinizer-ci.com/g/smi2/phpClickHouse) ## Features @@ -47,7 +44,9 @@ if (!$db->ping()) echo 'Error connect'; Last stable version for * php 5.6 <= `1.1.2` * php 7.2 <= `1.3.10` -* php 7.3 >= `1.4.x` +* php 7.3 >= `1.4.x ... 1.5.X` +* php 8.4 >= `1.6.0` + [Packagist](https://packagist.org/packages/smi2/phpclickhouse) @@ -59,7 +58,8 @@ $config = [ 'host' => '192.168.1.1', 'port' => '8123', 'username' => 'default', - 'password' => '' + 'password' => '', + 'https' => true ]; $db = new ClickHouseDB\Client($config); $db->database('default'); diff --git a/composer.json b/composer.json index c52c0ad..edd3a4c 100644 --- a/composer.json +++ b/composer.json @@ -13,7 +13,7 @@ } ], "require": { - "php": "^7.3|^8.0", + "php": "^8.0", "ext-curl": "*", "ext-json": "*" }, diff --git a/src/Client.php b/src/Client.php index 0196260..9cb841e 100644 --- a/src/Client.php +++ b/src/Client.php @@ -38,7 +38,7 @@ */ class Client { - const SUPPORTED_FORMATS = ['TabSeparated', 'TabSeparatedWithNames', 'CSV', 'CSVWithNames', 'JSONEachRow']; + const SUPPORTED_FORMATS = ['TabSeparated', 'TabSeparatedWithNames', 'CSV', 'CSVWithNames', 'JSONEachRow','CSVWithNamesAndTypes','TSVWithNamesAndTypes']; /** @var Http */ private $transport; @@ -52,7 +52,7 @@ class Client /** @var string */ private $connectHost; - /** @var string */ + /** @var int */ private $connectPort; /** @var int */ @@ -68,19 +68,19 @@ class Client public function __construct(array $connectParams, array $settings = []) { if (!isset($connectParams['username'])) { - throw new \InvalidArgumentException('not set username'); + throw new \InvalidArgumentException('not set username'); } if (!isset($connectParams['password'])) { - throw new \InvalidArgumentException('not set password'); + throw new \InvalidArgumentException('not set password'); } if (!isset($connectParams['port'])) { - throw new \InvalidArgumentException('not set port'); + throw new \InvalidArgumentException('not set port'); } if (!isset($connectParams['host'])) { - throw new \InvalidArgumentException('not set host'); + throw new \InvalidArgumentException('not set host'); } if (array_key_exists('auth_method', $connectParams)) { @@ -89,7 +89,7 @@ public function __construct(array $connectParams, array $settings = []) 'Invalid value for "auth_method" param. Should be one of [%s].', json_encode(Http::AUTH_METHODS_LIST) ); - throw new \InvalidArgumentException($errorMessage); + throw new \InvalidArgumentException($errorMessage); } $this->authMethod = $connectParams['auth_method']; @@ -97,7 +97,7 @@ public function __construct(array $connectParams, array $settings = []) $this->connectUsername = $connectParams['username']; $this->connectPassword = $connectParams['password']; - $this->connectPort = $connectParams['port']; + $this->connectPort = intval($connectParams['port']); $this->connectHost = $connectParams['host']; // init transport class @@ -164,7 +164,7 @@ public function addQueryDegeneration(Degeneration $degeneration) * * @return bool */ - public function enableQueryConditions() + public function enableQueryConditions(): bool { return $this->transport->addQueryDegeneration(new Conditions()); } @@ -174,22 +174,22 @@ public function enableQueryConditions() * * @param string $host */ - public function setHost($host) + public function setHost($host): void { $this->connectHost = $host; $this->transport()->setHost($host); } /** - * @return Settings + * max_execution_time , in int value (seconds) */ - public function setTimeout(int $timeout) + public function setTimeout($timeout): Settings { - return $this->settings()->max_execution_time($timeout); + return $this->settings()->max_execution_time(intval($timeout)); } /** - * @return float + * @return int */ public function getTimeout(): int { @@ -199,7 +199,7 @@ public function getTimeout(): int /** * ConnectTimeOut in seconds ( support 1.5 = 1500ms ) */ - public function setConnectTimeOut(float $connectTimeOut) + public function setConnectTimeOut(float $connectTimeOut): void { $this->transport()->setConnectTimeOut($connectTimeOut); } @@ -215,10 +215,10 @@ public function getConnectTimeOut(): float /** * @return Http */ - public function transport() + public function transport(): Http { if (!$this->transport) { - throw new \InvalidArgumentException('Empty transport class'); + throw new \InvalidArgumentException('Empty transport class'); } return $this->transport; @@ -227,7 +227,7 @@ public function transport() /** * @return string */ - public function getConnectHost() + public function getConnectHost(): string { return $this->connectHost; } @@ -235,7 +235,7 @@ public function getConnectHost() /** * @return string */ - public function getConnectPassword() + public function getConnectPassword(): string { return $this->connectPassword; } @@ -243,9 +243,9 @@ public function getConnectPassword() /** * @return string */ - public function getConnectPort() + public function getConnectPort(): string { - return $this->connectPort; + return strval($this->connectPort); } /** @@ -292,7 +292,7 @@ public function settings() * @param string|null $useSessionId * @return $this */ - public function useSession(string $useSessionId = null) + public function useSession(string $useSessionId = '') { if (!$this->settings()->getSessionId()) { if (!$useSessionId) { @@ -383,14 +383,15 @@ public function enableExtremes(bool $flag = true) } /** - * @param mixed[] $bindings + * @param string $sql + * @param array $bindings * @return Statement */ public function select( string $sql, array $bindings = [], - WhereInFile $whereInFile = null, - WriteToFile $writeToFile = null + ?WhereInFile $whereInFile = null, + ?WriteToFile $writeToFile = null ) { return $this->transport()->select($sql, $bindings, $whereInFile, $writeToFile); @@ -437,8 +438,8 @@ public function progressFunction(callable $callback) public function selectAsync( string $sql, array $bindings = [], - WhereInFile $whereInFile = null, - WriteToFile $writeToFile = null + ?WhereInFile $whereInFile = null, + ?WriteToFile $writeToFile = null ) { return $this->transport()->selectAsync($sql, $bindings, $whereInFile, $writeToFile); @@ -612,7 +613,7 @@ public function insertBatchFiles(string $tableName, $fileNames, array $columns = foreach ($fileNames as $fileName) { if (!is_file($fileName) || !is_readable($fileName)) { - throw new QueryException('Cant read file: ' . $fileName . ' ' . (is_file($fileName) ? '' : ' is not file')); + throw new QueryException('Cant read file: ' . $fileName . ' ' . (is_file($fileName) ? '' : ' is not file')); } if (empty($columns)) { @@ -805,14 +806,14 @@ public function isExists(string $database, string $table) /** * List of partitions * - * @return mixed[][] + * @return array * @throws \Exception */ - public function partitions(string $table, int $limit = null, bool $active = null) + public function partitions(string $table, int $limit = 0, ?bool $active = null) { $database = $this->settings()->getDatabase(); $whereActiveClause = $active === null ? '' : sprintf(' AND active = %s', (int)$active); - $limitClause = $limit !== null ? ' LIMIT ' . $limit : ''; + $limitClause = $limit > 0 ? ' LIMIT ' . $limit : ''; return $this->select(<<sql); + return preg_match('#{[\w+]+:[\w+()]+}#',$this->sql); } public function getUrlBindingsParams():array diff --git a/src/Quote/StrictQuoteLine.php b/src/Quote/StrictQuoteLine.php index 1191e67..8d71252 100644 --- a/src/Quote/StrictQuoteLine.php +++ b/src/Quote/StrictQuoteLine.php @@ -113,6 +113,8 @@ function ($v) use ($enclosure_esc, $encode_esc) { return (string) $value; } + $value = is_bool($value) ? ($value ? 'true' : 'false') : $value; + if (is_string($value) && $encode) { if ($tabEncode) { return str_replace(["\t", "\n"], ['\\t', '\\n'], $value); diff --git a/src/Settings.php b/src/Settings.php index 3b33fcb..2ff547c 100644 --- a/src/Settings.php +++ b/src/Settings.php @@ -2,15 +2,8 @@ namespace ClickHouseDB; -use ClickHouseDB\Transport\Http; - class Settings { - /** - * @var Http - */ - private $client = null; - /** * @var array */ @@ -18,16 +11,10 @@ class Settings private $_ReadOnlyUser = false; - /** - * @var bool - */ - private $_isHttps = false; - /** * Settings constructor. - * @param Http $client */ - public function __construct(Http $client) + public function __construct() { $default = [ 'extremes' => false, @@ -38,7 +25,6 @@ public function __construct(Http $client) ]; $this->settings = $default; - $this->client = $client; } /** @@ -233,4 +219,9 @@ public function getSetting(string $name) return $this->get($name); } + + public function clear():void + { + $this->settings = []; + } } diff --git a/src/Statement.php b/src/Statement.php index 7639c61..9701d51 100644 --- a/src/Statement.php +++ b/src/Statement.php @@ -11,6 +11,8 @@ class Statement implements \Iterator { + private const CLICKHOUSE_ERROR_REGEX = "%Code:\s(\d+)\.\s*DB::Exception\s*:\s*(.*)(?:,\s*e\.what|\(version).*%ius"; + /** * @var string|mixed */ @@ -133,18 +135,32 @@ public function sql() * @param string $body * @return array|bool */ - private function parseErrorClickHouse($body) + private function parseErrorClickHouse(string $body) { $body = trim($body); - $mathes = []; + $matches = []; // Code: 115. DB::Exception: Unknown setting readonly[0], e.what() = DB::Exception // Code: 192. DB::Exception: Unknown user x, e.what() = DB::Exception // Code: 60. DB::Exception: Table default.ZZZZZ doesn't exist., e.what() = DB::Exception // Code: 516. DB::Exception: test_username: Authentication failed: password is incorrect or there is no user with such name. (AUTHENTICATION_FAILED) (version 22.8.3.13 (official build)) - if (preg_match("%Code:\s(\d+).\s*DB\:\:Exception\s*:\s*(.*)(?:\,\s*e\.what|\(version).*%ius", $body, $mathes)) { - return ['code' => $mathes[1], 'message' => $mathes[2]]; + if (preg_match(self::CLICKHOUSE_ERROR_REGEX, $body, $matches)) { + return ['code' => $matches[1], 'message' => $matches[2]]; + } + + return false; + } + + private function hasErrorClickhouse(string $body, string $contentType): bool { + if (false === stripos($contentType, 'application/json')) { + return preg_match(self::CLICKHOUSE_ERROR_REGEX, $body) === 1; + } + + try { + json_decode($body, true, 512, JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + return true; } return false; @@ -197,12 +213,24 @@ public function error() * @return bool * @throws Exception\TransportException */ - public function isError() + public function isError(): bool { - return ($this->response()->http_code() !== 200 || $this->response()->error_no()); + if ($this->response()->http_code() !== 200) { + return true; + } + + if ($this->response()->error_no()) { + return true; + } + + if ($this->hasErrorClickhouse($this->response()->body(), $this->response()->content_type())) { + return true; + } + + return false; } - private function check() : bool + private function check(): bool { if (!$this->_request->isResponseExists()) { throw QueryException::noResponse(); @@ -534,6 +562,14 @@ public function rows() return $this->array_data; } + /** + * @return false|string + */ + public function jsonRows() + { + return json_encode($this->rows(), JSON_PRETTY_PRINT); + } + /** * @param array|string $arr * @param null|string|array $path diff --git a/src/Transport/CurlerRequest.php b/src/Transport/CurlerRequest.php index ad1c5fc..e62b9a2 100644 --- a/src/Transport/CurlerRequest.php +++ b/src/Transport/CurlerRequest.php @@ -124,7 +124,6 @@ public function __construct($id = false) CURLOPT_HEADER => TRUE, CURLOPT_FOLLOWLOCATION => TRUE, CURLOPT_AUTOREFERER => 1, // при редиректе подставлять в «Referer:» значение из «Location:» - CURLOPT_BINARYTRANSFER => 1, // передавать в binary-safe CURLOPT_RETURNTRANSFER => TRUE, CURLOPT_USERAGENT => 'smi2/PHPClickHouse/client', ); diff --git a/src/Transport/CurlerRolling.php b/src/Transport/CurlerRolling.php index 989b19f..e21c0b5 100644 --- a/src/Transport/CurlerRolling.php +++ b/src/Transport/CurlerRolling.php @@ -57,7 +57,7 @@ class CurlerRolling /** * */ - public function __destructor() + public function __destruct() { $this->close(); } diff --git a/src/Transport/Http.php b/src/Transport/Http.php index be59d49..cd1d286 100644 --- a/src/Transport/Http.php +++ b/src/Transport/Http.php @@ -95,6 +95,12 @@ class Http * @var null|resource */ private $stdErrOut = null; + + /** + * @var null|resource + */ + private $handle = null; + /** * Http constructor. * @param string $host @@ -113,7 +119,7 @@ public function __construct($host, $port, $username, $password, $authMethod = nu $this->_authMethod = $authMethod; } - $this->_settings = new Settings($this); + $this->_settings = new Settings(); $this->setCurler(); } @@ -266,7 +272,8 @@ private function newRequest($extendinfo): CurlerRequest } $new->timeOut($this->settings()->getTimeOut()); - $new->connectTimeOut($this->_connectTimeOut);//->keepAlive(); // one sec + $new->connectTimeOut($this->_connectTimeOut); + $new->keepAlive(); $new->verbose(boolval($this->_verbose)); return $new; @@ -301,12 +308,12 @@ private function makeRequest(Query $query, array $urlParams = [], bool $query_as */ if ($query->isUseInUrlBindingsParams()) { - $urlParams=array_replace_recursive($query->getUrlBindingsParams()); + $urlParams = array_replace_recursive($urlParams, $query->getUrlBindingsParams()); } + $url = $this->getUrl($urlParams); $new->url($url); - if (!$query_as_string) { $new->parameters_json($sql); } @@ -625,6 +632,15 @@ private function prepareWrite($sql, $bindings = []): CurlerRequest } $query = $this->prepareQuery($sql, $bindings); + + if (strpos($sql, 'ON CLUSTER') === false) { + return $this->getRequestWrite($query); + } + + if (strpos($sql, 'CREATE') === 0 || strpos($sql, 'DROP') === 0 || strpos($sql, 'ALTER') === 0) { + $query->setFormat('JSON'); + } + return $this->getRequestWrite($query); } @@ -739,13 +755,11 @@ private function streaming(Stream $streamRW, CurlerRequest $request): Statement $request->header('Transfer-Encoding', 'chunked'); - if ($streamRW->isWrite()) { $request->setReadFunction($callable); } else { $request->setWriteFunction($callable); - // $request->setHeaderFunction($callableHead); } diff --git a/tests/BindingsPostTest.php b/tests/BindingsPostTest.php index 1cc7462..d48ee78 100644 --- a/tests/BindingsPostTest.php +++ b/tests/BindingsPostTest.php @@ -57,4 +57,16 @@ public function testSelectAsKeys() } } + public function testArrayAsPostParam() + { + $arr = [1,3,6]; + $result = $this->client->select( + 'SELECT {arr:Array(UInt8)} as arr', + [ + 'arr'=>json_encode($arr) + ] + ); + $this->assertEquals($arr, $result->fetchRow('arr')); + } + } diff --git a/tests/ClientTest.php b/tests/ClientTest.php index 2922083..13f7f50 100644 --- a/tests/ClientTest.php +++ b/tests/ClientTest.php @@ -118,7 +118,7 @@ private function create_fake_file($file_name, $size = 1, $file_type = 'CSV') fwrite($handle, json_encode($j) . PHP_EOL); break; default: - fputcsv($handle, $j); + fputcsv($handle, $j,",",'"',"\\"); } $rows++; } @@ -556,7 +556,7 @@ private function make_csv_SelectWhereIn($file_name, $array) $handle = fopen($file_name, 'w'); foreach ($array as $row) { - fputcsv($handle, $row); + fputcsv($handle, $row,",",'"',"\\"); } fclose($handle); diff --git a/tests/FormatQueryTest.php b/tests/FormatQueryTest.php index 3862292..fccf056 100644 --- a/tests/FormatQueryTest.php +++ b/tests/FormatQueryTest.php @@ -61,8 +61,13 @@ public function testClientTimeoutSettings() // max_execution_time - is integer in clickhouse source - Seconds $this->client->database('default'); - $timeout = 0.55; // un support, "clickhouse source - Seconds" - $this->client->setTimeout($timeout); // 1500 ms + $timeout = 1.515; // un support, "clickhouse source - Seconds" + $this->client->setTimeout($timeout); // 550 ms + $this->client->select('SELECT 123,123 as ping ')->rows(); + $this->assertSame(intval($timeout), intval($this->client->getTimeout())); + + $timeout = 2.55; // un support, "clickhouse source - Seconds" + $this->client->setTimeout($timeout); // 550 ms $this->client->select('SELECT 123,123 as ping ')->rows(); $this->assertSame(intval($timeout), intval($this->client->getTimeout())); @@ -71,12 +76,14 @@ public function testClientTimeoutSettings() $this->client->select('SELECT 123,123 as ping ')->rows(); $this->assertSame(intval($timeout), $this->client->getTimeout()); - // getConnectTimeOut is curl, can be float $timeout = 5.14; $this->client->setConnectTimeOut($timeout); // 5 seconds $this->client->select('SELECT 123,123 as ping ')->rows(); + + $this->assertSame(5.14, $this->client->getConnectTimeOut()); + } diff --git a/tests/StatementTest.php b/tests/StatementTest.php index f4117a4..8b44b26 100644 --- a/tests/StatementTest.php +++ b/tests/StatementTest.php @@ -17,6 +17,59 @@ */ final class StatementTest extends TestCase { + use WithClient; + + public function testIsError() + { + $result = $this->client->select( + 'SELECT throwIf(1=1, \'Raised error\');' + ); + + $this->assertGreaterThanOrEqual(500, $result->getRequest()->response()->http_code()); + $this->assertTrue($result->isError()); + } + + /** + * @link https://github.com/smi2/phpClickHouse/issues/144 + * @link https://clickhouse.com/docs/en/interfaces/http#http_response_codes_caveats + * + * During execution of query it is possible to get ExceptionWhileProcessing in Clickhouse + * In that case HTTP status code of Clickhouse interface would be 200 + * and it is kind of "expected" behaviour of CH + */ + public function testIsErrorWithOkStatusCode() + { + // value of "number" in query must be greater than 100 thousand + // for part of CH response to be flushed to client with 200 status code + // and further ExceptionWhileProcessing occurrence + $result = $this->client->select( + 'SELECT number, throwIf(number=100100, \'Raised error\') FROM system.numbers;' + ); + + $this->assertEquals(200, $result->getRequest()->response()->http_code()); + $this->assertTrue($result->isError()); + } + + /** + * @link https://github.com/smi2/phpClickHouse/issues/223 + * @see src/Statement.php:14 + * + * The response data may legitimately contain text that matches the + * CLICKHOUSE_ERROR_REGEX pattern. This is particularly common when querying + * system tables like system.mutations, where error messages are stored as data + */ + public function testIsNotErrorWhenJsonBodyContainsDbExceptionMessage() + { + $result = $this->client->select( + "SELECT + 'mutation_123456' AS mutation_id, + 'Code: 243. DB::Exception: Cannot reserve 61.64 GiB, not enough space. (NOT_ENOUGH_SPACE) (version 24.3.2.23 (official build))' AS latest_fail_reason" + ); + + $this->assertEquals(200, $result->getRequest()->response()->http_code()); + $this->assertFalse($result->isError()); + } + /** * @dataProvider dataProvider */ @@ -38,7 +91,7 @@ public function testParseErrorClickHouse( $this->assertInstanceOf(Statement::class, $statement); $this->expectException(DatabaseException::class); - $this->expectDeprecationMessage($exceptionMessage); + $this->expectExceptionMessage($exceptionMessage); $this->expectExceptionCode($exceptionCode); $statement->error(); diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 1b5fe1c..f883d6b 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -6,7 +6,7 @@ services: hostname: clickhouse container_name: clickhouse ports: - - 9000:9000 + - 19000:9000 - 8123:8123 sysctls: net.core.somaxconn: 1024