Merge pull request #19 from sartor/batch-insert

Author: sartor

Command.php

@@ -7,7 +7,7 @@
 
 namespace bashkarev\clickhouse;
 
-use ClickHouseDB\Exception\DatabaseException;
+use ClickHouseDB\Exception\QueryException;
 use ClickHouseDB\Statement;
 use Yii;
 use yii\db\Exception;
@@ -57,7 +57,7 @@ protected function queryInternal($method, $fetchMode = null)
             }
             $result = call_user_func_array([$this, $method], [$statement, $fetchMode]);
             Yii::endProfile($token, 'bashkarev\clickhouse\Command::query');
-        } catch (DatabaseException $e) {
+        } catch (QueryException $e) {
             Yii::endProfile($token, 'bashkarev\clickhouse\Command::query');
             throw new Exception($e->getMessage());
         } catch (\Exception $e) {
@@ -91,7 +91,7 @@ public function execute()
             $statement = $this->db->execute($rawSql);
             $this->refreshTableSchema();
             return (int)(!$statement->isError());
-        } catch (DatabaseException $e) {
+        } catch (QueryException $e) {
             Yii::endProfile($token, __METHOD__);
             throw new Exception($e->getMessage());
         } catch (\Exception $e) {

Connection.php

@@ -148,6 +148,11 @@ public function getIsActive()
         return ($this->_client !== null);
     }
 
+    public function getClient(): Client
+    {
+        return $this->_client;
+    }
+
     /**
      * @inheritdoc
      * @throws NotSupportedException

README.md

@@ -2,6 +2,15 @@ Extension ClickHouse for Yii 2
 ==============================
 
 This extension provides the [ClickHouse](https://clickhouse.yandex/) integration for the [Yii framework 2.0](http://www.yiiframework.com).
+Main features:
+- SQL commands
+- Query builder
+- Schema builder
+- Migrations
+- Batch Insert
+- Parallel insert from large CSV files
+- Valid handling of UInt64 type in PHP
+- Supports Decimals and Nullable fields
 
 [![Build Status](https://travis-ci.org/bashkarev/clickhouse.svg?branch=master)](https://travis-ci.org/bashkarev/clickhouse)
 
@@ -84,23 +93,27 @@ yii clickhouse-migrate
 # reverts the last applied migration
 yii clickhouse-migrate/down
 ```
+Access to native SMI2 ClickHouse client
+---------------------------------------
+```php
+$client = \Yii::$app->clickhouse->getClient();
+```
 
 Insert csv files
 ----------------
 
 > Files are uploaded in parallel.
 
 ```php
-/**
- * @var \bashkarev\clickhouse\InsertFiles $insert
- */
-$insert = Yii::$app->clickhouse->createCommand()->batchInsertFiles('csv',[
-    '@vendor/bashkarev/clickhouse/tests/data/csv/e1e747f9901e67ca121768b36921fbae.csv',
-    '@vendor/bashkarev/clickhouse/tests/data/csv/ebe191dfc36d73aece91e92007d24e3e.csv',
-]);
-$insert
-    ->setFiles(fopen('/csv/ebe191dfc36d73aece91e92007d24e3e.csv','rb'))
-    ->setChunkSize(8192) // default 4096 bytes
-    ->execute();
+$db = \Yii::$app->clickhouse;
+$client = $db->getClient();
+
+$results = $client->insertBatchFiles('table_name', ['file_with_data.csv']);
+
+$state = $results['file_with_data.csv'];
+$isSuccess = !$state->isError();
+$uploadInfo = $state->responseInfo();
+
+print_r($uploadInfo);
 ```
 

tests/ConnectionTest.php

@@ -8,9 +8,10 @@
 namespace bashkarev\clickhouse\tests;
 
 use bashkarev\clickhouse\Connection;
+use bashkarev\clickhouse\Query;
+use ClickHouseDB\Statement;
 use yii\base\InvalidConfigException;
 use yii\base\NotSupportedException;
-use yii\db\Exception;
 
 /**
  * @author Dmitry Bashkarev <[email protected]>
@@ -144,4 +145,28 @@ public function testTransaction()
         $this->expectException(NotSupportedException::class);
         $this->getConnection()->beginTransaction();
     }
+
+    public function testBatchInsert()
+    {
+        $connection = $this->getConnection();
+        $client = $connection->getClient();
+
+        $files = [
+            \Yii::getAlias('@data/csv/e1e747f9901e67ca121768b36921fbae.csv'),
+            \Yii::getAlias('@data/csv/ebe191dfc36d73aece91e92007d24e3e.csv'),
+            \Yii::getAlias('@data/csv/empty.csv'),
+        ];
+
+        $result = $client->insertBatchFiles('csv', $files);
+        $count = (new Query)
+            ->from('csv')
+            ->count('*', $connection);
+
+        foreach ($result as $filename => $state) {
+            /** @var Statement $state */
+            $this->assertEquals($state->isError(), false);
+        }
+
+        $this->assertContains('2000', $count);
+    }
 }

tests/QueryTest.php

@@ -117,23 +117,28 @@ public function testSubQuery()
     public function testUnionAll()
     {
         $db = $this->getConnection();
-        $secondQuery = (new Query())
+        $query1 = (new Query())
             ->select('name')
             ->from('{{customer}}')
             ->where([
                 'id' => [1]
             ]);
 
-        $rows = (new Query())
+        $query2 = (new Query())
             ->select('name')
             ->from('{{customer}}')
             ->where([
                 'id' => [2]
-            ])
-            ->union($secondQuery, true)
+            ]);
+
+        $rows = (new Query())
+            ->select('name')
+            ->from(['result' => $query1->union($query2, true)])
             ->orderBy(['name' => SORT_ASC])
             ->all($db);
 
+        // Using Subquery, because of ClickHouse does not sort result
+
         $this->assertEquals([
             ['name' => 'user1'],
             ['name' => 'user2']