Skip to content

Commit f40f778

Browse files
PaultagorasPaultagoras
authored
Merge pull request #1752 from ClickHouse/feat_text_formats_examples
[client-v2] Text formats examples
2 parents 48d5937 + 657249a commit f40f778

File tree

7 files changed

+347
-53
lines changed

7 files changed

+347
-53
lines changed

‎client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java‎

Lines changed: 26 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import com.clickhouse.data.ClickHouseDataType;
3434
import com.clickhouse.data.ClickHouseFormat;
3535
import com.fasterxml.jackson.databind.JsonNode;
36+
import com.fasterxml.jackson.databind.MappingIterator;
3637
import com.fasterxml.jackson.databind.ObjectMapper;
3738
import org.testng.Assert;
3839
import org.testng.annotations.AfterMethod;
@@ -174,34 +175,30 @@ public void testQueryAllNoResult() throws Exception {
174175
Assert.assertTrue(records.isEmpty());
175176
}
176177

177-
@Test(groups = {"integration"}, enabled = false)
178-
public void testQueryJSONWith64BitIntegers() throws ExecutionException, InterruptedException {
179-
// won't work because format settings are set thru separate statement.
180-
prepareSimpleDataSet();
181-
List<QuerySettings> settingsList = Arrays.asList(
182-
new QuerySettings()
183-
.setFormat(ClickHouseFormat.JSONEachRow)
184-
.setOption("format_json_quote_64bit_integers", "true"),
185-
186-
new QuerySettings().setFormat(ClickHouseFormat.JSONEachRow));
187-
List<Boolean> expected = Arrays.asList(true, false);
188-
189-
Iterator<Boolean> expectedIterator = expected.iterator();
190-
for (QuerySettings settings : settingsList) {
191-
Future<QueryResponse> response = client.query("SELECT * FROM " + DATASET_TABLE, settings);
192-
QueryResponse queryResponse = response.get();
193-
ArrayList<JsonNode> records = new ArrayList<>();
194-
final ObjectMapper objectMapper = new ObjectMapper();
195-
try (BufferedReader br = new BufferedReader(new InputStreamReader(queryResponse.getInputStream()))) {
196-
String line = null;
197-
while ((line = br.readLine()) != null) {
198-
System.out.println(line);
199-
records.add(objectMapper.readTree(line));
200-
Assert.assertEquals(records.get(0).get("param4").isTextual(), expectedIterator.next());
201-
}
202-
} catch (IOException e) {
203-
Assert.fail("failed to read response", e);
178+
@Test(groups = {"integration"})
179+
public void testQueryJSON() throws ExecutionException, InterruptedException {
180+
Map<String, Object> datasetRecord = prepareSimpleDataSet();
181+
QuerySettings settings = new QuerySettings().setFormat(ClickHouseFormat.JSONEachRow);
182+
Future<QueryResponse> response = client.query("SELECT * FROM " + DATASET_TABLE, settings);
183+
final ObjectMapper objectMapper = new ObjectMapper();
184+
try (QueryResponse queryResponse = response.get(); MappingIterator<JsonNode> jsonIter = objectMapper.readerFor(JsonNode.class)
185+
.readValues(queryResponse.getInputStream())) {
186+
187+
188+
while (jsonIter.hasNext()) {
189+
JsonNode node = jsonIter.next();
190+
System.out.println(node);
191+
long col1 = node.get("col1").asLong();
192+
Assert.assertEquals(col1, datasetRecord.get("col1"));
193+
int col2 = node.get("col2").asInt();
194+
Assert.assertEquals(col2, datasetRecord.get("col2"));
195+
String col3 = node.get("col3").asText();
196+
Assert.assertEquals(col3, datasetRecord.get("col3"));
197+
long col4 = node.get("col4").asLong();
198+
Assert.assertEquals(col4, datasetRecord.get("col4"));
204199
}
200+
} catch (Exception e) {
201+
Assert.fail("failed to read response", e);
205202
}
206203
}
207204

@@ -994,8 +991,8 @@ public void testQueryMetrics() throws Exception {
994991

995992
private final static String DATASET_TABLE = "query_test_table";
996993

997-
private void prepareSimpleDataSet() {
998-
prepareDataSet(DATASET_TABLE, DATASET_COLUMNS, DATASET_VALUE_GENERATORS, 1);
994+
private Map<String, Object> prepareSimpleDataSet() {
995+
return prepareDataSet(DATASET_TABLE, DATASET_COLUMNS, DATASET_VALUE_GENERATORS, 1).get(0);
999996
}
1000997

1001998
private List<Map<String, Object>> prepareDataSet(String table, List<String> columns, List<Function<String, Object>> valueGenerators,

‎examples/client-v2/pom.xml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,32 @@
105105
<version>1.8.0</version>
106106
</dependency>
107107

108+
<!-- Recommended for JSON parsing -->
109+
<dependency>
110+
<groupId>com.fasterxml.jackson.core</groupId>
111+
<artifactId>jackson-core</artifactId>
112+
<version>2.17.2</version>
113+
</dependency>
114+
<dependency>
115+
<groupId>com.fasterxml.jackson.core</groupId>
116+
<artifactId>jackson-databind</artifactId>
117+
<scope>test</scope>
118+
<version>2.17.2</version>
119+
</dependency>
120+
121+
<!-- Popular Alternative to Jackson -->
122+
<dependency>
123+
<groupId>com.google.code.gson</groupId>
124+
<artifactId>gson</artifactId>
125+
<version>2.11.0</version>
126+
</dependency>
127+
128+
<!-- For parsing DB text formats -->
129+
<dependency>
130+
<groupId>org.apache.commons</groupId>
131+
<artifactId>commons-csv</artifactId>
132+
<version>1.11.0</version>
133+
</dependency>
108134

109135
<!-- Miscellaneous application dependencies -->
110136

@@ -130,6 +156,12 @@
130156
<version>2.0.13</version>
131157
<scope>runtime</scope>
132158
</dependency>
159+
<dependency>
160+
<groupId>com.fasterxml.jackson.core</groupId>
161+
<artifactId>jackson-databind</artifactId>
162+
<version>2.17.2</version>
163+
<scope>compile</scope>
164+
</dependency>
133165

134166
</dependencies>
135167

‎examples/client-v2/src/main/java/com/clickhouse/examples/client_v2/Main.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,15 @@ public static void main(String[] args) {
4141
reader.readDataAll();
4242
reader.readData();
4343

44+
// Read as Text format
45+
TextFormatsReader textFormatsReader = new TextFormatsReader(endpoint, user, password, database);
46+
textFormatsReader.readAsJsonEachRow();
47+
textFormatsReader.readAsJsonEachRowButGSon();
48+
textFormatsReader.readJSONEachRowIntoArrayOfObject();
49+
textFormatsReader.readJSONEachRowIntoArrayOfObjectGson();
50+
textFormatsReader.readAsCSV();
51+
textFormatsReader.readAsTSV();
52+
4453
// Insert data using POJO
4554
POJO2DbWriter pojoWriter = new POJO2DbWriter(endpoint, user, password, database);
4655
pojoWriter.resetTable();

‎examples/client-v2/src/main/java/com/clickhouse/examples/client_v2/POJO2DbWriter.java‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -54,21 +54,21 @@ public void resetTable() {
5454
client.query("drop table if exists " + TABLE_NAME).get(3, TimeUnit.SECONDS);
5555

5656
// Reading the SQL file and executing it
57-
BufferedReader reader = new BufferedReader(new InputStreamReader(initSql));
58-
String sql = reader.lines().collect(Collectors.joining("\n"));
59-
log.debug("Executing Create Table: {}", sql);
60-
client.query(sql).get(10, TimeUnit.SECONDS);
61-
log.info("Table initialized. Registering class.");
62-
client.register(ArticleViewEvent.class, client.getTableSchema(TABLE_NAME));
57+
try (BufferedReader reader = new BufferedReader(new InputStreamReader(initSql))) {
58+
String sql = reader.lines().collect(Collectors.joining("\n"));
59+
log.debug("Executing Create Table: {}", sql);
60+
client.query(sql).get(10, TimeUnit.SECONDS);
61+
log.info("Table initialized. Registering class.");
62+
client.register(ArticleViewEvent.class, client.getTableSchema(TABLE_NAME));
63+
}
6364
} catch (Exception e) {
6465
log.error("Failed to initialize table", e);
6566
}
6667
}
6768

6869
public void printLastEvents() {
69-
try {
70-
QueryResponse response = client.query("select * from " + TABLE_NAME + " order by viewTime desc limit 10 format CSV")
71-
.get(10, TimeUnit.SECONDS);
70+
try (QueryResponse response = client.query("select * from " + TABLE_NAME + " order by viewTime desc limit 10 format CSV")
71+
.get(10, TimeUnit.SECONDS)) {
7272

7373
log.info("Last 10 events:");
7474
try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getInputStream()))) {
@@ -83,7 +83,6 @@ public void printLastEvents() {
8383
}
8484

8585
public synchronized void submit(ArticleViewEvent event) {
86-
8786
events.add(event);
8887

8988
if (events.size() >= EVENTS_BATCH_SIZE) {

‎examples/client-v2/src/main/java/com/clickhouse/examples/client_v2/SimpleReader.java‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -52,14 +52,11 @@ public boolean isServerAlive() {
5252
}
5353

5454
public void readDataUsingBinaryFormat() {
55-
try {
56-
// Read data from the table
57-
log.info("Reading data from table: {}", TABLE_NAME);
58-
59-
final String sql = "select * from " + TABLE_NAME + " where title <> '' limit 10";
55+
log.info("Reading data from table: {}", TABLE_NAME);
56+
final String sql = "select * from " + TABLE_NAME + " where title <> '' limit 10";
6057

61-
// Default format is RowBinaryWithNamesAndTypesFormatReader so reader have all information about columns
62-
QueryResponse response = client.query(sql).get(3, TimeUnit.SECONDS);
58+
// Default format is RowBinaryWithNamesAndTypesFormatReader so reader have all information about columns
59+
try (QueryResponse response = client.query(sql).get(3, TimeUnit.SECONDS);) {
6360

6461
// Create a reader to access the data in a convenient way
6562
ClickHouseBinaryFormatReader reader = new RowBinaryWithNamesAndTypesFormatReader(response.getInputStream());
@@ -79,6 +76,7 @@ public void readDataUsingBinaryFormat() {
7976
} catch (Exception e) {
8077
log.error("Failed to read data", e);
8178
}
79+
// Response object must be closed to release resources
8280
}
8381

8482
public void readDataAll() {
@@ -100,10 +98,9 @@ public void readDataAll() {
10098
}
10199

102100
public void readData() {
103-
try {
104-
log.info("Reading data from table: {} using Records iterator", TABLE_NAME);
105-
final String sql = "select * from " + TABLE_NAME + " where title <> '' limit 10";
106-
Records records = client.queryRecords(sql).get(3, TimeUnit.SECONDS);
101+
log.info("Reading data from table: {} using Records iterator", TABLE_NAME);
102+
final String sql = "select * from " + TABLE_NAME + " where title <> '' limit 10";
103+
try (Records records = client.queryRecords(sql).get(3, TimeUnit.SECONDS);) {
107104

108105
// Get some metrics
109106
log.info("Data read successfully: {} ms", TimeUnit.NANOSECONDS.toMillis(records.getServerTime()));

‎examples/client-v2/src/main/java/com/clickhouse/examples/client_v2/Stream2DbWriter.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,9 @@ public void resetTable() {
7373
*/
7474
public void insertData_JSONEachRowFormat(InputStream inputStream) {
7575
InsertSettings insertSettings = new InsertSettings();
76-
try {
77-
InsertResponse response = client.insert(TABLE_NAME, inputStream, ClickHouseFormat.JSONEachRow,
78-
insertSettings).get(3, TimeUnit.SECONDS);
76+
try (InsertResponse response = client.insert(TABLE_NAME, inputStream, ClickHouseFormat.JSONEachRow,
77+
insertSettings).get(3, TimeUnit.SECONDS)) {
78+
7979
log.info("Insert finished: {} rows written", response.getMetrics().getMetric(ServerMetrics.NUM_ROWS_WRITTEN).getLong());
8080
} catch (Exception e) {
8181
log.error("Failed to write JSONEachRow data", e);

0 commit comments

Comments
 (0)