Test for checking results aws

grajoria · grajoria · commit f3f022a83e96 · 2021-03-26T03:34:42.000-04:00
Added query generator which generates equivalent queries for aws
and ceph. After runing these queries, their results are matched.

Signed-off-by: Girjesh Rajoria &lt;grajoria@redhat.com&gt;
diff --git a/test/queries_generator/aws_cmds.sh b/test/queries_generator/aws_cmds.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+set -x
+set -e
+
+mkdir -p aws_results
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select lower(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from s3object;" "aws_results/output1.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select to_timestamp('1999-11-04T20:27:03.479340Z') from s3object;" "aws_results/output2.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select date_add(day, 8, to_timestamp('1982-02-03T00:20:12.42831Z')) from s3object;" "aws_results/output3.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select substring(upper('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  '), cast((avg(cast(_3 as int)-cast(_2 as int)) - 3) as int), min(cast(_1 as int)-cast(_1 as int)) + 7) from s3object;" "aws_results/output4.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select 10-cast(_2 as int) from s3object;" "aws_results/output5.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select character_length(to_string(to_timestamp('1963-06-22T05:35:39.954350Z'), 'SSSSS HH ')) from s3object;" "aws_results/output6.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select cast(_2 as int)-2 from s3object;" "aws_results/output7.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select char_length(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from s3object;" "aws_results/output8.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select to_string(date_add(second, 5, to_timestamp('1938-09-24T10:57:42.142042Z')), 'hh ') from s3object;" "aws_results/output9.csv"
+aws s3api select-object-content --bucket girjesh-bucket --key test_data.csv  --expression-type 'SQL' --input-serialization '{"CSV": {}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --profile openshift-dev --expression "select to_timestamp('1931-12-23T02:54:25.451925Z') from s3object;" "aws_results/output10.csv"
diff --git a/test/queries_generator/aws_queries.txt b/test/queries_generator/aws_queries.txt
@@ -0,0 +1,10 @@
+select lower(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from s3object;
+select to_timestamp('1999-11-04T20:27:03.479340Z') from s3object;
+select date_add(day, 8, to_timestamp('1982-02-03T00:20:12.42831Z')) from s3object;
+select substring(upper('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  '), cast((avg(cast(_3 as int)-cast(_2 as int)) - 3) as int), min(cast(_1 as int)-cast(_1 as int)) + 7) from s3object;
+select 10-cast(_2 as int) from s3object;
+select character_length(to_string(to_timestamp('1963-06-22T05:35:39.954350Z'), 'SSSSS HH ')) from s3object;
+select cast(_2 as int)-2 from s3object;
+select char_length(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from s3object;
+select to_string(date_add(second, 5, to_timestamp('1938-09-24T10:57:42.142042Z')), 'hh ') from s3object;
+select to_timestamp('1931-12-23T02:54:25.451925Z') from s3object;
diff --git a/test/queries_generator/aws_results/output1.csv b/test/queries_generator/aws_results/output1.csv
@@ -0,0 +1,10 @@
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
+  %%abcdefghijklmnopqrstuvwxyz##  
diff --git a/test/queries_generator/aws_results/output10.csv b/test/queries_generator/aws_results/output10.csv
@@ -0,0 +1,10 @@
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
+1931-12-23T02:54:25.451925Z
diff --git a/test/queries_generator/aws_results/output2.csv b/test/queries_generator/aws_results/output2.csv
@@ -0,0 +1,10 @@
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
+1999-11-04T20:27:03.479340Z
diff --git a/test/queries_generator/aws_results/output3.csv b/test/queries_generator/aws_results/output3.csv
@@ -0,0 +1,10 @@
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
+1982-02-11T00:20:12.42831Z
diff --git a/test/queries_generator/aws_results/output4.csv b/test/queries_generator/aws_results/output4.csv
@@ -0,0 +1 @@
+
diff --git a/test/queries_generator/aws_results/output5.csv b/test/queries_generator/aws_results/output5.csv
@@ -0,0 +1,10 @@
+-42916
+-21159
+-35571
+-38378
+-42792
+-45572
+-8538
+-22623
+-38429
+-6601
diff --git a/test/queries_generator/aws_results/output6.csv b/test/queries_generator/aws_results/output6.csv
@@ -0,0 +1,10 @@
+9
+9
+9
+9
+9
+9
+9
+9
+9
+9
diff --git a/test/queries_generator/aws_results/output7.csv b/test/queries_generator/aws_results/output7.csv
@@ -0,0 +1,10 @@
+42924
+21167
+35579
+38386
+42800
+45580
+8546
+22631
+38437
+6609
diff --git a/test/queries_generator/aws_results/output8.csv b/test/queries_generator/aws_results/output8.csv
@@ -0,0 +1,10 @@
+34
+34
+34
+34
+34
+34
+34
+34
+34
+34
diff --git a/test/queries_generator/aws_results/output9.csv b/test/queries_generator/aws_results/output9.csv
@@ -0,0 +1,10 @@
+10 
+10 
+10 
+10 
+10 
+10 
+10 
+10 
+10 
+10 
diff --git a/test/queries_generator/generate_aws_cmds b/test/queries_generator/generate_aws_cmds
diff --git a/test/queries_generator/generate_aws_cmds.cpp b/test/queries_generator/generate_aws_cmds.cpp
@@ -0,0 +1,26 @@
+#include <iostream>
+#include <fstream>
+
+using namespace std;
+
+int main()
+{
+    fstream query_file, cmd_file;
+    query_file.open("aws_queries.txt", ios::in);
+    cmd_file.open("aws_cmds.sh", ios::out);
+    cmd_file << "#!/bin/sh\nset -x\nset -e\n\n";
+    cmd_file << "mkdir -p aws_results\n";
+    string bucket, csv_file, query, aws_cmd;
+    cout << "Enter bucket name: ";
+    cin >> bucket;
+    cout << "Enter file name: ";
+    cin >> csv_file;
+    for(int i = 1; getline(query_file, query); i++)
+    {
+        aws_cmd = "aws s3api select-object-content --bucket " + bucket + " --key " + csv_file + "  --expression-type \'SQL\' --input-serialization \'{\"CSV\": {}, \"CompressionType\": \"NONE\"}\' --output-serialization \'{\"CSV\": {}}\' --profile openshift-dev --expression \"" + query + "\" \"aws_results/output" + to_string(i) + ".csv\"";
+	cmd_file << aws_cmd << endl;
+    }
+    cmd_file.close();
+    query_file.close();
+    return 0;
+}
diff --git a/test/queries_generator/queries.txt b/test/queries_generator/queries.txt
@@ -0,0 +1,10 @@
+select lower(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from stdin;
+select to_timestamp('1999-11-04T20:27:03.479340Z') from stdin;
+select date_add(day, int(8), to_timestamp('1982-02-03T00:20:12.42831Z')) from stdin;
+select substring(upper('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  '), int(avg(int(_3)-int(_2)) - int(3)), min(int(_1)-int(_1)) + int(7)) from stdin;
+select int(10)-int(_2) from stdin;
+select character_length(to_string(to_timestamp('1963-06-22T05:35:39.954350Z'), 'SSSSS HH ')) from stdin;
+select int(_2)-int(2) from stdin;
+select char_length(lower('  %%AbCdEfGhIjKlMnOpQrStUvWxYz##  ')) from stdin;
+select to_string(date_add(second, int(5), to_timestamp('1938-09-24T10:57:42.142042Z')), 'hh ') from stdin;
+select to_timestamp('1931-12-23T02:54:25.451925Z') from stdin;
diff --git a/test/queries_generator/queries_generator.cpp b/test/queries_generator/queries_generator.cpp
diff --git a/test/queries_generator/run.sh b/test/queries_generator/run.sh
diff --git a/test/s3select_test.cpp b/test/s3select_test.cpp

-Original file line number
+Diff line change
@@ @@ -0,0 +1,10 @@ @@
 +-42916
 +-21159
 +-35571
 +-38378
 +-42792
 +-45572
 +-8538
 +-22623
 +-38429
 +-6601