From 87753a3bdf9c8b07c70e6a5d1849e7996c154cfe Mon Sep 17 00:00:00 2001 From: fimac Date: Mon, 21 Oct 2024 11:25:59 +1100 Subject: [PATCH 01/10] wip --- languages/go/goeql/goeql.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/languages/go/goeql/goeql.go b/languages/go/goeql/goeql.go index 2a2de479..4646beeb 100644 --- a/languages/go/goeql/goeql.go +++ b/languages/go/goeql/goeql.go @@ -32,6 +32,41 @@ type EncryptedColumn struct { } // EncryptedText is a string value to be encrypted +// def for_match(value) +// for_query(value, "match") +// end + +// def for_ore(value) +// for_query(value, "ore") +// end + +// def for_unique(value) +// for_query(value, "unique") +// end + +// def for_ste_vec(value) +// for_query(value, "ste_vec") +// end + +// def for_query(value, for_query) +// eql_payload(value, for_query).to_json() +// end + +// def eql_payload(value, for_query) +// { +// k: "pt", +// p: serialize_plaintext_value(value), +// i: { +// t: table, +// c: column +// }, +// v: 1, +// q: for_query, +// } +// end +// +// Creating custom types for encrypted fields to enable creating methods for +// serialization/deserialization of these types. type EncryptedText string // EncryptedJsonb is a jsonb value to be encrypted From 84d50fc185ec90beab30192f0e377ec0d4203b8d Mon Sep 17 00:00:00 2001 From: fimac Date: Mon, 21 Oct 2024 11:55:23 +1100 Subject: [PATCH 02/10] update payload and queries --- languages/go/goeql/goeql.go | 86 ++++++++++++---------------- languages/go/goeql/goeql_test.go | 4 +- languages/go/xorm/e2e_test.go | 16 +++--- languages/go/xorm/example_queries.go | 16 +++--- 4 files changed, 56 insertions(+), 66 deletions(-) diff --git a/languages/go/goeql/goeql.go b/languages/go/goeql/goeql.go index 4646beeb..62b77042 100644 --- a/languages/go/goeql/goeql.go +++ b/languages/go/goeql/goeql.go @@ -29,44 +29,10 @@ type EncryptedColumn struct { P string `json:"p"` I TableColumn `json:"i"` V int `json:"v"` + Q any `json:"q"` } // EncryptedText is a string value to be encrypted -// def for_match(value) -// for_query(value, "match") -// end - -// def for_ore(value) -// for_query(value, "ore") -// end - -// def for_unique(value) -// for_query(value, "unique") -// end - -// def for_ste_vec(value) -// for_query(value, "ste_vec") -// end - -// def for_query(value, for_query) -// eql_payload(value, for_query).to_json() -// end - -// def eql_payload(value, for_query) -// { -// k: "pt", -// p: serialize_plaintext_value(value), -// i: { -// t: table, -// c: column -// }, -// v: 1, -// q: for_query, -// } -// end -// -// Creating custom types for encrypted fields to enable creating methods for -// serialization/deserialization of these types. type EncryptedText string // EncryptedJsonb is a jsonb value to be encrypted @@ -80,7 +46,7 @@ type EncryptedBool bool // Serialize turns a EncryptedText value into a jsonb payload for CipherStash Proxy func (et EncryptedText) Serialize(table string, column string) ([]byte, error) { - val, err := ToEncryptedColumn(string(et), table, column) + val, err := ToEncryptedColumn(string(et), table, column, nil) if err != nil { return nil, fmt.Errorf("error serializing: %v", err) } @@ -103,7 +69,7 @@ func (et *EncryptedText) Deserialize(data []byte) (EncryptedText, error) { // Serialize turns a EncryptedJsonb value into a jsonb payload for CipherStash Proxy func (ej EncryptedJsonb) Serialize(table string, column string) ([]byte, error) { - val, err := ToEncryptedColumn(map[string]any(ej), table, column) + val, err := ToEncryptedColumn(map[string]any(ej), table, column, nil) if err != nil { return nil, fmt.Errorf("error serializing: %v", err) } @@ -131,7 +97,7 @@ func (ej *EncryptedJsonb) Deserialize(data []byte) (EncryptedJsonb, error) { // Serialize turns a EncryptedInt value into a jsonb payload for CipherStash Proxy func (et EncryptedInt) Serialize(table string, column string) ([]byte, error) { - val, err := ToEncryptedColumn(int(et), table, column) + val, err := ToEncryptedColumn(int(et), table, column, nil) if err != nil { return nil, fmt.Errorf("error serializing: %v", err) } @@ -158,7 +124,7 @@ func (et *EncryptedInt) Deserialize(data []byte) (EncryptedInt, error) { // Serialize turns a EncryptedBool value into a jsonb payload for CipherStash Proxy func (eb EncryptedBool) Serialize(table string, column string) ([]byte, error) { - val, err := ToEncryptedColumn(bool(eb), table, column) + val, err := ToEncryptedColumn(bool(eb), table, column, nil) if err != nil { return nil, fmt.Errorf("error serializing: %v", err) } @@ -184,9 +150,22 @@ func (eb *EncryptedBool) Deserialize(data []byte) (EncryptedBool, error) { return false, fmt.Errorf("invalid format: missing 'p' field") } +func SerializeMatchQuery(value any, table string, column string) ([]byte, error) { + return SerializeQuery(value, table, column, "match") +} +func SerializeOreQuery(value any, table string, column string) ([]byte, error) { + return SerializeQuery(value, table, column, "ore") +} +func SerializeUniqueQuery(value any, table string, column string) ([]byte, error) { + return SerializeQuery(value, table, column, "unique") +} +func SerializeJsonbQuery(value any, table string, column string) ([]byte, error) { + return SerializeQuery(value, table, column, "ste_vec") +} + // SerializeQuery produces a jsonb payload used by EQL query functions to perform search operations like equality checks, range queries, and unique constraints. -func SerializeQuery(value any, table string, column string) ([]byte, error) { - query, err := ToEncryptedColumn(value, table, column) +func SerializeQuery(value any, table string, column string, queryType any) ([]byte, error) { + query, err := ToEncryptedColumn(value, table, column, queryType) if err != nil { return nil, fmt.Errorf("error converting to EncryptedColumn: %v", err) } @@ -200,15 +179,26 @@ func SerializeQuery(value any, table string, column string) ([]byte, error) { } // ToEncryptedColumn converts a plaintext value to a string, and returns the EncryptedColumn struct for inserting into a database. -func ToEncryptedColumn(value any, table string, column string) (EncryptedColumn, error) { - str, err := convertToString(value) - if err != nil { - return EncryptedColumn{}, fmt.Errorf("error: %v", err) - } +func ToEncryptedColumn(value any, table string, column string, queryType any) (EncryptedColumn, error) { + if queryType == nil { + str, err := convertToString(value) + if err != nil { + return EncryptedColumn{}, fmt.Errorf("error: %v", err) + } - data := EncryptedColumn{K: "pt", P: str, I: TableColumn{T: table, C: column}, V: 1} + data := EncryptedColumn{K: "pt", P: str, I: TableColumn{T: table, C: column}, V: 1, Q: nil} - return data, nil + return data, nil + } else { + str, err := convertToString(value) + if err != nil { + return EncryptedColumn{}, fmt.Errorf("error: %v", err) + } + + data := EncryptedColumn{K: "pt", P: str, I: TableColumn{T: table, C: column}, V: 1, Q: queryType} + + return data, nil + } } func convertToString(value any) (string, error) { diff --git a/languages/go/goeql/goeql_test.go b/languages/go/goeql/goeql_test.go index 89086ee4..65d5ebab 100644 --- a/languages/go/goeql/goeql_test.go +++ b/languages/go/goeql/goeql_test.go @@ -225,7 +225,7 @@ func TestSerializeQuery(t *testing.T) { } for _, tt := range tests { - serializedData, err := SerializeQuery(tt.value, tt.table, tt.column) + serializedData, err := SerializeQuery(tt.value, tt.table, tt.column, nil) if err != nil { t.Fatalf("SerializeQuery returned error: %v", err) } @@ -257,7 +257,7 @@ func TestToEncryptedColumn(t *testing.T) { } for _, tt := range tests { - ec, err := ToEncryptedColumn(tt.value, tt.table, tt.column) + ec, err := ToEncryptedColumn(tt.value, tt.table, tt.column, nil) if err != nil { t.Fatalf("ToEncryptedColumn returned error: %v", err) } diff --git a/languages/go/xorm/e2e_test.go b/languages/go/xorm/e2e_test.go index fc570057..ec85c23d 100644 --- a/languages/go/xorm/e2e_test.go +++ b/languages/go/xorm/e2e_test.go @@ -104,7 +104,7 @@ func TestMatchQueryLongString(t *testing.T) { assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows") - query, err := goeql.SerializeQuery("this", "examples", "encrypted_text_field") + query, err := goeql.SerializeMatchQuery("this", "examples", "encrypted_text_field") if err != nil { log.Fatalf("Error marshaling encrypted_text_field query: %v", err) } @@ -157,7 +157,7 @@ func TestMatchQueryEmail(t *testing.T) { assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows") - query, err := goeql.SerializeQuery("test", "examples", "encrypted_text_field") + query, err := goeql.SerializeMatchQuery("test", "examples", "encrypted_text_field") if err != nil { log.Fatalf("Error marshaling encrypted_text_field query: %v", err) } @@ -218,7 +218,7 @@ func TestJsonbQuerySimple(t *testing.T) { }, } - query, errTwo := goeql.SerializeQuery(jsonbQuery, "examples", "encrypted_jsonb_field") + query, errTwo := goeql.SerializeJsonbQuery(jsonbQuery, "examples", "encrypted_jsonb_field") if errTwo != nil { log.Fatalf("Error marshaling encrypted_jsonb_field: %v", errTwo) } @@ -284,7 +284,7 @@ func TestJsonbQueryNested(t *testing.T) { }, } - query, errTwo := goeql.SerializeQuery(jsonbQuery, "examples", "encrypted_jsonb_field") + query, errTwo := goeql.SerializeJsonbQuery(jsonbQuery, "examples", "encrypted_jsonb_field") if errTwo != nil { log.Fatalf("Error marshaling encrypted_jsonb_field: %v", errTwo) } @@ -331,7 +331,7 @@ func TestOreStringRangeQuery(t *testing.T) { assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows") // Query - query, errQuery := goeql.SerializeQuery("tree", "examples", "encrypted_text_field") + query, errQuery := goeql.SerializeOreQuery("tree", "examples", "encrypted_text_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -378,7 +378,7 @@ func TestOreIntRangeQuery(t *testing.T) { assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows") // Query - query, errQuery := goeql.SerializeQuery(32, "examples", "encrypted_int_field") + query, errQuery := goeql.SerializeOreQuery(32, "examples", "encrypted_int_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -434,7 +434,7 @@ func TestOreBoolRangeQuery(t *testing.T) { assert.Equal(t, int64(3), inserted, "Expected to insert 3 rows") // Query - query, errQuery := goeql.SerializeQuery(false, "examples", "encrypted_bool_field") + query, errQuery := goeql.SerializeOreQuery(false, "examples", "encrypted_bool_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -490,7 +490,7 @@ func TestUniqueStringQuery(t *testing.T) { assert.Equal(t, int64(3), inserted, "Expected to insert 3 rows") // Query - query, errQuery := goeql.SerializeQuery("testing two", "examples", "encrypted_text_field") + query, errQuery := goeql.SerializeUniqueQuery("testing two", "examples", "encrypted_text_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } diff --git a/languages/go/xorm/example_queries.go b/languages/go/xorm/example_queries.go index 41497b4b..a4989b51 100644 --- a/languages/go/xorm/example_queries.go +++ b/languages/go/xorm/example_queries.go @@ -60,7 +60,7 @@ func MatchQueryLongString(engine *xorm.Engine) { fmt.Printf("Example one inserted: %+v\n", newExample) fmt.Println("") - query, err := goeql.SerializeQuery("this", "examples", "encrypted_text_field") + query, err := goeql.SerializeMatchQuery("this", "examples", "encrypted_text_field") if err != nil { log.Fatalf("Error marshaling encrypted_text_field: %v", err) } @@ -93,7 +93,7 @@ func MatchQueryEmail(engine *xorm.Engine) { fmt.Printf("Example two inserted!: %+v\n", newExampleTwo) fmt.Println("") - query, errTwo := goeql.SerializeQuery("some", "examples", "encrypted_text_field") + query, errTwo := goeql.SerializeMatchQuery("some", "examples", "encrypted_text_field") if errTwo != nil { log.Fatalf("Error marshaling encrypted_text_field: %v", errTwo) } @@ -141,7 +141,7 @@ func JsonbQuerySimple(engine *xorm.Engine) { }, } - query, errTwo := goeql.SerializeQuery(jsonbQuery, "examples", "encrypted_jsonb_field") + query, errTwo := goeql.SerializeJsonbQuery(jsonbQuery, "examples", "encrypted_jsonb_field") if errTwo != nil { log.Fatalf("Error marshaling encrypted_jsonb_field: %v", errTwo) } @@ -201,7 +201,7 @@ func JsonbQueryDeepNested(engine *xorm.Engine) { }, } - jsonbQuery, errQuery := goeql.SerializeQuery(query, "examples", "encrypted_jsonb_field") + jsonbQuery, errQuery := goeql.SerializeJsonbQuery(query, "examples", "encrypted_jsonb_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -238,7 +238,7 @@ func OreStringRangeQuery(engine *xorm.Engine) { fmt.Println("Examples inserted!") // Query - query, errQuery := goeql.SerializeQuery("tree", "examples", "encrypted_text_field") + query, errQuery := goeql.SerializeOreQuery("tree", "examples", "encrypted_text_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -276,7 +276,7 @@ func OreIntRangeQuery(engine *xorm.Engine) { fmt.Println("Examples inserted!", example1) fmt.Println("Examples inserted!", example2) - serializedOreIntQuery, errQuery := goeql.SerializeQuery(32, "examples", "encrypted_int_field") + serializedOreIntQuery, errQuery := goeql.SerializeOreQuery(32, "examples", "encrypted_int_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -322,7 +322,7 @@ func OreBoolQuery(engine *xorm.Engine) { fmt.Println("Example3 inserted!", example3) // Query - query, errQuery := goeql.SerializeQuery(false, "examples", "encrypted_bool_field") + query, errQuery := goeql.SerializeOreQuery(false, "examples", "encrypted_bool_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -364,7 +364,7 @@ func UniqueStringQuery(engine *xorm.Engine) { fmt.Println("Example3 inserted!", example3) var allExamples []Example - query, errQuery := goeql.SerializeQuery("test two", "examples", "encrypted_text_field") + query, errQuery := goeql.SerializeUniqueQuery("test two", "examples", "encrypted_text_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } From c4ae6999d292f0ed434f5d973aa19f41d3015e0e Mon Sep 17 00:00:00 2001 From: fimac Date: Mon, 21 Oct 2024 14:09:14 +1100 Subject: [PATCH 03/10] update naming of query functions --- languages/go/goeql/goeql.go | 8 ++--- languages/go/xorm/e2e_test.go | 52 ++++++++++++++++------------ languages/go/xorm/example_queries.go | 16 ++++----- 3 files changed, 41 insertions(+), 35 deletions(-) diff --git a/languages/go/goeql/goeql.go b/languages/go/goeql/goeql.go index 62b77042..bb8342ca 100644 --- a/languages/go/goeql/goeql.go +++ b/languages/go/goeql/goeql.go @@ -150,16 +150,16 @@ func (eb *EncryptedBool) Deserialize(data []byte) (EncryptedBool, error) { return false, fmt.Errorf("invalid format: missing 'p' field") } -func SerializeMatchQuery(value any, table string, column string) ([]byte, error) { +func MatchQuery(value any, table string, column string) ([]byte, error) { return SerializeQuery(value, table, column, "match") } -func SerializeOreQuery(value any, table string, column string) ([]byte, error) { +func OreQuery(value any, table string, column string) ([]byte, error) { return SerializeQuery(value, table, column, "ore") } -func SerializeUniqueQuery(value any, table string, column string) ([]byte, error) { +func UniqueQuery(value any, table string, column string) ([]byte, error) { return SerializeQuery(value, table, column, "unique") } -func SerializeJsonbQuery(value any, table string, column string) ([]byte, error) { +func JsonbQuery(value any, table string, column string) ([]byte, error) { return SerializeQuery(value, table, column, "ste_vec") } diff --git a/languages/go/xorm/e2e_test.go b/languages/go/xorm/e2e_test.go index ec85c23d..e48da964 100644 --- a/languages/go/xorm/e2e_test.go +++ b/languages/go/xorm/e2e_test.go @@ -51,10 +51,10 @@ func TestWhereQueryOnUnencryptedColumn(t *testing.T) { t.Fatalf("Could not insert new example: %v", err) } - var example Example + var returnedExample Example text := "sydney" - has, err := engine.Where("non_encrypted_field = ?", text).Get(&example) + has, err := engine.Where("non_encrypted_field = ?", text).Get(&returnedExample) if err != nil { t.Fatalf("Could not retrieve example: %v", err) } @@ -63,10 +63,10 @@ func TestWhereQueryOnUnencryptedColumn(t *testing.T) { t.Errorf("Expected has to equal true, got: %v", has) } - assert.Equal(t, newExample.NonEncryptedField, example.NonEncryptedField, "NonEncryptedField does not match") - assert.Equal(t, newExample.EncryptedIntField, example.EncryptedIntField, "EncryptedIntField does not match") - assert.Equal(t, newExample.EncryptedTextField, example.EncryptedTextField, "EncryptedTextField does not match") - assert.Equal(t, newExample.EncryptedJsonbField, example.EncryptedJsonbField, "EncryptedJsonbField does not match") + assert.Equal(t, newExample.NonEncryptedField, returnedExample.NonEncryptedField, "NonEncryptedField does not match") + assert.Equal(t, newExample.EncryptedIntField, returnedExample.EncryptedIntField, "EncryptedIntField does not match") + assert.Equal(t, newExample.EncryptedTextField, returnedExample.EncryptedTextField, "EncryptedTextField does not match") + assert.Equal(t, newExample.EncryptedJsonbField, returnedExample.EncryptedJsonbField, "EncryptedJsonbField does not match") } func TestMatchQueryLongString(t *testing.T) { @@ -82,6 +82,12 @@ func TestMatchQueryLongString(t *testing.T) { } examples := []Example{ + { + NonEncryptedField: "brisbane", + EncryptedIntField: 23, + EncryptedTextField: "another string that shouldn't be returned", + EncryptedJsonbField: jsonData, + }, { NonEncryptedField: "sydney", EncryptedIntField: 23, @@ -102,9 +108,9 @@ func TestMatchQueryLongString(t *testing.T) { t.Errorf("Error inserting examples: %v", err) } - assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows") + assert.Equal(t, int64(3), inserted, "Expected to insert 2 rows") - query, err := goeql.SerializeMatchQuery("this", "examples", "encrypted_text_field") + query, err := goeql.MatchQuery("this", "examples", "encrypted_text_field") if err != nil { log.Fatalf("Error marshaling encrypted_text_field query: %v", err) } @@ -119,7 +125,7 @@ func TestMatchQueryLongString(t *testing.T) { t.Errorf("Expected has to equal true, got: %v", has) } - assert.Equal(t, returnedExample.EncryptedTextField, EncryptedTextField("this is a long string"), "EncryptedTextField should match") + assert.Equal(t, EncryptedTextField("this is a long string"), returnedExample.EncryptedTextField, "EncryptedTextField should match") } func TestMatchQueryEmail(t *testing.T) { @@ -157,7 +163,7 @@ func TestMatchQueryEmail(t *testing.T) { assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows") - query, err := goeql.SerializeMatchQuery("test", "examples", "encrypted_text_field") + query, err := goeql.MatchQuery("test", "examples", "encrypted_text_field") if err != nil { log.Fatalf("Error marshaling encrypted_text_field query: %v", err) } @@ -172,7 +178,7 @@ func TestMatchQueryEmail(t *testing.T) { t.Errorf("Expected has to equal true, got: %v", has) } - assert.Equal(t, returnedExample.EncryptedTextField, EncryptedTextField("testemail@test.com"), "EncryptedTextField should match") + assert.Equal(t, EncryptedTextField("testemail@test.com"), returnedExample.EncryptedTextField, "EncryptedTextField should match") } func TestJsonbQuerySimple(t *testing.T) { @@ -218,7 +224,7 @@ func TestJsonbQuerySimple(t *testing.T) { }, } - query, errTwo := goeql.SerializeJsonbQuery(jsonbQuery, "examples", "encrypted_jsonb_field") + query, errTwo := goeql.JsonbQuery(jsonbQuery, "examples", "encrypted_jsonb_field") if errTwo != nil { log.Fatalf("Error marshaling encrypted_jsonb_field: %v", errTwo) } @@ -233,7 +239,7 @@ func TestJsonbQuerySimple(t *testing.T) { t.Errorf("Expected has to equal true, got: %v", has) } - assert.Equal(t, returnedExample.EncryptedJsonbField, EncryptedJsonbField(expectedJson), "EncryptedJsonb field should match") + assert.Equal(t, EncryptedJsonbField(expectedJson), returnedExample.EncryptedJsonbField, "EncryptedJsonb field should match") } func TestJsonbQueryNested(t *testing.T) { @@ -284,7 +290,7 @@ func TestJsonbQueryNested(t *testing.T) { }, } - query, errTwo := goeql.SerializeJsonbQuery(jsonbQuery, "examples", "encrypted_jsonb_field") + query, errTwo := goeql.JsonbQuery(jsonbQuery, "examples", "encrypted_jsonb_field") if errTwo != nil { log.Fatalf("Error marshaling encrypted_jsonb_field: %v", errTwo) } @@ -299,7 +305,7 @@ func TestJsonbQueryNested(t *testing.T) { t.Errorf("Expected has to equal true, got: %v", has) } - assert.Equal(t, returnedExample.EncryptedJsonbField, EncryptedJsonbField(expectedJson), "EncryptedJsonb field should match") + assert.Equal(t, EncryptedJsonbField(expectedJson), returnedExample.EncryptedJsonbField, "EncryptedJsonb field should match") } func TestOreStringRangeQuery(t *testing.T) { @@ -331,7 +337,7 @@ func TestOreStringRangeQuery(t *testing.T) { assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows") // Query - query, errQuery := goeql.SerializeOreQuery("tree", "examples", "encrypted_text_field") + query, errQuery := goeql.OreQuery("tree", "examples", "encrypted_text_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -346,7 +352,7 @@ func TestOreStringRangeQuery(t *testing.T) { t.Errorf("Expected has to equal true, got: %v", has) } - assert.Equal(t, returnedExample.EncryptedTextField, expected, "EncryptedText field should match") + assert.Equal(t, expected, returnedExample.EncryptedTextField, "EncryptedText field should match") } func TestOreIntRangeQuery(t *testing.T) { @@ -378,7 +384,7 @@ func TestOreIntRangeQuery(t *testing.T) { assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows") // Query - query, errQuery := goeql.SerializeOreQuery(32, "examples", "encrypted_int_field") + query, errQuery := goeql.OreQuery(32, "examples", "encrypted_int_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -393,7 +399,7 @@ func TestOreIntRangeQuery(t *testing.T) { t.Errorf("Expected has to equal true, got: %v", has) } - assert.Equal(t, returnedExample.EncryptedIntField, expected, "EncryptedInt field should match") + assert.Equal(t, expected, returnedExample.EncryptedIntField, "EncryptedInt field should match") } func TestOreBoolRangeQuery(t *testing.T) { @@ -434,7 +440,7 @@ func TestOreBoolRangeQuery(t *testing.T) { assert.Equal(t, int64(3), inserted, "Expected to insert 3 rows") // Query - query, errQuery := goeql.SerializeOreQuery(false, "examples", "encrypted_bool_field") + query, errQuery := goeql.OreQuery(false, "examples", "encrypted_bool_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -449,7 +455,7 @@ func TestOreBoolRangeQuery(t *testing.T) { t.Errorf("Expected has to equal true, got: %v", has) } - assert.Equal(t, returnedExample.EncryptedBoolField, expected, "EncryptedBool field should match") + assert.Equal(t, expected, returnedExample.EncryptedBoolField, "EncryptedBool field should match") } func TestUniqueStringQuery(t *testing.T) { @@ -490,7 +496,7 @@ func TestUniqueStringQuery(t *testing.T) { assert.Equal(t, int64(3), inserted, "Expected to insert 3 rows") // Query - query, errQuery := goeql.SerializeUniqueQuery("testing two", "examples", "encrypted_text_field") + query, errQuery := goeql.UniqueQuery("testing two", "examples", "encrypted_text_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -505,5 +511,5 @@ func TestUniqueStringQuery(t *testing.T) { t.Errorf("Expected has to equal true, got: %v", has) } - assert.Equal(t, returnedExample.EncryptedTextField, expected, "EncryptedText field should match") + assert.Equal(t, expected, returnedExample.EncryptedTextField, "EncryptedText field should match") } diff --git a/languages/go/xorm/example_queries.go b/languages/go/xorm/example_queries.go index a4989b51..af74eee0 100644 --- a/languages/go/xorm/example_queries.go +++ b/languages/go/xorm/example_queries.go @@ -60,7 +60,7 @@ func MatchQueryLongString(engine *xorm.Engine) { fmt.Printf("Example one inserted: %+v\n", newExample) fmt.Println("") - query, err := goeql.SerializeMatchQuery("this", "examples", "encrypted_text_field") + query, err := goeql.MatchQuery("this", "examples", "encrypted_text_field") if err != nil { log.Fatalf("Error marshaling encrypted_text_field: %v", err) } @@ -93,7 +93,7 @@ func MatchQueryEmail(engine *xorm.Engine) { fmt.Printf("Example two inserted!: %+v\n", newExampleTwo) fmt.Println("") - query, errTwo := goeql.SerializeMatchQuery("some", "examples", "encrypted_text_field") + query, errTwo := goeql.MatchQuery("some", "examples", "encrypted_text_field") if errTwo != nil { log.Fatalf("Error marshaling encrypted_text_field: %v", errTwo) } @@ -141,7 +141,7 @@ func JsonbQuerySimple(engine *xorm.Engine) { }, } - query, errTwo := goeql.SerializeJsonbQuery(jsonbQuery, "examples", "encrypted_jsonb_field") + query, errTwo := goeql.JsonbQuery(jsonbQuery, "examples", "encrypted_jsonb_field") if errTwo != nil { log.Fatalf("Error marshaling encrypted_jsonb_field: %v", errTwo) } @@ -201,7 +201,7 @@ func JsonbQueryDeepNested(engine *xorm.Engine) { }, } - jsonbQuery, errQuery := goeql.SerializeJsonbQuery(query, "examples", "encrypted_jsonb_field") + jsonbQuery, errQuery := goeql.JsonbQuery(query, "examples", "encrypted_jsonb_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -238,7 +238,7 @@ func OreStringRangeQuery(engine *xorm.Engine) { fmt.Println("Examples inserted!") // Query - query, errQuery := goeql.SerializeOreQuery("tree", "examples", "encrypted_text_field") + query, errQuery := goeql.OreQuery("tree", "examples", "encrypted_text_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -276,7 +276,7 @@ func OreIntRangeQuery(engine *xorm.Engine) { fmt.Println("Examples inserted!", example1) fmt.Println("Examples inserted!", example2) - serializedOreIntQuery, errQuery := goeql.SerializeOreQuery(32, "examples", "encrypted_int_field") + serializedOreIntQuery, errQuery := goeql.OreQuery(32, "examples", "encrypted_int_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -322,7 +322,7 @@ func OreBoolQuery(engine *xorm.Engine) { fmt.Println("Example3 inserted!", example3) // Query - query, errQuery := goeql.SerializeOreQuery(false, "examples", "encrypted_bool_field") + query, errQuery := goeql.OreQuery(false, "examples", "encrypted_bool_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } @@ -364,7 +364,7 @@ func UniqueStringQuery(engine *xorm.Engine) { fmt.Println("Example3 inserted!", example3) var allExamples []Example - query, errQuery := goeql.SerializeUniqueQuery("test two", "examples", "encrypted_text_field") + query, errQuery := goeql.UniqueQuery("test two", "examples", "encrypted_text_field") if errQuery != nil { log.Fatalf("err: %v", errQuery) } From 4da98103c95cb9fe39ccb9a446396a4ceb2c4c39 Mon Sep 17 00:00:00 2001 From: fimac Date: Mon, 21 Oct 2024 14:33:55 +1100 Subject: [PATCH 04/10] make serializeQuery private --- languages/go/goeql/goeql.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/languages/go/goeql/goeql.go b/languages/go/goeql/goeql.go index bb8342ca..75a939d1 100644 --- a/languages/go/goeql/goeql.go +++ b/languages/go/goeql/goeql.go @@ -150,21 +150,28 @@ func (eb *EncryptedBool) Deserialize(data []byte) (EncryptedBool, error) { return false, fmt.Errorf("invalid format: missing 'p' field") } +// serializes a plaintext value used in a match query func MatchQuery(value any, table string, column string) ([]byte, error) { - return SerializeQuery(value, table, column, "match") + return serializeQuery(value, table, column, "match") } + +// serializes a plaintext value used in an ore query func OreQuery(value any, table string, column string) ([]byte, error) { - return SerializeQuery(value, table, column, "ore") + return serializeQuery(value, table, column, "ore") } + +// serializes a plaintext value used in a unique query func UniqueQuery(value any, table string, column string) ([]byte, error) { - return SerializeQuery(value, table, column, "unique") + return serializeQuery(value, table, column, "unique") } + +// serializes a plaintext value used in a jsonb query func JsonbQuery(value any, table string, column string) ([]byte, error) { - return SerializeQuery(value, table, column, "ste_vec") + return serializeQuery(value, table, column, "ste_vec") } -// SerializeQuery produces a jsonb payload used by EQL query functions to perform search operations like equality checks, range queries, and unique constraints. -func SerializeQuery(value any, table string, column string, queryType any) ([]byte, error) { +// serializeQuery produces a jsonb payload used by EQL query functions to perform search operations like equality checks, range queries, and unique constraints. +func serializeQuery(value any, table string, column string, queryType any) ([]byte, error) { query, err := ToEncryptedColumn(value, table, column, queryType) if err != nil { return nil, fmt.Errorf("error converting to EncryptedColumn: %v", err) From cd9dc7570a6111d498a89f5279d586abfa642eda Mon Sep 17 00:00:00 2001 From: fimac Date: Mon, 21 Oct 2024 14:34:12 +1100 Subject: [PATCH 05/10] add goeql tests for new query formats --- languages/go/goeql/goeql_test.go | 119 ++++++++++++++++++++++++------- 1 file changed, 95 insertions(+), 24 deletions(-) diff --git a/languages/go/goeql/goeql_test.go b/languages/go/goeql/goeql_test.go index 65d5ebab..bd5f04d0 100644 --- a/languages/go/goeql/goeql_test.go +++ b/languages/go/goeql/goeql_test.go @@ -210,34 +210,105 @@ func TestEncryptedBool_Deserialize(t *testing.T) { } } -// Test SerializeQuery Function -func TestSerializeQuery(t *testing.T) { - tests := []struct { - value interface{} - table string - column string - expectedP string - }{ - {value: "test_string", table: "table1", column: "column1", expectedP: "test_string"}, - {value: 123, table: "table2", column: "column2", expectedP: "123"}, - {value: true, table: "table3", column: "column3", expectedP: "true"}, - {value: map[string]interface{}{"key": "value"}, table: "table4", column: "column4", expectedP: `{"key":"value"}`}, +func TestMatchQuerySerialization(t *testing.T) { + value := "test_string" + table := "table1" + column := "column1" + expectedP := "test_string" + expectedQ := "match" + + serializedData, err := MatchQuery(value, table, column) + if err != nil { + t.Fatalf("SerializeQuery returned error: %v", err) } - for _, tt := range tests { - serializedData, err := SerializeQuery(tt.value, tt.table, tt.column, nil) - if err != nil { - t.Fatalf("SerializeQuery returned error: %v", err) - } + var ec EncryptedColumn + if err := json.Unmarshal(serializedData, &ec); err != nil { + t.Fatalf("Error unmarshaling serialized data: %v", err) + } - var ec EncryptedColumn - if err := json.Unmarshal(serializedData, &ec); err != nil { - t.Fatalf("Error unmarshaling serialized data: %v", err) - } + if ec.P != expectedP { + t.Errorf("Expected P to be '%s', got '%s'", expectedP, ec.P) + } - if ec.P != tt.expectedP { - t.Errorf("Expected P to be '%s', got '%s'", tt.expectedP, ec.P) - } + if ec.Q != expectedQ { + t.Errorf("Expected Q to be '%s', got '%s'", expectedQ, ec.Q) + } +} +func TestOreQuerySerialization(t *testing.T) { + value := 123 + table := "table1" + column := "column1" + expectedP := "123" + expectedQ := "ore" + + serializedData, err := OreQuery(value, table, column) + if err != nil { + t.Fatalf("SerializeQuery returned error: %v", err) + } + + var ec EncryptedColumn + if err := json.Unmarshal(serializedData, &ec); err != nil { + t.Fatalf("Error unmarshaling serialized data: %v", err) + } + + if ec.P != expectedP { + t.Errorf("Expected P to be '%s', got '%s'", expectedP, ec.P) + } + + if ec.Q != expectedQ { + t.Errorf("Expected Q to be '%s', got '%s'", expectedQ, ec.Q) + } +} + +func TestUniqueQuerySerialization(t *testing.T) { + value := true + table := "table1" + column := "column1" + expectedP := "true" + expectedQ := "unique" + + serializedData, err := UniqueQuery(value, table, column) + if err != nil { + t.Fatalf("SerializeQuery returned error: %v", err) + } + + var ec EncryptedColumn + if err := json.Unmarshal(serializedData, &ec); err != nil { + t.Fatalf("Error unmarshaling serialized data: %v", err) + } + + if ec.P != expectedP { + t.Errorf("Expected P to be '%s', got '%s'", expectedP, ec.P) + } + + if ec.Q != expectedQ { + t.Errorf("Expected Q to be '%s', got '%s'", expectedQ, ec.Q) + } +} + +func TestJsonbQuerySerialization(t *testing.T) { + value := map[string]interface{}{"key": "value"} + table := "table1" + column := "column1" + expectedP := `{"key":"value"}` + expectedQ := "ste_vec" + + serializedData, err := JsonbQuery(value, table, column) + if err != nil { + t.Fatalf("SerializeQuery returned error: %v", err) + } + + var ec EncryptedColumn + if err := json.Unmarshal(serializedData, &ec); err != nil { + t.Fatalf("Error unmarshaling serialized data: %v", err) + } + + if ec.P != expectedP { + t.Errorf("Expected P to be '%s', got '%s'", expectedP, ec.P) + } + if ec.Q != expectedQ { + t.Errorf("Expected Q to be '%s', got '%s'", expectedQ, ec.Q) } } From 9366f2eff6288450fb0cd3315de5e78d4cd2e0f4 Mon Sep 17 00:00:00 2001 From: fimac Date: Mon, 21 Oct 2024 14:48:08 +1100 Subject: [PATCH 06/10] point to central release folder for eql dsl sql file --- languages/go/xorm/main.go | 5 ----- languages/go/xorm/migrations.go | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/languages/go/xorm/main.go b/languages/go/xorm/main.go index 7c92c77c..830b9134 100644 --- a/languages/go/xorm/main.go +++ b/languages/go/xorm/main.go @@ -13,11 +13,6 @@ import ( "xorm.io/xorm/names" ) -// To setup postgres: -// Run: docker compose up -// To run examples -// Run: go run . - // Create a separate custom type for each field that is being encrypted, using the relevant go type. // This custom type can then be used to access the conversion interface to use toDB and fromDb. type EncryptedTextField string diff --git a/languages/go/xorm/migrations.go b/languages/go/xorm/migrations.go index e0ac3fb8..a0b5c2d4 100644 --- a/languages/go/xorm/migrations.go +++ b/languages/go/xorm/migrations.go @@ -30,7 +30,7 @@ func installCsCustomTypes(engine *sql.DB) { // Installing EQL func installDsl(engine *sql.DB) { - path := "./cipherstash-encrypt-dsl.sql" + path := "../../../release/cipherstash-encrypt-dsl.sql" sql, err := os.ReadFile(path) if err != nil { log.Fatalf("Failed to read SQL file: %v", err) From 43dc4576308adee679d64b65fbc70bd72a19081c Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Mon, 21 Oct 2024 15:35:04 +1100 Subject: [PATCH 07/10] Document functions idiomatically (and so go lint doesn't cry) --- languages/go/goeql/goeql.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/languages/go/goeql/goeql.go b/languages/go/goeql/goeql.go index 75a939d1..daf64baf 100644 --- a/languages/go/goeql/goeql.go +++ b/languages/go/goeql/goeql.go @@ -150,22 +150,22 @@ func (eb *EncryptedBool) Deserialize(data []byte) (EncryptedBool, error) { return false, fmt.Errorf("invalid format: missing 'p' field") } -// serializes a plaintext value used in a match query +// MatchQuery serializes a plaintext value used in a match query func MatchQuery(value any, table string, column string) ([]byte, error) { return serializeQuery(value, table, column, "match") } -// serializes a plaintext value used in an ore query +// OreQuery serializes a plaintext value used in an ore query func OreQuery(value any, table string, column string) ([]byte, error) { return serializeQuery(value, table, column, "ore") } -// serializes a plaintext value used in a unique query +// UniqueQuery serializes a plaintext value used in a unique query func UniqueQuery(value any, table string, column string) ([]byte, error) { return serializeQuery(value, table, column, "unique") } -// serializes a plaintext value used in a jsonb query +// JsonbQuery serializes a plaintext value used in a jsonb query func JsonbQuery(value any, table string, column string) ([]byte, error) { return serializeQuery(value, table, column, "ste_vec") } From ff650ca0b15623f4ae0bfca41a390302d995a8a4 Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Mon, 21 Oct 2024 15:41:32 +1100 Subject: [PATCH 08/10] Simplify the function --- languages/go/goeql/goeql.go | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/languages/go/goeql/goeql.go b/languages/go/goeql/goeql.go index daf64baf..96b2f6df 100644 --- a/languages/go/goeql/goeql.go +++ b/languages/go/goeql/goeql.go @@ -187,25 +187,15 @@ func serializeQuery(value any, table string, column string, queryType any) ([]by // ToEncryptedColumn converts a plaintext value to a string, and returns the EncryptedColumn struct for inserting into a database. func ToEncryptedColumn(value any, table string, column string, queryType any) (EncryptedColumn, error) { - if queryType == nil { - str, err := convertToString(value) - if err != nil { - return EncryptedColumn{}, fmt.Errorf("error: %v", err) - } - - data := EncryptedColumn{K: "pt", P: str, I: TableColumn{T: table, C: column}, V: 1, Q: nil} - - return data, nil - } else { - str, err := convertToString(value) - if err != nil { - return EncryptedColumn{}, fmt.Errorf("error: %v", err) - } - - data := EncryptedColumn{K: "pt", P: str, I: TableColumn{T: table, C: column}, V: 1, Q: queryType} - - return data, nil + str, err := convertToString(value) + if err != nil { + return EncryptedColumn{}, fmt.Errorf("error: %v", err) + } + data := EncryptedColumn{K: "pt", P: str, I: TableColumn{T: table, C: column}, V: 1, Q: nil} + if queryType != nil { + data.Q = queryType } + return data, nil } func convertToString(value any) (string, error) { From bdc63ee8a0acb7b53f924c40734aeb8f868e84b5 Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Mon, 21 Oct 2024 15:50:35 +1100 Subject: [PATCH 09/10] Remove unnecessary assignment --- languages/go/goeql/goeql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/languages/go/goeql/goeql.go b/languages/go/goeql/goeql.go index 96b2f6df..42d6d32c 100644 --- a/languages/go/goeql/goeql.go +++ b/languages/go/goeql/goeql.go @@ -191,7 +191,7 @@ func ToEncryptedColumn(value any, table string, column string, queryType any) (E if err != nil { return EncryptedColumn{}, fmt.Errorf("error: %v", err) } - data := EncryptedColumn{K: "pt", P: str, I: TableColumn{T: table, C: column}, V: 1, Q: nil} + data := EncryptedColumn{K: "pt", P: str, I: TableColumn{T: table, C: column}, V: 1} if queryType != nil { data.Q = queryType } From b76f6ef367cdf400071c108c4493150871da33e4 Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Mon, 21 Oct 2024 16:11:35 +1100 Subject: [PATCH 10/10] Make Q a concrete type, per the spec --- languages/go/goeql/goeql.go | 16 ++++++++-------- languages/go/goeql/goeql_test.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/languages/go/goeql/goeql.go b/languages/go/goeql/goeql.go index 42d6d32c..2ba1a71e 100644 --- a/languages/go/goeql/goeql.go +++ b/languages/go/goeql/goeql.go @@ -29,7 +29,7 @@ type EncryptedColumn struct { P string `json:"p"` I TableColumn `json:"i"` V int `json:"v"` - Q any `json:"q"` + Q string `json:"q"` } // EncryptedText is a string value to be encrypted @@ -46,7 +46,7 @@ type EncryptedBool bool // Serialize turns a EncryptedText value into a jsonb payload for CipherStash Proxy func (et EncryptedText) Serialize(table string, column string) ([]byte, error) { - val, err := ToEncryptedColumn(string(et), table, column, nil) + val, err := ToEncryptedColumn(string(et), table, column) if err != nil { return nil, fmt.Errorf("error serializing: %v", err) } @@ -69,7 +69,7 @@ func (et *EncryptedText) Deserialize(data []byte) (EncryptedText, error) { // Serialize turns a EncryptedJsonb value into a jsonb payload for CipherStash Proxy func (ej EncryptedJsonb) Serialize(table string, column string) ([]byte, error) { - val, err := ToEncryptedColumn(map[string]any(ej), table, column, nil) + val, err := ToEncryptedColumn(map[string]any(ej), table, column) if err != nil { return nil, fmt.Errorf("error serializing: %v", err) } @@ -97,7 +97,7 @@ func (ej *EncryptedJsonb) Deserialize(data []byte) (EncryptedJsonb, error) { // Serialize turns a EncryptedInt value into a jsonb payload for CipherStash Proxy func (et EncryptedInt) Serialize(table string, column string) ([]byte, error) { - val, err := ToEncryptedColumn(int(et), table, column, nil) + val, err := ToEncryptedColumn(int(et), table, column) if err != nil { return nil, fmt.Errorf("error serializing: %v", err) } @@ -124,7 +124,7 @@ func (et *EncryptedInt) Deserialize(data []byte) (EncryptedInt, error) { // Serialize turns a EncryptedBool value into a jsonb payload for CipherStash Proxy func (eb EncryptedBool) Serialize(table string, column string) ([]byte, error) { - val, err := ToEncryptedColumn(bool(eb), table, column, nil) + val, err := ToEncryptedColumn(bool(eb), table, column) if err != nil { return nil, fmt.Errorf("error serializing: %v", err) } @@ -171,7 +171,7 @@ func JsonbQuery(value any, table string, column string) ([]byte, error) { } // serializeQuery produces a jsonb payload used by EQL query functions to perform search operations like equality checks, range queries, and unique constraints. -func serializeQuery(value any, table string, column string, queryType any) ([]byte, error) { +func serializeQuery(value any, table string, column string, queryType string) ([]byte, error) { query, err := ToEncryptedColumn(value, table, column, queryType) if err != nil { return nil, fmt.Errorf("error converting to EncryptedColumn: %v", err) @@ -186,14 +186,14 @@ func serializeQuery(value any, table string, column string, queryType any) ([]by } // ToEncryptedColumn converts a plaintext value to a string, and returns the EncryptedColumn struct for inserting into a database. -func ToEncryptedColumn(value any, table string, column string, queryType any) (EncryptedColumn, error) { +func ToEncryptedColumn(value any, table string, column string, queryType ...string) (EncryptedColumn, error) { str, err := convertToString(value) if err != nil { return EncryptedColumn{}, fmt.Errorf("error: %v", err) } data := EncryptedColumn{K: "pt", P: str, I: TableColumn{T: table, C: column}, V: 1} if queryType != nil { - data.Q = queryType + data.Q = queryType[0] } return data, nil } diff --git a/languages/go/goeql/goeql_test.go b/languages/go/goeql/goeql_test.go index bd5f04d0..7dcba214 100644 --- a/languages/go/goeql/goeql_test.go +++ b/languages/go/goeql/goeql_test.go @@ -328,7 +328,7 @@ func TestToEncryptedColumn(t *testing.T) { } for _, tt := range tests { - ec, err := ToEncryptedColumn(tt.value, tt.table, tt.column, nil) + ec, err := ToEncryptedColumn(tt.value, tt.table, tt.column) if err != nil { t.Fatalf("ToEncryptedColumn returned error: %v", err) }