Unfold OR expression into unions of AND ones.

Using SQL Union queries is way more efficient than OR queries.

Note: aggregation functions are not yet supported when using union
queries.

Author: clems4ever

internal/database/mariadb.go

@@ -647,15 +647,14 @@ func (mc *MariaDBCursor) Read(ctx context.Context, doc interface{}) error {
 			}
 			output[i] = a
 		case knowledge.EdgeExprType:
-			items, err := q.Get(5)
+			items, err := q.Get(3)
 			if err != nil {
 				return nil
 			}
 			r := knowledge.RelationWithID{
-				ID:   fmt.Sprintf("%v", reflect.ValueOf(items[0])),
-				From: fmt.Sprintf("%v", reflect.ValueOf(items[1])),
-				To:   fmt.Sprintf("%v", reflect.ValueOf(items[2])),
-				Type: schema.RelationKeyType(fmt.Sprintf("%v", reflect.ValueOf(items[3]))),
+				From: fmt.Sprintf("%v", reflect.ValueOf(items[0])),
+				To:   fmt.Sprintf("%v", reflect.ValueOf(items[1])),
+				Type: schema.RelationKeyType(fmt.Sprintf("%v", reflect.ValueOf(items[2]))),
 			}
 			output[i] = r
 		case knowledge.PropertyExprType:

internal/knowledge/query_expression_builder.go

@@ -105,10 +105,7 @@ func (sev *SQLExpressionVisitor) OnExitParenthesizedExpression() error {
 
 func (sev *SQLExpressionVisitor) OnExitPropertyOrLabelsExpression(e query.QueryPropertyOrLabelsExpression) error {
 	if sev.variableName != nil {
-		path := "*"
-		if len(sev.propertiesPath) > 0 {
-			path = strings.Join(sev.propertiesPath, ".")
-		}
+		var properties []string
 		typeAndIndex, err := sev.queryGraph.FindVariable(*sev.variableName)
 		if err != nil {
 			return err
@@ -118,12 +115,22 @@ func (sev *SQLExpressionVisitor) OnExitPropertyOrLabelsExpression(e query.QueryP
 		switch typeAndIndex.Type {
 		case NodeType:
 			alias = "a"
+			properties = []string{"id", "value", "type"}
 		case RelationType:
 			alias = "r"
+			properties = []string{"from_id", "to_id", "type"}
 		}
 		alias += fmt.Sprintf("%d", typeAndIndex.Index)
+		if len(sev.propertiesPath) > 0 {
+			properties = []string{strings.Join(sev.propertiesPath, ".")}
+		}
+
+		projection := []string{}
+		for _, p := range properties {
+			projection = append(projection, fmt.Sprintf("%s.%s", alias, p))
+		}
 
-		sev.propertyLabelsExpression = fmt.Sprintf("%s.%s", alias, path)
+		sev.propertyLabelsExpression = strings.Join(projection, ", ")
 		sev.variableName = nil
 		sev.propertiesPath = nil
 	} else if sev.stringLiteral != nil {

internal/knowledge/query_expression_builder_test.go

@@ -52,6 +52,14 @@ func TestShouldBuildExpression(t *testing.T) {
 			})
 			require.NoError(t, err)
 
+			require.NoError(t, err)
+			_, _, err = qg.PushRelation(query.QueryRelationshipPattern{
+				RelationshipDetail: &query.QueryRelationshipDetail{
+					Variable: "r",
+				},
+			}, 0, 0)
+			require.NoError(t, err)
+
 			expr := CypherToExpr(tc.Cypher)
 
 			sql, err := ep.Build(&expr)
@@ -116,7 +124,11 @@ var testCases = []ExpressionTestCase{
 	},
 	ExpressionTestCase{
 		Cypher: "a",
-		SQL:    "a0.*",
+		SQL:    "a0.id, a0.value, a0.type",
+	},
+	ExpressionTestCase{
+		Cypher: "r",
+		SQL:    "r0.from_id, r0.to_id, r0.type",
 	},
 	ExpressionTestCase{
 		Cypher: "a.value",

internal/knowledge/query_projection_visitor.go

@@ -10,8 +10,12 @@ type ProjectionVisitor struct {
 	QueryGraph *QueryGraph
 
 	Aggregation    bool
-	ExpressionType ExpressionType
 	TypeAndIndex   TypeAndIndex
+	ExpressionType ExpressionType
+
+	funcInvoc  bool
+	etype      ExpressionType
+	properties []string
 }
 
 // ParseExpression return whether the expression require aggregation
@@ -26,6 +30,7 @@ func (pv *ProjectionVisitor) ParseExpression(q *query.QueryExpression) error {
 func (pv *ProjectionVisitor) OnEnterFunctionInvocation(name string) error {
 	if name == "COUNT" {
 		pv.Aggregation = true
+		pv.funcInvoc = true
 	} else {
 		return fmt.Errorf("Function %s is not supported", name)
 	}
@@ -40,12 +45,28 @@ func (pv *ProjectionVisitor) OnVariable(name string) error {
 
 	switch typeAndIndex.Type {
 	case NodeType:
-		pv.ExpressionType = NodeExprType
+		pv.etype = NodeExprType
 	case RelationType:
-		pv.ExpressionType = EdgeExprType
+		pv.etype = EdgeExprType
 	default:
-		pv.ExpressionType = PropertyExprType
+		pv.etype = PropertyExprType
 	}
 	pv.TypeAndIndex = typeAndIndex
 	return nil
 }
+
+func (pv *ProjectionVisitor) OnVariablePropertiesPath(properties []string) error {
+	pv.properties = properties
+	return nil
+}
+
+func (pv *ProjectionVisitor) OnExitPropertyOrLabelsExpression(e query.QueryPropertyOrLabelsExpression) error {
+	if len(pv.properties) > 0 || pv.funcInvoc {
+		pv.ExpressionType = PropertyExprType
+	} else {
+		pv.ExpressionType = pv.etype
+	}
+	pv.properties = nil
+	pv.funcInvoc = false
+	return nil
+}

internal/knowledge/query_sql.go

@@ -69,9 +69,113 @@ func BuildAndOrExpression(tree AndOrExpression) (string, error) {
 	return "", nil
 }
 
+func CrossProductExpressions(and1 []AndOrExpression, and2 []AndOrExpression) []AndOrExpression {
+	outExpr := []AndOrExpression{}
+	for i := range and1 {
+		for j := range and2 {
+			children := AndOrExpression{
+				And:      true,
+				Children: []AndOrExpression{and1[i], and2[j]},
+			}
+			outExpr = append(outExpr, children)
+		}
+	}
+	return outExpr
+}
+
+// UnwindOrExpressions in order to transform query with or relations into a union
+// query which is more performant, an AndOrExpression is transformed into a list of AndExpressions
+func UnwindOrExpressions(tree AndOrExpression) ([]AndOrExpression, error) {
+	if tree.Expression != "" {
+		child := AndOrExpression{Children: []AndOrExpression{tree}, And: true}
+		return []AndOrExpression{child}, nil
+	} else if !tree.And {
+		exprs := []AndOrExpression{}
+		for i := range tree.Children {
+			nestedExpressions, err := UnwindOrExpressions(tree.Children[i])
+			if err != nil {
+				return nil, err
+			}
+			exprs = append(exprs, nestedExpressions...)
+		}
+		return exprs, nil
+	} else if tree.And {
+		exprs := []AndOrExpression{}
+		for i := range tree.Children {
+			expr, err := UnwindOrExpressions(tree.Children[i])
+			if err != nil {
+				return nil, err
+			}
+
+			if len(exprs) == 0 {
+				exprs = append(exprs, expr...)
+			} else {
+				exprs = CrossProductExpressions(exprs, expr)
+			}
+		}
+		return exprs, nil
+	}
+	return nil, fmt.Errorf("Unable to detect kind of node")
+}
+
 func (sqt *SQLQueryTranslator) buildSQLSelect(
+	distinct bool, projections []string, projectionTypes []Projection, fromTables []string,
+	whereExpressions AndOrExpression, groupBy []int, limit int, offset int) (string, error) {
+	var sqlQuery string
+
+	andExpressions, err := UnwindOrExpressions(whereExpressions)
+	if err != nil {
+		return "", err
+	}
+
+	if len(andExpressions) > 1 {
+		singleQueries := []string{}
+		for _, where := range andExpressions {
+			singleQuery, err := sqt.buildSingleSQLSelect(false, projections, fromTables, where, nil, 0, 0)
+			if err != nil {
+				return "", err
+			}
+			singleQueries = append(singleQueries, fmt.Sprintf("(%s)", singleQuery))
+		}
+		if distinct {
+			sqlQuery = strings.Join(singleQueries, "\nUNION\n")
+		} else {
+			sqlQuery = strings.Join(singleQueries, "\nUNION ALL\n")
+		}
+
+		if len(groupBy) > 0 {
+			groupByProjections := []string{}
+			for i := range groupBy {
+				groupByProjections = append(groupByProjections, projections[groupBy[i]])
+			}
+
+			sqlQuery = fmt.Sprintf("SELECT %s FROM\n(%s)\nGROUP BY %s",
+				strings.Join(projections, ", "), sqlQuery, strings.Join(groupByProjections, ","))
+		}
+
+		if limit > 0 {
+			sqlQuery += fmt.Sprintf("\nLIMIT %d", limit)
+		}
+
+		if offset > 0 {
+			sqlQuery += fmt.Sprintf("\nOFFSET %d", offset)
+		}
+
+	} else {
+		and := AndOrExpression{And: true, Children: andExpressions}
+		singleQuery, err := sqt.buildSingleSQLSelect(distinct, projections, fromTables, and, groupBy, limit, offset)
+		if err != nil {
+			return "", err
+		}
+		sqlQuery = singleQuery
+	}
+
+	return sqlQuery, nil
+}
+
+func (sqt *SQLQueryTranslator) buildSingleSQLSelect(
 	distinct bool, projections []string, fromTables []string,
-	whereExpressions AndOrExpression, groupBy []string, limit int, offset int) (string, error) {
+	whereExpressions AndOrExpression, groupBy []int, limit int, offset int) (string, error) {
 
 	projectionsStr := ""
 	if distinct {
@@ -92,7 +196,11 @@ func (sqt *SQLQueryTranslator) buildSQLSelect(
 	}
 
 	if len(groupBy) > 0 {
-		sqlQuery += fmt.Sprintf("\nGROUP BY %s", strings.Join(groupBy, ", "))
+		groupByProjection := make([]string, len(groupBy))
+		for i := range groupBy {
+			groupByProjection[i] = projections[groupBy[i]]
+		}
+		sqlQuery += fmt.Sprintf("\nGROUP BY %s", strings.Join(groupByProjection, ", "))
 	}
 
 	if limit > 0 {
@@ -154,10 +262,10 @@ func (sqt *SQLQueryTranslator) Translate(query *query.QueryCypher) (*SQLTranslat
 	projectionTypes := make([]Projection, 0)
 	from := make([]string, 0)
 
-	unaggregatedProjectionItems := make([]string, 0)
+	unaggregatedProjectionItems := []int{}
 	aggregationRequired := false
 
-	for _, p := range query.QuerySinglePartQuery.ProjectionBody.ProjectionItems {
+	for i, p := range query.QuerySinglePartQuery.ProjectionBody.ProjectionItems {
 		projectionVisitor := ProjectionVisitor{QueryGraph: &sqt.QueryGraph}
 		err := projectionVisitor.ParseExpression(&p.Expression)
 		if err != nil {
@@ -170,7 +278,7 @@ func (sqt *SQLQueryTranslator) Translate(query *query.QueryCypher) (*SQLTranslat
 		}
 
 		if !projectionVisitor.Aggregation {
-			unaggregatedProjectionItems = append(unaggregatedProjectionItems, projection)
+			unaggregatedProjectionItems = append(unaggregatedProjectionItems, i)
 		} else {
 			aggregationRequired = true
 		}
@@ -197,18 +305,24 @@ func (sqt *SQLQueryTranslator) Translate(query *query.QueryCypher) (*SQLTranslat
 			})
 		}
 
-		// Append assets constraints
-		andExpressions.Children = append(andExpressions.Children, typesConstraints)
+		if len(typesConstraints.Children) > 0 {
+			// Append assets constraints
+			andExpressions.Children = append(andExpressions.Children, typesConstraints)
+		}
 	}
 	for i, r := range sqt.QueryGraph.Relations {
 		alias := fmt.Sprintf("r%d", i)
 		from = append(from, fmt.Sprintf("relations %s", alias))
 
+		typesConstraints := AndOrExpression{And: false}
 		for _, label := range r.Labels {
-			andExpressions.Children = append(andExpressions.Children, AndOrExpression{
+			typesConstraints.Children = append(typesConstraints.Children, AndOrExpression{
 				Expression: fmt.Sprintf("%s.type = '%s'", alias, label),
 			})
 		}
+		if len(typesConstraints.Children) > 0 {
+			andExpressions.Children = append(andExpressions.Children, typesConstraints)
+		}
 
 		out := AndOrExpression{
 			And: true,
@@ -280,7 +394,6 @@ func (sqt *SQLQueryTranslator) Translate(query *query.QueryCypher) (*SQLTranslat
 				}
 				andExpressions.Children = append(andExpressions.Children, orExpression)
 			}
-
 		}
 	}
 
@@ -309,11 +422,14 @@ func (sqt *SQLQueryTranslator) Translate(query *query.QueryCypher) (*SQLTranslat
 		offset = int(skipVisitor.Skip)
 	}
 
-	andExpressions.Children = append(andExpressions.Children, filterExpressions)
+	if len(filterExpressions.Children) > 0 {
+		andExpressions.Children = append(andExpressions.Children, filterExpressions)
+	}
 
 	sqlQuery, err := sqt.buildSQLSelect(
 		query.QuerySinglePartQuery.ProjectionBody.Distinct,
 		projections,
+		projectionTypes,
 		from,
 		andExpressions,
 		unaggregatedProjectionItems,