Merge pull request #84 from michaeladler/perf/improve-n-quads-2

perf: improve n-quad parser using map-based lookups

Author: kazarena

ld/api_normalize.go

@@ -244,13 +244,13 @@ func (na *NormalisationAlgorithm) Normalize(dataset *RDFDataset) {
 		// node identifiers using the canonical identifiers previously issued by
 		// canonical issuer.
 		// Note: We optimize away the copy here.
-		for _, attrNode := range []Node{quad.Subject, quad.Object, quad.Graph} {
-			if attrNode != nil {
-				attrValue := attrNode.GetValue()
-				if IsBlankNode(attrNode) && strings.Index(attrValue, "_:c14n") != 0 {
-					bn := attrNode.(*BlankNode)
-					bn.Attribute = na.canonicalIssuer.GetId(attrValue)
-				}
+		for _, nodePtr := range []*Node{&quad.Subject, &quad.Object, &quad.Graph} {
+			if *nodePtr == nil {
+				continue
+			}
+			attrValue := (*nodePtr).GetValue()
+			if IsBlankNode(*nodePtr) && !strings.HasPrefix(attrValue, "_:c14n") {
+				*nodePtr = NewBlankNode(na.canonicalIssuer.GetId(attrValue))
 			}
 		}
 

ld/node.go

@@ -31,7 +31,7 @@ type Node interface {
 	// GetValue returns the node's value.
 	GetValue() string
 
-	// Equal returns true id this node is equal to the given node.
+	// Equal returns true if this node is equal to the given node.
 	Equal(n Node) bool
 }
 
@@ -43,8 +43,8 @@ type Literal struct {
 }
 
 // NewLiteral creates a new instance of Literal.
-func NewLiteral(value string, datatype string, language string) *Literal {
-	l := &Literal{
+func NewLiteral(value string, datatype string, language string) Literal {
+	l := Literal{
 		Value:    value,
 		Language: language,
 	}
@@ -59,12 +59,12 @@ func NewLiteral(value string, datatype string, language string) *Literal {
 }
 
 // GetValue returns the node's value.
-func (l *Literal) GetValue() string {
+func (l Literal) GetValue() string {
 	return l.Value
 }
 
-// Equal returns true id this node is equal to the given node.
-func (l *Literal) Equal(n Node) bool {
+// Equal returns true if this node is equal to the given node.
+func (l Literal) Equal(n Node) bool {
 	ol, ok := n.(*Literal)
 	if !ok {
 		return false
@@ -91,21 +91,21 @@ type IRI struct {
 }
 
 // NewIRI creates a new instance of IRI.
-func NewIRI(iri string) *IRI {
-	i := &IRI{
+func NewIRI(iri string) IRI {
+	i := IRI{
 		Value: iri,
 	}
 
 	return i
 }
 
 // GetValue returns the node's value.
-func (iri *IRI) GetValue() string {
+func (iri IRI) GetValue() string {
 	return iri.Value
 }
 
-// Equal returns true id this node is equal to the given node.
-func (iri *IRI) Equal(n Node) bool {
+// Equal returns true if this node is equal to the given node.
+func (iri IRI) Equal(n Node) bool {
 	if oiri, ok := n.(*IRI); ok {
 		return iri.Value == oiri.Value
 	}
@@ -119,21 +119,21 @@ type BlankNode struct {
 }
 
 // NewBlankNode creates a new instance of BlankNode.
-func NewBlankNode(attribute string) *BlankNode {
-	bn := &BlankNode{
+func NewBlankNode(attribute string) BlankNode {
+	bn := BlankNode{
 		Attribute: attribute,
 	}
 
 	return bn
 }
 
 // GetValue returns the node's value.
-func (bn *BlankNode) GetValue() string {
+func (bn BlankNode) GetValue() string {
 	return bn.Attribute
 }
 
-// Equal returns true id this node is equal to the given node.
-func (bn *BlankNode) Equal(n Node) bool {
+// Equal returns true if this node is equal to the given node.
+func (bn BlankNode) Equal(n Node) bool {
 	if obn, ok := n.(*BlankNode); ok {
 		return bn.Attribute == obn.Attribute
 	}
@@ -143,19 +143,19 @@ func (bn *BlankNode) Equal(n Node) bool {
 
 // IsBlankNode returns true if the given node is a blank node
 func IsBlankNode(node Node) bool {
-	_, isBlankNode := node.(*BlankNode)
+	_, isBlankNode := node.(BlankNode)
 	return isBlankNode
 }
 
 // IsIRI returns true if the given node is an IRI node
 func IsIRI(node Node) bool {
-	_, isIRI := node.(*IRI)
+	_, isIRI := node.(IRI)
 	return isIRI
 }
 
 // IsLiteral returns true if the given node is a literal node
 func IsLiteral(node Node) bool {
-	_, isLiteral := node.(*Literal)
+	_, isLiteral := node.(Literal)
 	return isLiteral
 }
 
@@ -173,7 +173,7 @@ func RdfToObject(n Node, useNativeTypes bool) (map[string]interface{}, error) {
 		}, nil
 	}
 
-	literal := n.(*Literal)
+	literal := n.(Literal)
 
 	// convert literal object to JSON-LD
 	rval := map[string]interface{}{

ld/rdf_dataset.go

@@ -306,11 +306,11 @@ var (
 func InvalidNode(node Node) bool {
 
 	switch v := node.(type) {
-	case *IRI:
+	case IRI:
 		if !validIRI(v.Value) {
 			return true
 		}
-	case *Literal:
+	case Literal:
 		if v.Language != "" && !validLanguageRegex.MatchString(v.Language) {
 			return true
 		}

ld/serialize_nquads.go

@@ -84,7 +84,7 @@ func toNQuad(triple *Quad, graphName string) string {
 	} else if IsBlankNode(o) {
 		quad += o.GetValue()
 	} else {
-		literal := o.(*Literal)
+		literal := o.(Literal)
 		escaped := escape(literal.GetValue())
 		quad += "\"" + escaped + "\""
 		if literal.Datatype == RDFLangString {
@@ -237,6 +237,9 @@ func ParseNQuadsFrom(o interface{}) (*RDFDataset, error) {
 	// build RDF dataset
 	dataset := NewRDFDataset()
 
+	// maintain a set of triples for each graph to check for duplicates
+	triplesByGraph := make(map[string]map[Quad]struct{})
+
 	scanner, err := newScannerFor(o)
 	if err != nil {
 		return nil, err
@@ -302,21 +305,19 @@ func ParseNQuadsFrom(o interface{}) (*RDFDataset, error) {
 
 		// initialise graph in dataset
 		triples, present := dataset.Graphs[name]
+		if triplesByGraph[name] == nil {
+			triplesByGraph[name] = make(map[Quad]struct{})
+		}
+
 		if !present {
 			dataset.Graphs[name] = []*Quad{triple}
 		} else {
 			// add triple if unique to its graph
-			containsTriple := false
-			for _, elem := range triples {
-				if triple.Equal(elem) {
-					containsTriple = true
-					break
-				}
-			}
-			if !containsTriple {
+			if _, hasTriple := triplesByGraph[name][*triple]; !hasTriple {
 				dataset.Graphs[name] = append(triples, triple)
 			}
 		}
+		triplesByGraph[name][*triple] = struct{}{}
 	}
 	if err := scanner.Err(); err != nil {
 		return nil, NewJsonLdError(IOError, err)