Rename importers into data sources for clarity.

clems4ever · clems4ever · commit 91c70533b36d · 2021-01-13T19:31:49.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -19,7 +19,7 @@
 # Release artifacts
 *.tar.gz
 go-graphkb
-importer-csv
+datasource-csv
 
 .config.yml
 main
diff --git a/.travis.yml b/.travis.yml
@@ -9,7 +9,7 @@ script:
   - .travis/run.sh
 
 before_deploy:
-  - tar cvf graphkb-$TRAVIS_TAG.tar.gz web/build go-graphkb importer-csv
+  - tar cvf graphkb-$TRAVIS_TAG.tar.gz web/build go-graphkb datasource-csv
   - echo Deploying GraphKB $TRAVIS_TAG to GitHub Releases
 deploy:
   provider: releases
diff --git a/.travis/run.sh b/.travis/run.sh
@@ -4,6 +4,6 @@ docker build -t go-graphkb-build .
 
 cid=`docker create go-graphkb-build`
 docker cp $cid:/node/src/go-graphkb .
-docker cp $cid:/node/src/importer-csv .
+docker cp $cid:/node/src/datasource-csv .
 docker cp $cid:/node/src/build web/
 docker rm $cid
diff --git a/Dockerfile b/Dockerfile
@@ -9,7 +9,7 @@ COPY cmd cmd
 COPY graphkb graphkb
 COPY internal internal
 RUN cd cmd/go-graphkb && GOOS=linux GOARCH=amd64 go build -o go-graphkb main.go
-RUN cd cmd/go-graphkb && GOOS=linux GOARCH=amd64 go build -o importer-csv main.go
+RUN cd cmd/go-graphkb && GOOS=linux GOARCH=amd64 go build -o datasource-csv main.go
 
 
 
@@ -22,4 +22,4 @@ COPY web .
 RUN yarn install && yarn build
 
 COPY --from=go-builder /go/src/cmd/go-graphkb/go-graphkb ./
-COPY --from=go-builder /go/src/cmd/go-graphkb/importer-csv ./
+COPY --from=go-builder /go/src/cmd/go-graphkb/datasource-csv ./
diff --git a/README.md b/README.md
@@ -17,7 +17,7 @@ Run the following commands
 
     # Insert the example data available in examples/ directory
     # with the following command:
-    go run cmd/importer-csv/main.go --config cmd/importer-csv/config.yml
+    go run cmd/datasource-csv/main.go --config cmd/datasource-csv/config.yml
 
 Then visit the web UI accessible at http://127.0.0.1:3000.
 
diff --git a/cmd/datasource-csv/config.yml b/cmd/datasource-csv/config.yml
@@ -1,5 +1,5 @@
 graphkb_url: "http://localhost:8080"
-graphkb_auth_token: "importer-csv"
+graphkb_auth_token: "datasource-csv"
 graphkb_skip_verify: true
 
 path: "example.csv"
diff --git a/cmd/datasource-csv/example.csv b/cmd/datasource-csv/example.csv
@@ -1,3 +1,4 @@
+source_type,source_value,relation_type,destination_type,destination_value
 employee,James,work_at,office,New York
 employee,Paul,work_at,office,Paris
 employee,Jean,work_at,office,Paris
diff --git a/cmd/datasource-csv/main.go b/cmd/datasource-csv/main.go
@@ -12,10 +12,12 @@ import (
 	"github.com/spf13/viper"
 )
 
+// CSVSource represent a CSV source file
 type CSVSource struct {
 	dataPath string
 }
 
+// NewCSVSource create a new instance of CSV source
 func NewCSVSource() *CSVSource {
 	csvSource := new(CSVSource)
 	csvSource.dataPath = viper.GetString("path")
@@ -27,7 +29,8 @@ func NewCSVSource() *CSVSource {
 	return csvSource
 }
 
-func (cs *CSVSource) Start(importer *graphkb.GraphImporter) error {
+// Start the CSVSource fetching
+func (cs *CSVSource) Start(dataSource *graphkb.DataSource) error {
 	file, err := os.Open(cs.dataPath)
 	if err != nil {
 		return err
@@ -36,12 +39,12 @@ func (cs *CSVSource) Start(importer *graphkb.GraphImporter) error {
 
 	r := csv.NewReader(file)
 
-	previousGraph, err := importer.ReadCurrentGraph()
+	previousGraph, err := dataSource.ReadCurrentGraph()
 	if err != nil {
 		return fmt.Errorf("Unable to read previous graph: %v", err)
 	}
 
-	tx := importer.CreateTransaction(previousGraph)
+	tx := dataSource.CreateTransaction(previousGraph)
 
 	header := true
 
@@ -74,6 +77,7 @@ func (cs *CSVSource) Start(importer *graphkb.GraphImporter) error {
 	return err
 }
 
+// Stop the CSV source
 func (cs *CSVSource) Stop() error {
 	return nil
 }
@@ -99,7 +103,7 @@ func main() {
 		Use: "source-csv [opts]",
 		Run: func(cmd *cobra.Command, args []string) {
 
-			options := graphkb.ImporterOptions{
+			options := graphkb.DataSourceOptions{
 				URL:        viper.GetString("graphkb_url"),
 				AuthToken:  viper.GetString("graphkb_auth_token"),
 				SkipVerify: viper.GetBool("graphkb_skip_verify"),
diff --git a/graphkb/data_source.go b/graphkb/data_source.go
@@ -3,20 +3,20 @@ package graphkb
 import (
 	"fmt"
 
-	"github.com/clems4ever/go-graphkb/internal/importers"
 	"github.com/clems4ever/go-graphkb/internal/knowledge"
 	"github.com/clems4ever/go-graphkb/internal/schema"
+	"github.com/clems4ever/go-graphkb/internal/sources"
 )
 
-// ImporterOptions options for configuring importer
-type ImporterOptions struct {
+// DataSourceOptions options for configuring the data source
+type DataSourceOptions struct {
 	URL        string
 	AuthToken  string
 	SkipVerify bool
 }
 
-// Start the importer with provided options
-func Start(source importers.Importer, options ImporterOptions) error {
+// Start the data source with provided options
+func Start(source sources.DataSource, options DataSourceOptions) error {
 	if options.URL == "" {
 		return fmt.Errorf("Please provide graphkb URL in configuration file")
 	}
@@ -25,10 +25,10 @@ func Start(source importers.Importer, options ImporterOptions) error {
 	}
 
 	api := knowledge.NewGraphAPI(options.URL, options.AuthToken, options.SkipVerify)
-	graphImporter := knowledge.NewGraphImporter(api)
+	dataSourceAPI := knowledge.NewDataSource(api)
 
-	if err := source.Start(graphImporter); err != nil {
-		return fmt.Errorf("Unable to start importer: %v", err)
+	if err := source.Start(dataSourceAPI); err != nil {
+		return fmt.Errorf("Unable to start data source: %v", err)
 	}
 
 	return nil
diff --git a/graphkb/graph_importer.go b/graphkb/graph_importer.go
@@ -2,6 +2,6 @@ package graphkb
 
 import "github.com/clems4ever/go-graphkb/internal/knowledge"
 
-type GraphImporter = knowledge.GraphImporter
+type DataSource = knowledge.DataSource
 
 type Transaction = knowledge.Transaction
diff --git a/importer-csv b/importer-csv
diff --git a/internal/database/mariadb.go b/internal/database/mariadb.go
@@ -86,7 +86,7 @@ CREATE TABLE IF NOT EXISTS relations (
 	_, err = m.db.ExecContext(context.Background(), `
 CREATE TABLE IF NOT EXISTS graph_schema (
 	id INTEGER AUTO_INCREMENT NOT NULL,
-	importer VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+	source VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
 	graph TEXT NOT NULL,
 	timestamp TIMESTAMP,
 
@@ -95,21 +95,20 @@ CREATE TABLE IF NOT EXISTS graph_schema (
 		return err
 	}
 
-	// Create the table storing importers tokens
+	// Create the table storing data sources tokens
 	_, err = m.db.ExecContext(context.Background(), `
-CREATE TABLE IF NOT EXISTS importers (
+CREATE TABLE IF NOT EXISTS sources (
 	id INTEGER AUTO_INCREMENT NOT NULL,
 	name VARCHAR(64) NOT NULL,
 	auth_token VARCHAR(64) NOT NULL,
 
-	CONSTRAINT pk_importer PRIMARY KEY (id),
-	UNIQUE unique_importer_idx (name, auth_token)
+	CONSTRAINT pk_source PRIMARY KEY (id),
+	UNIQUE unique_source_idx (name, auth_token)
 )`)
 	if err != nil {
 		return err
 	}
 
-	// Create the table storing importers tokens
 	_, err = m.db.ExecContext(context.Background(), `
 		CREATE TABLE IF NOT EXISTS query_history (
 			id INTEGER AUTO_INCREMENT NOT NULL,
@@ -568,7 +567,7 @@ func (m *MariaDB) SaveSchema(ctx context.Context, sourceName string, schema sche
 		return fmt.Errorf("Unable to json encode schema: %v", err)
 	}
 
-	_, err = m.db.ExecContext(ctx, "INSERT INTO graph_schema (importer, graph, timestamp) VALUES (?, ?, CURRENT_TIMESTAMP())",
+	_, err = m.db.ExecContext(ctx, "INSERT INTO graph_schema (source, graph, timestamp) VALUES (?, ?, CURRENT_TIMESTAMP())",
 		sourceName, string(b))
 	if err != nil {
 		return fmt.Errorf("Unable to save schema in DB: %v", err)
@@ -579,7 +578,7 @@ func (m *MariaDB) SaveSchema(ctx context.Context, sourceName string, schema sche
 
 // LoadSchema load the schema graph of the source from DB
 func (m *MariaDB) LoadSchema(ctx context.Context, sourceName string) (schema.SchemaGraph, error) {
-	row := m.db.QueryRowContext(ctx, "SELECT graph FROM graph_schema WHERE importer = ? ORDER BY id DESC LIMIT 1", sourceName)
+	row := m.db.QueryRowContext(ctx, "SELECT graph FROM graph_schema WHERE source = ? ORDER BY id DESC LIMIT 1", sourceName)
 	var rawJSON string
 	if err := row.Scan(&rawJSON); err != nil {
 		if err == sql.ErrNoRows {
@@ -597,25 +596,25 @@ func (m *MariaDB) LoadSchema(ctx context.Context, sourceName string) (schema.Sch
 	return graph, nil
 }
 
-// ListImporters list importers with their authentication tokens
-func (m *MariaDB) ListImporters(ctx context.Context) (map[string]string, error) {
-	rows, err := m.db.QueryContext(ctx, "SELECT name, auth_token FROM importers")
+// ListSources list sources with their authentication tokens
+func (m *MariaDB) ListSources(ctx context.Context) (map[string]string, error) {
+	rows, err := m.db.QueryContext(ctx, "SELECT name, auth_token FROM sources")
 
 	if err != nil {
 		return nil, fmt.Errorf("Unable to read sources from database: %v", err)
 	}
 	defer rows.Close()
 
-	importers := make(map[string]string)
+	sources := make(map[string]string)
 	for rows.Next() {
-		var importerName string
+		var sourceName string
 		var authToken string
-		if err := rows.Scan(&importerName, &authToken); err != nil {
+		if err := rows.Scan(&sourceName, &authToken); err != nil {
 			return nil, err
 		}
-		importers[importerName] = authToken
+		sources[sourceName] = authToken
 	}
-	return importers, nil
+	return sources, nil
 }
 
 // MariaDBCursor is a cursor of data retrieved by MariaDB
diff --git a/internal/importers/importer.go b/internal/importers/importer.go
diff --git a/internal/importers/registry.go b/internal/importers/registry.go
diff --git a/internal/knowledge/graph_importer.go b/internal/knowledge/graph_importer.go
@@ -1,26 +1,27 @@
 package knowledge
 
-// GraphImporter an interface for importers allowing to read current graph and push graph updates
-type GraphImporter struct {
+// DataSource an interface for data source allowing to read current graph and push graph updates
+type DataSource struct {
 	api *GraphAPI
 }
 
-// NewGraphEmitter create an emitter of graph
-func NewGraphImporter(api *GraphAPI) *GraphImporter {
-	return &GraphImporter{api: api}
+// NewDataSource create an emitter of graph
+func NewDataSource(api *GraphAPI) *DataSource {
+	return &DataSource{api: api}
 }
 
 // CreateTransaction create a full graph transaction. This kind of transaction will diff the new graph
 // with previous version of it.
-func (gi *GraphImporter) CreateTransaction(currentGraph *Graph) *Transaction {
+func (ds *DataSource) CreateTransaction(currentGraph *Graph) *Transaction {
 	transaction := new(Transaction)
 	transaction.newGraph = NewGraph()
 	transaction.binder = NewGraphBinder(transaction.newGraph)
-	transaction.api = gi.api
+	transaction.api = ds.api
 	transaction.currentGraph = currentGraph
 	return transaction
 }
 
-func (gi *GraphImporter) ReadCurrentGraph() (*Graph, error) {
-	return gi.api.ReadCurrentGraph()
+// ReadCurrentGraph read the graph related to that data source
+func (ds *DataSource) ReadCurrentGraph() (*Graph, error) {
+	return ds.api.ReadCurrentGraph()
 }
diff --git a/internal/server/server.go b/internal/server/server.go
diff --git a/internal/sources/data_source.go b/internal/sources/data_source.go
diff --git a/internal/sources/registry.go b/internal/sources/registry.go

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+source_type,source_value,relation_type,destination_type,destination_value`
`1`	`2`	`employee,James,work_at,office,New York`
`2`	`3`	`employee,Paul,work_at,office,Paris`
`3`	`4`	`employee,Jean,work_at,office,Paris`