feat json api

Author: moezakura

cmd/fti/main.go

@@ -1,5 +1,69 @@
 package main
 
+import (
+	"context"
+	"flag"
+	"log"
+	"path/filepath"
+	"time"
+
+	"cloud.google.com/go/firestore"
+	_ "github.com/BurntSushi/toml"
+	"github.com/go-generalize/fti/pkg/config"
+	"github.com/go-generalize/fti/pkg/files"
+	"github.com/go-generalize/fti/pkg/inserter"
+	"github.com/heetch/confita"
+	"github.com/heetch/confita/backend/env"
+	"github.com/heetch/confita/backend/file"
+	"github.com/heetch/confita/backend/flags"
+	"golang.org/x/xerrors"
+	_ "gopkg.in/yaml.v2"
+)
+
+var (
+	configPath = flag.String("c", "config.yaml", "-c config.yaml")
+)
+
 func main() {
+	flag.Parse()
+
+	if !files.Exists(*configPath) {
+		configFullPath, _ := filepath.Abs(*configPath)
+		log.Fatalf("not found configuration file: %s", configFullPath)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	cfg := config.Config{}
+	loader := confita.NewLoader(
+		env.NewBackend(),
+		file.NewBackend(*configPath),
+		flags.NewBackend(),
+	)
+	err := loader.Load(ctx, &cfg)
+	if err != nil {
+		log.Fatalf("cannot load configuration: %+v", err)
+	}
+
+	client, err := initFirestore(ctx, &cfg)
+	if err != nil {
+		log.Fatalf("failed to init firestore: %+v", err)
+	}
+
+	is := inserter.NewInserter(client)
+	err = is.Execute(context.Background(), &cfg)
+	if err != nil {
+		log.Fatalf("failed to execute insert: %+v", err)
+	}
+}
+
+func initFirestore(ctx context.Context, cfg *config.Config) (*firestore.Client, error) {
+	projectID := config.GetProjectID(cfg)
+	client, err := firestore.NewClient(ctx, projectID)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to initialize firestore client: %w", err)
+	}
 
+	return client, nil
 }

pkg/config/config.go

@@ -0,0 +1,7 @@
+package config
+
+type Config struct {
+	Targets                    []string `config:"targets,required" yaml:"targets"`
+	FirestoreProjectOnEmulator string   `config:"firestore_project_on_emulator,required" yaml:"firestore_project_on_emulator"`
+	FirestoreEmulatorHost      string   `config:"firestore_emulator_host,required" yaml:"firestore_emulator_host"`
+}

pkg/config/project.go

@@ -0,0 +1,22 @@
+package config
+
+import "os"
+
+func GetProjectID(cfg *Config) string {
+	if cfg.FirestoreProjectOnEmulator != "" {
+		_ = os.Setenv("FIRESTORE_EMULATOR_HOST", cfg.FirestoreEmulatorHost)
+		return cfg.FirestoreProjectOnEmulator
+	}
+
+	id, ok := os.LookupEnv("GCP_PROJECT")
+	if ok {
+		return id
+	}
+
+	id, ok = os.LookupEnv("GOOGLE_CLOUD_PROJECT")
+	if ok {
+		return id
+	}
+
+	return ""
+}

pkg/files/exists.go

@@ -0,0 +1,8 @@
+package files
+
+import "os"
+
+func Exists(filename string) bool {
+	_, err := os.Stat(filename)
+	return err == nil
+}

pkg/inserter/inserter.go

@@ -0,0 +1,131 @@
+package inserter
+
+import (
+	"context"
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"cloud.google.com/go/firestore"
+	"github.com/go-generalize/fti/pkg/config"
+	"github.com/go-generalize/fti/pkg/files"
+	"golang.org/x/xerrors"
+)
+
+type Inserter struct {
+	client *firestore.Client
+	refIDs map[string]string
+}
+
+func NewInserter(client *firestore.Client) *Inserter {
+	return &Inserter{
+		client: client,
+	}
+}
+
+func (i *Inserter) Execute(ctx context.Context, cfg *config.Config) error {
+	targetDir := cfg.Targets
+	for _, t := range targetDir {
+		if !files.Exists(t) {
+			return xerrors.Errorf("cannot find target directory: %s", targetDir)
+		}
+
+		err := filepath.Walk(t, i.executeFile(ctx))
+		if err != nil {
+			return xerrors.Errorf("failed to execute file: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (i *Inserter) executeFile(ctx context.Context) func(path string, info os.FileInfo, _ error) error {
+	return func(path string, info os.FileInfo, _ error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		cn := filepath.Base(filepath.Dir(path))
+
+		var err error
+		switch {
+		case strings.HasSuffix(path, ".json"):
+			err = i.executeJSON(ctx, cn, path)
+			if err != nil {
+				log.Printf("failed to insert json file: %s\n%+v", path, err)
+			}
+		case strings.HasSuffix(path, ".js"):
+			// TODO: implements here for js api
+		}
+
+		return nil
+	}
+}
+
+func (i *Inserter) executeJSON(ctx context.Context, cn, path string) error {
+	jb, err := os.ReadFile(path)
+	if err != nil {
+		return xerrors.Errorf("failed to read json file: %+v", err)
+	}
+
+	jm := new(JsonModel)
+	err = json.Unmarshal(jb, jm)
+	if err != nil {
+		return xerrors.Errorf("failed to unmarshal json: %w", err)
+	}
+
+	if payload, ok := jm.Payload.([]interface{}); ok {
+		for idx, p := range payload {
+			mp, ok := p.(map[string]interface{})
+			if !ok {
+				continue
+			}
+			err := i.createItem(ctx, cn, mp)
+			if err != nil {
+				return xerrors.Errorf("failed to create item in array (index=%d): %w", idx, err)
+			}
+		}
+	} else if payload, ok := jm.Payload.(map[string]interface{}); ok {
+		err := i.createItem(ctx, cn, payload)
+		if err != nil {
+			return xerrors.Errorf("failed to create item: %w", err)
+		}
+	} else {
+		// print log or error?
+	}
+
+	return nil
+}
+
+func (i *Inserter) createItem(ctx context.Context, cn string, item map[string]interface{}) error {
+	item = i.tryParseDate(item)
+
+	log.Printf("collection: %s, item: %+v", cn, item)
+	_, err := i.client.Collection(cn).NewDoc().Create(ctx, item)
+	if err != nil {
+		return xerrors.Errorf("failed to create item: %w", err)
+	}
+	return nil
+}
+
+func (i *Inserter) tryParseDate(item map[string]interface{}) map[string]interface{} {
+	for k, v := range item {
+		switch vt := v.(type) {
+		case string:
+			pt, err := time.Parse(time.RFC3339, vt)
+			if err != nil {
+				// print log?
+				continue
+			}
+			item[k] = pt
+
+		case map[string]interface{}:
+			item[k] = i.tryParseDate(vt)
+		}
+	}
+
+	return item
+}

pkg/inserter/json_model.go

@@ -0,0 +1,7 @@
+package inserter
+
+type JsonModel struct {
+	Version string      `json:"version"`
+	Ref     string      `json:"ref"`
+	Payload interface{} `json:"payload"`
+}