Skip to content

Commit 01a0864

Browse files
moezakuramoezakura
committed
split json inserter
1 parent 3f74b32 commit 01a0864

File tree

2 files changed

+121
-101
lines changed

2 files changed

+121
-101
lines changed

‎pkg/inserter/inserter.go‎

Lines changed: 3 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,10 @@ package inserter
22

33
import (
44
"context"
5-
"encoding/json"
65
"log"
76
"os"
87
"path/filepath"
98
"strings"
10-
"time"
119

1210
"cloud.google.com/go/firestore"
1311
"github.com/go-generalize/fti/pkg/config"
@@ -16,14 +14,12 @@ import (
1614
)
1715

1816
type Inserter struct {
19-
client *firestore.Client
20-
refIDs map[string]string
17+
jsonInserter *JSONInserter
2118
}
2219

2320
func NewInserter(client *firestore.Client) *Inserter {
2421
return &Inserter{
25-
client: client,
26-
refIDs: map[string]string{},
22+
jsonInserter: NewJSONInserter(client),
2723
}
2824
}
2925

@@ -54,7 +50,7 @@ func (i *Inserter) executeFile(ctx context.Context) func(path string, info os.Fi
5450
var err error
5551
switch {
5652
case strings.HasSuffix(path, ".json"):
57-
err = i.executeJSON(ctx, cn, path)
53+
err = i.jsonInserter.executeJSON(ctx, cn, path)
5854
if err != nil {
5955
log.Printf("failed to insert json file: %s\n%+v", path, err)
6056
}
@@ -65,97 +61,3 @@ func (i *Inserter) executeFile(ctx context.Context) func(path string, info os.Fi
6561
return nil
6662
}
6763
}
68-
69-
func (i *Inserter) executeJSON(ctx context.Context, cn, path string) error {
70-
jb, err := os.ReadFile(path)
71-
if err != nil {
72-
return xerrors.Errorf("failed to read json file: %+v", err)
73-
}
74-
75-
jm := new(JsonModel)
76-
err = json.Unmarshal(jb, jm)
77-
if err != nil {
78-
return xerrors.Errorf("failed to unmarshal json: %w", err)
79-
}
80-
81-
if payload, ok := jm.Payload.([]interface{}); ok {
82-
for idx, p := range payload {
83-
mp, ok := p.(map[string]interface{})
84-
if !ok {
85-
continue
86-
}
87-
err := i.createItem(ctx, cn, jm.Ref, mp)
88-
if err != nil {
89-
return xerrors.Errorf("failed to create item in array (index=%d): %w", idx, err)
90-
}
91-
}
92-
} else if payload, ok := jm.Payload.(map[string]interface{}); ok {
93-
err := i.createItem(ctx, cn, jm.Ref, payload)
94-
if err != nil {
95-
return xerrors.Errorf("failed to create item: %w", err)
96-
}
97-
} else {
98-
// print log or error?
99-
}
100-
101-
return nil
102-
}
103-
104-
func (i *Inserter) createItem(ctx context.Context, cn, refID string, item map[string]interface{}) error {
105-
item = i.tryParseDate(item)
106-
item = i.setRefs(item)
107-
108-
d := i.client.Collection(cn).NewDoc()
109-
_, err := d.Create(ctx, item)
110-
if err != nil {
111-
return xerrors.Errorf("failed to create item: %w", err)
112-
}
113-
114-
if refID != "" {
115-
if _, ok := i.refIDs[refID]; ok {
116-
return xerrors.Errorf("already ref id: %s", refID)
117-
}
118-
i.refIDs[refID] = d.ID
119-
}
120-
121-
return nil
122-
}
123-
124-
func (i *Inserter) tryParseDate(item map[string]interface{}) map[string]interface{} {
125-
for k, v := range item {
126-
switch vt := v.(type) {
127-
case string:
128-
pt, err := time.Parse(time.RFC3339, vt)
129-
if err != nil {
130-
// print log?
131-
continue
132-
}
133-
item[k] = pt
134-
135-
case map[string]interface{}:
136-
item[k] = i.tryParseDate(vt)
137-
}
138-
}
139-
140-
return item
141-
}
142-
143-
func (i *Inserter) setRefs(item map[string]interface{}) map[string]interface{} {
144-
for k, v := range item {
145-
switch vt := v.(type) {
146-
case string:
147-
if strings.HasPrefix(vt, "$") {
148-
refID := strings.TrimPrefix(vt, "$")
149-
rv, ok := i.refIDs[refID]
150-
if !ok {
151-
log.Printf("%s was not found", refID)
152-
continue
153-
}
154-
item[k] = rv
155-
}
156-
}
157-
}
158-
159-
return item
160-
161-
}

‎pkg/inserter/json_inserter.go‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package inserter
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"log"
7+
"os"
8+
"strings"
9+
"time"
10+
11+
"cloud.google.com/go/firestore"
12+
"golang.org/x/xerrors"
13+
)
14+
15+
type JSONInserter struct {
16+
client *firestore.Client
17+
refIDs map[string]string
18+
}
19+
20+
func NewJSONInserter(client *firestore.Client) *JSONInserter {
21+
return &JSONInserter{
22+
client: client,
23+
refIDs: map[string]string{},
24+
}
25+
}
26+
27+
func (j *JSONInserter) executeJSON(ctx context.Context, cn, path string) error {
28+
jb, err := os.ReadFile(path)
29+
if err != nil {
30+
return xerrors.Errorf("failed to read json file: %+v", err)
31+
}
32+
33+
jm := new(JsonModel)
34+
err = json.Unmarshal(jb, jm)
35+
if err != nil {
36+
return xerrors.Errorf("failed to unmarshal json: %w", err)
37+
}
38+
39+
if payload, ok := jm.Payload.([]interface{}); ok {
40+
for idx, p := range payload {
41+
mp, ok := p.(map[string]interface{})
42+
if !ok {
43+
continue
44+
}
45+
err := j.createItem(ctx, cn, jm.Ref, mp)
46+
if err != nil {
47+
return xerrors.Errorf("failed to create item in array (index=%d): %w", idx, err)
48+
}
49+
}
50+
} else if payload, ok := jm.Payload.(map[string]interface{}); ok {
51+
err := j.createItem(ctx, cn, jm.Ref, payload)
52+
if err != nil {
53+
return xerrors.Errorf("failed to create item: %w", err)
54+
}
55+
} else {
56+
// print log or error?
57+
}
58+
59+
return nil
60+
}
61+
62+
func (j *JSONInserter) createItem(ctx context.Context, cn, refID string, item map[string]interface{}) error {
63+
item = j.tryParseDate(item)
64+
item = j.setRefs(item)
65+
66+
d := j.client.Collection(cn).NewDoc()
67+
_, err := d.Create(ctx, item)
68+
if err != nil {
69+
return xerrors.Errorf("failed to create item: %w", err)
70+
}
71+
72+
if refID != "" {
73+
if _, ok := j.refIDs[refID]; ok {
74+
return xerrors.Errorf("already ref id: %s", refID)
75+
}
76+
j.refIDs[refID] = d.ID
77+
}
78+
79+
return nil
80+
}
81+
82+
func (j *JSONInserter) tryParseDate(item map[string]interface{}) map[string]interface{} {
83+
for k, v := range item {
84+
switch vt := v.(type) {
85+
case string:
86+
pt, err := time.Parse(time.RFC3339, vt)
87+
if err != nil {
88+
// print log?
89+
continue
90+
}
91+
item[k] = pt
92+
93+
case map[string]interface{}:
94+
item[k] = j.tryParseDate(vt)
95+
}
96+
}
97+
98+
return item
99+
}
100+
101+
func (j *JSONInserter) setRefs(item map[string]interface{}) map[string]interface{} {
102+
for k, v := range item {
103+
switch vt := v.(type) {
104+
case string:
105+
if strings.HasPrefix(vt, "$") {
106+
refID := strings.TrimPrefix(vt, "$")
107+
rv, ok := j.refIDs[refID]
108+
if !ok {
109+
log.Printf("%s was not found", refID)
110+
continue
111+
}
112+
item[k] = rv
113+
}
114+
}
115+
}
116+
117+
return item
118+
}

0 commit comments

Comments
 (0)