Skip to content

Commit c20e816

Browse files
mvaledmvaled
committed
Merge remote-tracking branch 'ddvk/master'
2 parents 506bcb2 + 609cba8 commit c20e816

File tree

8 files changed

+121
-43
lines changed

8 files changed

+121
-43
lines changed

‎api/sync15/apictx.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -142,13 +142,13 @@ func (ctx *ApiCtx) CreateDir(parentId, name string, notify bool) (*model.Documen
142142
if err != nil {
143143
return nil, err
144144
}
145-
files.AddMap(objectName, filePath)
145+
files.AddMap(objectName, filePath, archive.MetadataExt)
146146

147147
objectName, filePath, err = archive.CreateContent(id, "", tmpDir, nil)
148148
if err != nil {
149149
return nil, err
150150
}
151-
files.AddMap(objectName, filePath)
151+
files.AddMap(objectName, filePath, archive.ContentExt)
152152

153153
doc := NewBlobDoc(name, id, model.DirectoryType, parentId)
154154

@@ -169,6 +169,7 @@ func (ctx *ApiCtx) CreateDir(parentId, name string, notify bool) (*model.Documen
169169
if err != nil {
170170
return nil, err
171171
}
172+
//does not accept rm-file in header
172173
err = ctx.blobStorage.UploadBlob(hashStr, f.Name, reader)
173174
reader.Close()
174175

@@ -185,7 +186,7 @@ func (ctx *ApiCtx) CreateDir(parentId, name string, notify bool) (*model.Documen
185186
return nil, err
186187
}
187188
// defer indexReader.Close()
188-
err = ctx.blobStorage.UploadBlob(doc.Hash, addSchema(doc.DocumentID), indexReader)
189+
err = ctx.blobStorage.UploadBlob(doc.Hash, addExt(doc.DocumentID, archive.DocSchemaExt), indexReader)
189190
if err != nil {
190191
return nil, err
191192
}
@@ -227,7 +228,7 @@ func Sync(b *BlobStorage, tree *HashTree, operation func(t *HashTree) error, not
227228
if err != nil {
228229
return err
229230
}
230-
err = b.UploadBlob(tree.Hash, addSchema("root"), indexReader)
231+
err = b.UploadBlob(tree.Hash, addExt("root", archive.DocSchemaExt), indexReader)
231232
if err != nil {
232233
return err
233234
}
@@ -305,7 +306,7 @@ func (ctx *ApiCtx) MoveEntry(src, dstDir *model.Node, name string) (*model.Node,
305306
return err
306307
}
307308

308-
err = ctx.blobStorage.UploadBlob(hashStr, doc.DocumentID, reader)
309+
err = ctx.blobStorage.UploadBlob(hashStr, addExt(doc.DocumentID, archive.MetadataExt), reader)
309310

310311
if err != nil {
311312
return err
@@ -317,7 +318,7 @@ func (ctx *ApiCtx) MoveEntry(src, dstDir *model.Node, name string) (*model.Node,
317318
return err
318319
}
319320
// defer indexReader.Close()
320-
return ctx.blobStorage.UploadBlob(doc.Hash, addSchema(doc.DocumentID), indexReader)
321+
return ctx.blobStorage.UploadBlob(doc.Hash, addExt(doc.DocumentID, archive.DocSchemaExt), indexReader)
321322
}, true)
322323

323324
if err != nil {
@@ -392,7 +393,7 @@ func (ctx *ApiCtx) UploadDocument(parentId string, sourceDocPath string, notify
392393
return nil, err
393394
}
394395
// defer indexReader.Close()
395-
err = ctx.blobStorage.UploadBlob(doc.Hash, addSchema(doc.DocumentID), indexReader)
396+
err = ctx.blobStorage.UploadBlob(doc.Hash, addExt(doc.DocumentID, archive.DocSchemaExt), indexReader)
396397
if err != nil {
397398
return nil, err
398399
}

‎api/sync15/blobdoc.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ func (d *BlobDoc) IndexReader() (io.Reader, error) {
9999
return nil, errors.New("no files")
100100
}
101101
var w bytes.Buffer
102-
w.WriteString(SchemaVersion)
102+
w.WriteString(SchemaVersionV3)
103103
w.WriteString("\n")
104104
for _, d := range d.Files {
105105
w.WriteString(d.Line())

‎api/sync15/tree.go‎

Lines changed: 63 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,15 @@ import (
1111
"sort"
1212
"strconv"
1313

14+
"github.com/juruen/rmapi/archive"
1415
"github.com/juruen/rmapi/log"
1516
"github.com/juruen/rmapi/transport"
1617
"golang.org/x/sync/errgroup"
1718
)
1819

19-
const SchemaVersion = "3"
20+
const SchemaVersionV3 = "3"
21+
const SchemaVersionV4 = "4"
22+
2023
const DocType = "80000000"
2124
const FileType = "0"
2225
const Delimiter = ':'
@@ -76,37 +79,79 @@ func parseEntry(line string) (*Entry, error) {
7679
return &entry, nil
7780
}
7881

82+
func parseSchemaV4(line string) (entriesCount int, totalSize int64, err error) {
83+
r := NewFieldReader(line)
84+
_, _ = r.Next() //0
85+
_, _ = r.Next() //.
86+
entriesCountStr, _ := r.Next() //count?
87+
totalSizeStr, _ := r.Next() //size?
88+
89+
entriesCount, err = strconv.Atoi(entriesCountStr)
90+
if err != nil {
91+
return
92+
}
93+
totalSize, err = strconv.ParseInt(totalSizeStr, 10, 64)
94+
if err != nil {
95+
return
96+
}
97+
98+
return
99+
}
100+
79101
func parseIndex(f io.Reader) ([]*Entry, error) {
80102
var entries []*Entry
81103
scanner := bufio.NewScanner(f)
82104
eof := scanner.Scan()
83105
if !eof {
84-
return nil, fmt.Errorf("empty file")
106+
return nil, fmt.Errorf("empty index file")
85107
}
86108
schema := scanner.Text()
109+
expectedCount := 0
110+
count := 0
111+
var err error
112+
switch schema {
87113

88-
if schema != SchemaVersion {
89-
return nil, fmt.Errorf("wrong schema got %s, expected: %s", schema, SchemaVersion)
90-
}
91-
for scanner.Scan() {
92-
line := scanner.Text()
93-
if line == "" {
94-
log.Warning.Printf("TODO: empty line in index file, ignored")
95-
continue
114+
case SchemaVersionV4:
115+
eof := scanner.Scan()
116+
if !eof {
117+
return nil, fmt.Errorf("expecting a schema v4 line")
96118
}
97-
entry, err := parseEntry(line)
119+
line := scanner.Text()
120+
expectedCount, _, err = parseSchemaV4(line)
98121
if err != nil {
99-
return nil, fmt.Errorf("cant parse line '%s', %w", line, err)
122+
return nil, fmt.Errorf("can't parse v4 line %v", err)
100123
}
124+
fallthrough
125+
case SchemaVersionV3:
126+
for scanner.Scan() {
127+
line := scanner.Text()
128+
if line == "" {
129+
log.Warning.Printf("TODO: empty line in index file, ignored")
130+
continue
131+
}
132+
count++
133+
entry, err := parseEntry(line)
134+
if err != nil {
135+
return nil, fmt.Errorf("cant parse line '%s', %w", line, err)
136+
}
101137

102-
entries = append(entries, entry)
138+
entries = append(entries, entry)
139+
}
140+
default:
141+
return nil, fmt.Errorf("unsupported schema %s", schema)
103142
}
143+
if schema == SchemaVersionV4 {
144+
if count != expectedCount {
145+
log.Warning.Printf("entries mismatch, expected %d, but was %d", expectedCount, count)
146+
}
147+
}
148+
104149
return entries, nil
105150
}
106151

107152
func (t *HashTree) IndexReader() (io.Reader, error) {
108153
var w bytes.Buffer
109-
w.WriteString(SchemaVersion)
154+
w.WriteString(SchemaVersionV3)
110155
w.WriteString("\n")
111156
for _, d := range t.Docs {
112157
w.WriteString(d.Line())
@@ -167,8 +212,9 @@ func (t *HashTree) Rehash() error {
167212
return nil
168213
}
169214

170-
func addSchema(name string) string {
171-
return name + ".docSchema"
215+
// adds the extensions to filename (for the rm-file header later)
216+
func addExt(name string, ext archive.RmExt) string {
217+
return name + "." + string(ext)
172218
}
173219

174220
// / Mirror makes the tree look like the storage
@@ -189,7 +235,7 @@ func (t *HashTree) Mirror(r RemoteStorage, maxconcurrent int) error {
189235
}
190236
log.Info.Printf("remote root hash different")
191237

192-
rootIndexReader, err := r.GetReader(rootHash, addSchema("root"))
238+
rootIndexReader, err := r.GetReader(rootHash, addExt("root", archive.DocSchemaExt))
193239
if err != nil {
194240
return fmt.Errorf("cannot get root hash %v", err)
195241
}

‎api/sync15/tree_test.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,21 @@ func TestParseIndex(t *testing.T) {
3939
return
4040
}
4141
}
42+
func TestParseIndexV4(t *testing.T) {
43+
index := `4
44+
0:.:2:1823419036
45+
0f83178c4ebe6a60fae0360b74916ee9e1faa5de1c56ab3481eccdc5cb98754f:0:fe0039fb-56a0-4561-a36f-a820f0009622.content:0:993
46+
17eca6c9a540c993f5f5506bb09b7a40993c02fa8f065b1a6a442e412cf2fd04:0:fe0039fb-56a0-4561-a36f-a820f0009622.metadata:0:320`
47+
entries, err := parseIndex(strings.NewReader(index))
48+
if err != nil {
49+
t.Error(err)
50+
return
51+
}
52+
if len(entries) != 2 {
53+
t.Error("wrong number of entries")
54+
return
55+
}
56+
}
4257

4358
func TestCreateDocIndex(t *testing.T) {
4459
doc := &BlobDoc{

‎archive/blob.go‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,31 @@ import (
1717
"github.com/juruen/rmapi/util"
1818
)
1919

20+
// extensions of the internal rm files
21+
type RmExt string
22+
23+
const (
24+
// DocSchema
25+
DocSchemaExt RmExt = "docSchema"
26+
MetadataExt RmExt = "metadata"
27+
ContentExt RmExt = "content"
28+
)
29+
2030
type NamePath struct {
21-
Name string
22-
Path string
31+
Name string
32+
Path string
33+
FileType RmExt
2334
}
2435

2536
type DocumentFiles struct {
2637
Files []NamePath
2738
}
2839

29-
func (d *DocumentFiles) AddMap(name, filepath string) {
40+
func (d *DocumentFiles) AddMap(name, filepath string, filetype RmExt) {
3041
fs := NamePath{
31-
Name: name,
32-
Path: filepath,
42+
Name: name,
43+
Path: filepath,
44+
FileType: filetype,
3345
}
3446
d.Files = append(d.Files, fs)
3547
}
@@ -53,7 +65,7 @@ func Prepare(name, parentId, sourceDocPath, ext, tmpDir string) (files *Document
5365
err = err1
5466
return
5567
}
56-
files.AddMap(objectName, filePath)
68+
files.AddMap(objectName, filePath, MetadataExt)
5769
} else {
5870
err = FixMetadata(parentId, name, metadataPath)
5971
if err != nil {
@@ -71,19 +83,19 @@ func Prepare(name, parentId, sourceDocPath, ext, tmpDir string) (files *Document
7183
doctype = "notebook"
7284
pageIds = []string{pageId}
7385
}
74-
files.AddMap(objectName, sourceDocPath)
86+
files.AddMap(objectName, sourceDocPath, RmExt(doctype))
7587
objectName, filePath, err1 := CreateMetadata(id, name, parentId, model.DocumentType, tmpDir)
7688
if err1 != nil {
7789
err = err1
7890
return
7991
}
80-
files.AddMap(objectName, filePath)
92+
files.AddMap(objectName, filePath, MetadataExt)
8193

8294
objectName, filePath, err = CreateContent(id, doctype, tmpDir, pageIds)
8395
if err != nil {
8496
return
8597
}
86-
files.AddMap(objectName, filePath)
98+
files.AddMap(objectName, filePath, ContentExt)
8799
}
88100
return files, id, err
89101
}
@@ -123,7 +135,11 @@ func Unpack(src, dest string) (id string, files *DocumentFiles, metadataPath str
123135
for _, f := range r.File {
124136
fname := f.Name
125137

126-
if strings.HasSuffix(fname, ".content") {
138+
ext := filepath.Ext(fname)
139+
if len(ext) > 0 {
140+
ext = ext[1:]
141+
}
142+
if ext == string(ContentExt) {
127143
id = strings.TrimSuffix(fname, path.Ext(fname))
128144
}
129145
// Store filename/path for returning and using later on
@@ -140,10 +156,10 @@ func Unpack(src, dest string) (id string, files *DocumentFiles, metadataPath str
140156
os.MkdirAll(fpath, os.ModePerm)
141157
continue
142158
} else {
143-
files.AddMap(f.Name, fpath)
159+
files.AddMap(f.Name, fpath, RmExt(ext))
144160
}
145161

146-
if strings.HasSuffix(fname, ".metadata") {
162+
if ext == string(MetadataExt) {
147163
metadataPath = fpath
148164
}
149165

‎archive/reader.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ func (z *Zip) Read(r io.ReaderAt, size int64) error {
6060

6161
// readContent reads the .content file contained in an archive and the UUID
6262
func (z *Zip) readContent(zr *zip.Reader) error {
63-
files, err := zipExtFinder(zr, ".content")
63+
files, err := zipExtFinder(zr, "."+string(ContentExt))
6464
if err != nil {
6565
return err
6666
}

‎archive/writer.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func (z *Zip) writeContent(zw *zip.Writer) error {
5959
return err
6060
}
6161

62-
name := fmt.Sprintf("%s.content", z.UUID)
62+
name := fmt.Sprintf("%s.%s", z.UUID, ContentExt)
6363

6464
w, err := addToZip(zw, name)
6565
if err != nil {

‎archive/zipdoc.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ func CreateZipDocument(id, srcPath string) (zipPath string, err error) {
138138
f.Write(make([]byte, 0))
139139

140140
// Create content content
141-
f, err = w.Create(fmt.Sprintf("%s.content", id))
141+
f, err = w.Create(fmt.Sprintf("%s.%s", id, string(ContentExt)))
142142
if err != nil {
143143
log.Error.Println("failed to create content entry in zip file", err)
144144
return
@@ -168,7 +168,7 @@ func CreateZipDirectory(id string) (string, error) {
168168
defer w.Close()
169169

170170
// Create content content
171-
f, err := w.Create(fmt.Sprintf("%s.content", id))
171+
f, err := w.Create(fmt.Sprintf("%s.%s", id, string(ContentExt)))
172172
if err != nil {
173173
log.Error.Println("failed to create content entry in zip file", err)
174174
return "", err
@@ -218,7 +218,7 @@ func createZipContent(ext string, pageIDs []string) (string, error) {
218218
}
219219

220220
func CreateContent(id, ext, fpath string, pageIds []string) (fileName, filePath string, err error) {
221-
fileName = id + ".content"
221+
fileName = id + "." + string(ContentExt)
222222
filePath = path.Join(fpath, fileName)
223223
content := "{}"
224224

@@ -240,7 +240,7 @@ func UnixTimestamp() string {
240240
}
241241

242242
func CreateMetadata(id, name, parent, colType, fpath string) (fileName string, filePath string, err error) {
243-
fileName = id + ".metadata"
243+
fileName = id + "." + string(MetadataExt)
244244
filePath = path.Join(fpath, fileName)
245245
meta := MetadataFile{
246246
DocName: name,

0 commit comments

Comments
 (0)