Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions cmd/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"errors"
"fmt"
"io"
"math/rand"
"os"
"path"
"runtime"
Expand Down Expand Up @@ -139,7 +138,7 @@ func (j *juiceFS) Put(rCtx context.Context, key string, in io.Reader, getters ..
if len(name) > 200 {
name = name[:200]
}
tmp = path.Join(path.Dir(p), fmt.Sprintf(".%s.tmp.%d", name, rand.Int()))
tmp = object.TmpFilePath(p, name)
defer func() {
if err != nil {
if e := j.jfs.Delete(ctx, tmp); e != 0 {
Expand Down
2 changes: 1 addition & 1 deletion docs/en/guide/sync.md
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ If you notice the progress bar is not changing, use the methods below for monito
* Use [`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats) to quickly check current I/O status.
* Review the [client log](../administration/fault_diagnosis_and_analysis.md#client-log) (default path: `/var/log/juicefs.log`) for [slow requests or timeout errors](../administration/troubleshooting.md#io-error-object-storage).

* If the destination is a local disk, check the directory for temporary files with `.tmp.xxx`. During the synchronization process, the transfer results are written to these temporary files. Once the transfer is complete, they are renamed to finalize the write. By monitoring the size changes of the temporary files, you can determine the current I/O status.
* If the destination is a local disk, check the directory for temporary files with `.jfs.xxx.tmp.xxx`. During the synchronization process, the transfer results are written to these temporary files. Once the transfer is complete, they are renamed to finalize the write. By monitoring the size changes of the temporary files, you can determine the current I/O status.

* If both the source and destination are object storage systems, use tools like `nethogs` to check network I/O.

Expand Down
2 changes: 1 addition & 1 deletion docs/zh_cn/guide/sync.md
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ juicefs sync --worker host1,host2 /jfs-src /jfs-dst
* 如果数据同步的两方有任何一个是 JuiceFS 宿主机挂载点:
* 用 [`juicefs stats`](../administration/fault_diagnosis_and_analysis.md#stats) 快速查看文件系统是否正在写入(或读出);
* 阅读[客户端日志](../administration/fault_diagnosis_and_analysis.md#client-log)(默认 `/var/log/juicefs.log`),观察是否有[慢请求或者超时错误日志](../administration/troubleshooting.md#io-error-object-storage)。
* 如果数据同步的目的地是宿主机本地盘,可以直接观察目录下是否存在名称中带 `.tmp.xxx` 后缀的临时文件,`sync` 过程中会将传输结果写入临时文件,待传输完成后进行重命名,才完成最终的写入。观察临时文件大小是否变化,就能确定当前的写入状况;
* 如果数据同步的目的地是宿主机本地盘,可以直接观察目录下是否存在名称中带 `.jfs.xxx.tmp.xxx` 后缀的临时文件,`sync` 过程中会将传输结果写入临时文件,待传输完成后进行重命名,才完成最终的写入。观察临时文件大小是否变化,就能确定当前的写入状况;
* 如果传输目的地均为对象存储,可以通过类似 `nethogs` 的命令,查看出入网流量,来判断传输进展;
* 以上手段均未能获得有效排查信息,则需要对 `sync` 进程采集 goroutine,结合源码分析排查:

Expand Down
3 changes: 1 addition & 2 deletions pkg/object/cifs.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"context"
"fmt"
"io"
"math/rand"
"net"
"net/url"
"os"
Expand Down Expand Up @@ -237,7 +236,7 @@ func (c *cifsStore) Put(ctx context.Context, key string, in io.Reader, getters .
if len(name) > 200 {
name = name[:200]
}
tmp = path.Join(path.Dir(p), fmt.Sprintf(".%s.tmp.%d", name, rand.Int()))
tmp = TmpFilePath(p, name)
defer func() {
if err != nil {
_ = conn.share.Remove(tmp)
Expand Down
4 changes: 1 addition & 3 deletions pkg/object/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,11 @@ import (
"fmt"
"io"
"io/fs"
"math/rand"
"os"
"path"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"

"github.com/juicedata/juicefs/pkg/utils"
Expand Down Expand Up @@ -160,7 +158,7 @@ func (d *filestore) Put(ctx context.Context, key string, in io.Reader, getters .
if len(name) > 200 {
name = name[:200]
}
tmp = filepath.Join(filepath.Dir(p), "."+name+".tmp"+strconv.Itoa(rand.Int()))
tmp = TmpFilePath(p, name)
defer func() {
if err != nil {
_ = os.Remove(tmp)
Expand Down
3 changes: 1 addition & 2 deletions pkg/object/hdfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import (
"fmt"
"io"
"io/fs"
"math/rand"
"os"
"os/user"
"path"
Expand Down Expand Up @@ -139,7 +138,7 @@ func (h *hdfsclient) Put(ctx context.Context, key string, in io.Reader, getters
if len(name) > 200 {
name = name[:200]
}
tmp = path.Join(path.Dir(p), fmt.Sprintf(".%s.tmp.%d", name, rand.Int()))
tmp = TmpFilePath(p, name)
defer func() {
if err != nil {
_ = h.c.Remove(tmp)
Expand Down
3 changes: 1 addition & 2 deletions pkg/object/nfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"context"
"fmt"
"io"
"math/rand"
"os"
"os/user"
"path"
Expand Down Expand Up @@ -176,7 +175,7 @@ func (n *nfsStore) Put(ctx context.Context, key string, in io.Reader, getters ..
if len(name) > 200 {
name = name[:200]
}
tmp = path.Join(path.Dir(p), fmt.Sprintf(".%s.tmp.%d", name, rand.Int()))
tmp = TmpFilePath(p, name)
defer func() {
if err != nil {
_ = n.target.Remove(tmp)
Expand Down
6 changes: 6 additions & 0 deletions pkg/object/object_storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@ import (
"context"
"fmt"
"io"
"math/rand"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
Expand Down Expand Up @@ -315,3 +317,7 @@ func decodeKey(value string, typ *string) (string, error) {
}
return value, nil
}

func TmpFilePath(parent, name string) string {
return filepath.Join(filepath.Dir(parent), ".jfs."+name+".tmp."+strconv.Itoa(rand.Int()))
}
3 changes: 1 addition & 2 deletions pkg/object/sftp.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"context"
"fmt"
"io"
"math/rand"
"net"
"net/url"
"os"
Expand Down Expand Up @@ -236,7 +235,7 @@ func (f *sftpStore) Put(ctx context.Context, key string, in io.Reader, getters .
if len(name) > 200 {
name = name[:200]
}
tmp = path.Join(path.Dir(p), fmt.Sprintf(".%s.tmp.%d", name, rand.Int()))
tmp = TmpFilePath(p, name)
defer func() {
if err != nil {
_ = c.sftpClient.Remove(tmp)
Expand Down
Loading