Support synchronizing symbolic link itself (#230)

Author: mstmdev

fs/fs.go

@@ -71,7 +71,7 @@ func IsNonEOF(err error) bool {
 
 // GetFileTime get the creation time, last access time, last modify time of the path
 func GetFileTime(path string) (cTime time.Time, aTime time.Time, mTime time.Time, err error) {
-	stat, err := os.Stat(path)
+	stat, err := os.Lstat(path)
 	if err != nil {
 		return
 	}
@@ -131,7 +131,7 @@ func IsSymlinkMode(mode fs.FileMode) bool {
 
 // IsSymlinkSupported checks if the system supports symbolic links
 func IsSymlinkSupported() bool {
-	symlink := filepath.Join(os.TempDir(), "symlink_detect.symlink")
+	symlink := filepath.Join(os.TempDir(), fmt.Sprintf("%d.symlink_detect", time.Now().UnixNano()))
 	defer os.RemoveAll(symlink)
 	return Symlink(os.Args[0], symlink) == nil
 }

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/gin-contrib/sessions v0.0.5
 	github.com/gin-gonic/gin v1.9.1
 	github.com/minio/minio-go/v7 v7.0.60
-	github.com/no-src/fsctl v0.1.1
+	github.com/no-src/fsctl v0.1.2-0.20230808162217-242297fbaea4
 	github.com/no-src/log v0.2.3
 	github.com/no-src/nscache v0.1.1
 	github.com/pkg/sftp v1.13.5

go.sum

@@ -126,8 +126,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/no-src/fsctl v0.1.1 h1:E4JTXDhASk6LtZLWFVpDCqVIw7DtU8IPDydamYi/wTQ=
-github.com/no-src/fsctl v0.1.1/go.mod h1:ObH5aEHwS6JLtV81Igj2BxciD5W/Tm40szqhm8W55q8=
+github.com/no-src/fsctl v0.1.2-0.20230808162217-242297fbaea4 h1:l4/bFYazcDMwBEogCeLwNRVxGtmNYBwLZEoJRqOxdf0=
+github.com/no-src/fsctl v0.1.2-0.20230808162217-242297fbaea4/go.mod h1:sXkup5MSxKl0br3AbtkoOhaGD0QSuieO8J3pExKegVs=
 github.com/no-src/log v0.2.3 h1:lCJrWeMMuC2Wkp6CKEVylgllH+k4YAm0eRsFX9nVxaQ=
 github.com/no-src/log v0.2.3/go.mod h1:C58ahv0WdsBL3Yxi+v6wyUXzwqk7aEPN+/1ltOsdHXg=
 github.com/no-src/nscache v0.1.1 h1:gs9hJ4Y5CaeLRmexvE4Rto6Gnjf5maKoTxFnssXAuJE=

integration/testdata/test/test-gofs-local-disk.yaml

@@ -34,6 +34,20 @@ actions:
     source: ./source/empty2
     no-newline: true
   - run: echo "Bye Bye"
+  - symlink:
+    link: ./source/hello.symlink
+    dest: ./source/hello
+    ignore-error: false
+  - symlink:
+    link: ./source/hello.symlink.bak
+    dest: ./source/hello
+    ignore-error: false
+  - symlink:
+    link: ./source/not_exist.symlink
+    dest: ./source/not_exist
+    ignore-error: false
+  - rm:
+    source: ./source/hello.symlink.bak
   - sleep: 10s
   - is-equal:
     source: ./integration_test.go
@@ -75,6 +89,17 @@ actions:
     algorithm: sha1
     source: ./source/hello
     expect: f343874b5df87e887d85df2e790df33584463162
+  - is-symlink:
+    link: ./dest/hello.symlink
+    expect: true
+    ignore-error: false
+  - is-symlink:
+    link: ./dest/not_exist.symlink
+    expect: true
+    ignore-error: false
+  - is-exist:
+    source: ./dest/hello.symlink.bak
+    expect: false
 clear:
   - rm:
     source: ./source

integration/testdata/test/test-gofs-minio-push.yaml

@@ -30,6 +30,20 @@ actions:
   - echo:
     source: ./minio-push-client/source/empty2
     no-newline: true
+  - symlink:
+    link: ./minio-push-client/source/hello.symlink
+    dest: ./minio-push-client/source/hello
+    ignore-error: false
+  - symlink:
+    link: ./minio-push-client/source/hello.symlink.bak
+    dest: ./minio-push-client/source/hello
+    ignore-error: false
+  - symlink:
+    link: ./minio-push-client/source/not_exist.symlink
+    dest: ./minio-push-client/source/not_exist
+    ignore-error: false
+  - rm:
+    source: ./minio-push-client/source/hello.symlink.bak
   - sleep: 10s
   - is-equal:
     source: ./minio-push-client/source/integration_test.go.bak1
@@ -81,6 +95,15 @@ actions:
   - is-exist:
     source: ./minio-data-mount/integration_test.go.bak1
     expect: false
+  - is-empty:
+    source: ./minio-data-mount/hello.symlink
+    expect: false
+  - is-empty:
+    source: ./minio-data-mount/not_exist.symlink
+    expect: false
+  - is-exist:
+    source: ./minio-data-mount/hello.symlink.bak
+    expect: false
 clear:
   - rm:
     source: ./minio-push-client

integration/testdata/test/test-gofs-remote-disk.yaml

@@ -42,6 +42,20 @@ actions:
   - echo:
     source: ./rs/source/empty2
     no-newline: true
+  - symlink:
+    link: ./rs/source/hello.symlink
+    dest: ./rs/source/hello
+    ignore-error: false
+  - symlink:
+    link: ./rs/source/hello.symlink.bak
+    dest: ./rs/source/hello
+    ignore-error: false
+  - symlink:
+    link: ./rs/source/not_exist.symlink
+    dest: ./rs/source/not_exist
+    ignore-error: false
+  - rm:
+    source: ./rs/source/hello.symlink.bak
   - sleep: 10s
   - hash:
     algorithm: md5
@@ -90,6 +104,17 @@ actions:
     algorithm: sha1
     source: ./rc/dest/hello
     expect: f343874b5df87e887d85df2e790df33584463162
+  - is-symlink:
+    link: ./rc/dest/hello.symlink
+    expect: true
+    ignore-error: false
+  - is-symlink:
+    link: ./rc/dest/not_exist.symlink
+    expect: true
+    ignore-error: false
+  - is-exist:
+    source: ./rc/dest/hello.symlink.bak
+    expect: false
 clear:
   - rm:
     source: ./rs

integration/testdata/test/test-gofs-remote-push.yaml

@@ -42,6 +42,20 @@ actions:
   - echo:
     source: ./rc-push/source/empty2
     no-newline: true
+  - symlink:
+    link: ./rc-push/source/hello.symlink
+    dest: ./rc-push/source/hello
+    ignore-error: false
+  - symlink:
+    link: ./rc-push/source/hello.symlink.bak
+    dest: ./rc-push/source/hello
+    ignore-error: false
+  - symlink:
+    link: ./rc-push/source/not_exist.symlink
+    dest: ./rc-push/source/not_exist
+    ignore-error: false
+  - rm:
+    source: ./rc-push/source/hello.symlink.bak
   - sleep: 10s
   - hash:
     algorithm: md5
@@ -96,6 +110,17 @@ actions:
   - is-exist:
     source: ./rs-push/source/integration_test.go.bak1
     expect: false
+  - is-symlink:
+    link: ./rs-push/source/hello.symlink
+    expect: true
+    ignore-error: false
+  - is-symlink:
+    link: ./rs-push/source/not_exist.symlink
+    expect: true
+    ignore-error: false
+  - is-exist:
+    source: ./rs-push/source/hello.symlink.bak
+    expect: false
 clear:
   - rm:
     source: ./rs-push

monitor/fsnotify_monitor.go

@@ -10,6 +10,7 @@ import (
 	"github.com/fsnotify/fsnotify"
 	"github.com/no-src/gofs/core"
 	"github.com/no-src/gofs/eventlog"
+	nsfs "github.com/no-src/gofs/fs"
 	"github.com/no-src/gofs/ignore"
 	"github.com/no-src/gofs/internal/clist"
 	"github.com/no-src/gofs/report"
@@ -167,9 +168,13 @@ func (m *fsNotifyMonitor) startProcessEvents() error {
 			// the file "info.log" does not match the ignore rule and should be synchronized to destination directory.
 			m.monitorDirIfCreate(event)
 		} else if event.Op&fsnotify.Write == fsnotify.Write {
-			m.write(event)
+			symlink, err := nsfs.IsSymlink(event.Name)
+			log.ErrorIf(err, "[write] check is symlink error =>%s", event.Name)
+			if err == nil && !symlink {
+				m.write(event)
+			}
 		} else if event.Op&fsnotify.Create == fsnotify.Create {
-			m.create(event)
+			m.symlinkOrCreate(event)
 		} else if event.Op&fsnotify.Remove == fsnotify.Remove {
 			m.remove(event)
 		} else if event.Op&fsnotify.Rename == fsnotify.Rename {
@@ -212,7 +217,7 @@ func (m *fsNotifyMonitor) create(event fsnotify.Event) {
 			// rename a file will not trigger the Write event
 			// rename a dir will not trigger the Write event on Linux and some Windows environments
 			// in some cases it will trigger the Write event for the parent dir on Windows
-			// send a Write event manually
+			// send a Write event manually if the file is not a symbolic link
 			log.Debug("prepare to send a [write] event after [create] event [%s]", event.Name)
 			m.events.PushBack(fsnotify.Event{
 				Name: event.Name,
@@ -222,6 +227,28 @@ func (m *fsNotifyMonitor) create(event fsnotify.Event) {
 	}
 }
 
+func (m *fsNotifyMonitor) symlink(event fsnotify.Event) {
+	source, err := nsfs.Readlink(event.Name)
+	if err != nil {
+		log.Error(err, "[symlink] read link error => [%s]", event.Name)
+		return
+	}
+	log.ErrorIf(m.syncer.Symlink(source, event.Name), "[symlink] event execute error => [%s]", event.Name)
+}
+
+func (m *fsNotifyMonitor) symlinkOrCreate(event fsnotify.Event) {
+	symlink, err := nsfs.IsSymlink(event.Name)
+	if err != nil {
+		log.Error(err, "[create] check is symlink error =>%s", event.Name)
+		return
+	}
+	if symlink {
+		m.symlink(event)
+	} else {
+		m.create(event)
+	}
+}
+
 func (m *fsNotifyMonitor) remove(event fsnotify.Event) {
 	m.removeWrite(event.Name)
 	log.ErrorIf(m.syncer.Remove(event.Name), "[remove] event execute error => [%s]", event.Name)

monitor/remote_client_monitor.go

@@ -209,6 +209,8 @@ func (m *remoteClientMonitor) execSync(msg *monitor.MonitorMessage) (err error)
 	switch action.Action(msg.Action) {
 	case action.CreateAction:
 		err = m.syncer.Create(path)
+	case action.SymlinkAction:
+		err = m.syncer.Symlink(fi.LinkTo, path)
 	case action.WriteAction:
 		err = m.syncer.Create(path)
 		// ignore is not exist error

server/README.md

@@ -61,6 +61,7 @@ Response field description:
     - `c_time` file creation time
     - `a_time` file last access time
     - `m_time` file last modify time
+    - `link_to` link to the real file
 
 ##### Example
 
@@ -128,7 +129,8 @@ Here is an example response:
       ],
       "c_time": 1649431872,
       "a_time": 1649431873,
-      "m_time": 1647397031
+      "m_time": 1647397031,
+      "link_to": ""
     },
     {
       "path": "hello_gofs.txt",
@@ -143,7 +145,8 @@ Here is an example response:
       ],
       "c_time": 1649431542,
       "a_time": 1649434237,
-      "m_time": 1649434237
+      "m_time": 1649434237,
+      "link_to": ""
     },
     {
       "path": "resource",
@@ -153,7 +156,8 @@ Here is an example response:
       "hash_values": null,
       "c_time": 1649431669,
       "a_time": 1649431898,
-      "m_time": 1649431898
+      "m_time": 1649431898,
+      "link_to": ""
     }
   ]
 }
@@ -174,7 +178,7 @@ Push the file changes to the [Remote Push Server](/README.md#remote-push-server)
 Request field description:
 
 - `push_data` the request data of push api, contains basic push file info and chunk info etc
-    - `action` the action of file change, Create(1) Write(2) Remove(3) Rename(4) Chmod(5)
+    - `action` the action of file change, Create(1) Write(2) Remove(3) Rename(4) Chmod(5) Symlink(6)
     - `push_action` the file upload action, CompareFile(1) CompareChunk(2) CompareFileAndChunk(3) Write(4) Truncate(5)
     - `file_info` basic push file info
         - `path` file path
@@ -184,6 +188,7 @@ Request field description:
         - `c_time` file creation time
         - `a_time` file last access time
         - `m_time` file last modify time
+        - `link_to` link to the real file
     - `chunk`
         - `offset` the offset relative to the origin of the file
         - `size` file chunk size of bytes, directory is always `0`
@@ -208,7 +213,7 @@ Accept-Encoding: gzip
 --d9e3eb63103de1c2698b0675a70567e47bed1b06c71ff9e26199967312d1
 Content-Disposition: form-data; name="push_data"
 
-{"action":2,"push_action":4,"file_info":{"path":"hello_gofs.txt","is_dir":0,"size":5,"hash":"5d41402abc4b2a76b9719d911017c592","hash_values":null,"c_time":1651681421,"a_time":1651681421,"m_time":1651681421},"chunk":{"offset":0,"hash":"5d41402abc4b2a76b9719d911017c592","size":5},"force_checksum":false}
+{"action":2,"push_action":4,"file_info":{"path":"hello_gofs.txt","is_dir":0,"size":5,"hash":"5d41402abc4b2a76b9719d911017c592","hash_values":null,"c_time":1651681421,"a_time":1651681421,"m_time":1651681421,"link_to":""},"chunk":{"offset":0,"hash":"5d41402abc4b2a76b9719d911017c592","size":5},"force_checksum":false}
 --d9e3eb63103de1c2698b0675a70567e47bed1b06c71ff9e26199967312d1
 Content-Disposition: form-data; name="up_file"; filename="hello_gofs.txt"
 Content-Type: application/octet-stream