Merge pull request #267 from wttech/vault-cli

Push command, Vault-Cli wrapper, improvements

this PR main feature is to introduce working `content push`  command.

the Vault CLI is also added here but in 'incubating' mode. so it could be removed in the future if there will be no use case found. it is here because it was considered as a transport mode for content pull/push but after all it's not so effective.

we decided to keep it in AEMC to leverage auto-install capability that may become useful sometimes if sb will  want to use `vlt rcp` command or similar

Author: krystian-panek-vmltech

README.md

@@ -471,6 +471,9 @@ java:
       "arm64": "x64"
       "aarch64": "x64"
 
+vault:
+  "download_url": "https://repo1.maven.org/maven2/org/apache/jackrabbit/vault/vault-cli/3.7.2/vault-cli-3.7.2-bin.tar.gz"
+
 base:
   # Location of temporary files (downloaded AEM packages, etc)
   tmp_dir: aem/home/tmp
@@ -493,6 +496,52 @@ output:
     file: aem/home/var/log/aem.log
     # Controls where outputs and logs should be written to when format is 'text' (console|file|both)
     mode: console
+
+# Content clean options
+content:
+  clean:
+    # File patterns to be deleted
+    files_deleted:
+      - patterns:
+          - "**/.vlt"
+          - "**/.vlt*.tmp"
+          - "**/install/*.jar"
+    # File patterns to be flattened
+    files_flattened:
+      - "**/_cq_design_dialog/.content.xml"
+      - "**/_cq_dialog/.content.xml"
+      - "**/_cq_htmlTag/.content.xml"
+      - "**/_cq_template/.content.xml"
+    # Property patterns to be skipped, removed from cleaned file
+    properties_skipped:
+      - patterns: "jcr:uuid"
+        excluded_paths: [ "**/home/users/*", "**/home/groups/*" ]
+      - patterns: "cq:lastModified*"
+        excluded_paths: [ "**/content/experience-fragments/*" ]
+      - patterns: [ "dam:sha1", "dam:size" ]
+        included_paths: [ "**/content/dam/*.svg/*" ]
+      - patterns:
+          - "jcr:lastModified*"
+          - "jcr:created*"
+          - "jcr:isCheckedOut"
+          - "cq:lastReplicat*"
+          - "cq:lastRolledout*"
+          - "dam:extracted"
+          - "dam:assetState"
+          - "dc:modified"
+          - "*_x0040_*"
+          - "cq:name"
+          - "cq:parentPath"
+          - "dam:copiedAt"
+          - "dam:parentAssetID"
+          - "dam:relativePath"
+    # Mixin type patterns to be skipped, removed from cleaned file
+    mixin_types_skipped:
+      - patterns:
+          - "cq:ReplicationStatus"
+          - "mix:versionable"
+    # Unused namespaces to be skipped, removed from cleaned file
+    namespaces_skipped: true
 ```
 
 Note that environment variables may be injected in any part of config file. 

cmd/aem/content.go

@@ -5,7 +5,10 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/wttech/aemc/pkg"
 	"github.com/wttech/aemc/pkg/common/pathx"
+	"github.com/wttech/aemc/pkg/common/timex"
 	"github.com/wttech/aemc/pkg/content"
+	"os"
+	"path/filepath"
 	"strings"
 )
 
@@ -17,6 +20,7 @@ func (c *CLI) contentCmd() *cobra.Command {
 	}
 	cmd.AddCommand(c.contentCleanCmd())
 	cmd.AddCommand(c.contentPullCmd())
+	cmd.AddCommand(c.contentPushCmd())
 	cmd.AddCommand(c.contentDownloadCmd())
 	cmd.AddCommand(c.contentCopyCmd())
 	return cmd
@@ -38,17 +42,17 @@ func (c *CLI) contentCleanCmd() *cobra.Command {
 				c.Error(err)
 				return
 			}
+			path := dir
+			if path == "" {
+				path = file
+			}
+			if err = c.aem.ContentManager().Clean(path); err != nil {
+				c.Error(err)
+				return
+			}
 			if dir != "" {
-				if err = c.aem.ContentManager().CleanDir(dir); err != nil {
-					c.Error(err)
-					return
-				}
 				c.SetOutput("dir", dir)
 			} else if file != "" {
-				if err = c.aem.ContentManager().CleanFile(file); err != nil {
-					c.Error(err)
-					return
-				}
 				c.SetOutput("file", file)
 			}
 			c.Changed("content cleaned")
@@ -73,10 +77,14 @@ func (c *CLI) contentDownloadCmd() *cobra.Command {
 				return
 			}
 			pid, _ := cmd.Flags().GetString("pid")
+			if pid == "" {
+				pid = fmt.Sprintf("aemc:content-download:%s-SNAPSHOT", timex.FileTimestampForNow())
+			}
 			targetFile, _ := cmd.Flags().GetString("target-file")
-			filterRoots, _ := cmd.Flags().GetStringSlice("filter-roots")
+			filterRoots := determineFilterRoots(cmd)
 			filterFile, _ := cmd.Flags().GetString("filter-file")
-			if err = instance.ContentManager().Download(targetFile, pkg.PackageCreateOpts{
+			clean, _ := cmd.Flags().GetBool("clean")
+			if err = c.aem.ContentManager().Download(instance, targetFile, clean, pkg.PackageCreateOpts{
 				PID:         pid,
 				FilterRoots: filterRoots,
 				FilterFile:  filterFile,
@@ -94,6 +102,7 @@ func (c *CLI) contentDownloadCmd() *cobra.Command {
 	cmd.Flags().StringSliceP("filter-roots", "r", []string{}, "Vault filter root paths")
 	cmd.Flags().StringP("filter-file", "f", "", "Vault filter file path")
 	cmd.MarkFlagsOneRequired("filter-roots", "filter-file")
+	cmd.Flags().Bool("clean", false, "Normalize content after downloading")
 	return cmd
 }
 
@@ -108,8 +117,6 @@ func (c *CLI) contentPullCmd() *cobra.Command {
 				c.Error(err)
 				return
 			}
-			clean, _ := cmd.Flags().GetBool("clean")
-			replace, _ := cmd.Flags().GetBool("replace")
 			dir, err := determineContentDir(cmd)
 			if err != nil {
 				c.Error(err)
@@ -120,21 +127,26 @@ func (c *CLI) contentPullCmd() *cobra.Command {
 				c.Error(err)
 				return
 			}
+			filterRoots := determineFilterRoots(cmd)
+			filterFile, _ := cmd.Flags().GetString("filter-file")
+			filterRootExcludes := determineFilterRootExcludes(cmd)
+			clean, _ := cmd.Flags().GetBool("clean")
+			replace, _ := cmd.Flags().GetBool("replace")
 			if dir != "" {
-				filterRoots, _ := cmd.Flags().GetStringSlice("filter-roots")
-				filterFile, _ := cmd.Flags().GetString("filter-file")
-				if err = instance.ContentManager().PullDir(dir, clean, replace, pkg.PackageCreateOpts{
+				if err = c.aem.ContentManager().PullDir(instance, dir, clean, replace, pkg.PackageCreateOpts{
+					PID:         fmt.Sprintf("aemc:content-pull:%s-SNAPSHOT", timex.FileTimestampForNow()),
 					FilterRoots: filterRoots,
 					FilterFile:  filterFile,
-					ContentDir:  dir,
 				}); err != nil {
 					c.Error(err)
 					return
 				}
 				c.SetOutput("dir", dir)
 			} else if file != "" {
-				if err = instance.ContentManager().PullFile(file, clean, pkg.PackageCreateOpts{
-					ContentFile: file,
+				if err = c.aem.ContentManager().PullFile(instance, file, clean, replace, pkg.PackageCreateOpts{
+					PID:                fmt.Sprintf("aemc:content-pull:%s-SNAPSHOT", timex.FileTimestampForNow()),
+					FilterRoots:        filterRoots,
+					FilterRootExcludes: filterRootExcludes,
 				}); err != nil {
 					c.Error(err)
 					return
@@ -156,6 +168,66 @@ func (c *CLI) contentPullCmd() *cobra.Command {
 	return cmd
 }
 
+func (c *CLI) contentPushCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "push",
+		Aliases: []string{"ps"},
+		Short:   "Push content from JCR root directory or local file to running instance",
+		Run: func(cmd *cobra.Command, args []string) {
+			instances, err := c.aem.InstanceManager().Some()
+			if err != nil {
+				c.Error(err)
+				return
+			}
+			dir, err := determineContentDir(cmd)
+			if err != nil {
+				c.Error(err)
+				return
+			}
+			file, err := determineContentFile(cmd)
+			if err != nil {
+				c.Error(err)
+				return
+			}
+			path := dir
+			if path == "" {
+				path = file
+			}
+			if !pathx.Exists(path) {
+				c.Error(fmt.Errorf("cannot push content as it does not exist '%s'", path))
+				return
+			}
+			filterRoots := determineFilterRoots(cmd)
+			filterRootExcludes := determineFilterRootExcludes(cmd)
+			clean, _ := cmd.Flags().GetBool("clean")
+			filterMode := determineFilterMode(cmd)
+			if err = c.aem.ContentManager().Push(instances, clean, pkg.PackageCreateOpts{
+				PID:                fmt.Sprintf("aemc:content-push:%s-SNAPSHOT", timex.FileTimestampForNow()),
+				FilterRoots:        filterRoots,
+				FilterRootExcludes: filterRootExcludes,
+				ContentPath:        path,
+				FilterMode:         filterMode,
+			}); err != nil {
+				c.Error(err)
+				return
+			}
+			if dir != "" {
+				c.SetOutput("dir", dir)
+			} else if file != "" {
+				c.SetOutput("file", file)
+			}
+			c.Changed("content pushed")
+		},
+	}
+	cmd.Flags().StringP("dir", "d", "", "JCR root path")
+	cmd.Flags().StringP("file", "f", "", "Local file path")
+	cmd.Flags().StringP("path", "p", "", "JCR root path or local file path")
+	cmd.MarkFlagsOneRequired("dir", "file", "path")
+	cmd.Flags().Bool("clean", false, "Normalize content while uploading")
+	cmd.Flags().Bool("update", false, "Existing content on running instance is updated, new content is added and none is deleted")
+	return cmd
+}
+
 func (c *CLI) contentCopyCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "copy",
@@ -167,15 +239,16 @@ func (c *CLI) contentCopyCmd() *cobra.Command {
 				c.Error(err)
 				return
 			}
-			targetInstance, err := determineContentTargetInstance(cmd, c.aem.InstanceManager())
+			targetInstances, err := determineContentTargetInstances(cmd, c.aem.InstanceManager())
 			if err != nil {
 				c.Error(err)
 				return
 			}
-			filterRoots, _ := cmd.Flags().GetStringSlice("filter-roots")
+			filterRoots := determineFilterRoots(cmd)
 			filterFile, _ := cmd.Flags().GetString("filter-file")
 			clean, _ := cmd.Flags().GetBool("clean")
-			if err = instance.ContentManager().Copy(targetInstance, clean, pkg.PackageCreateOpts{
+			if err = c.aem.ContentManager().Copy(instance, targetInstances, clean, pkg.PackageCreateOpts{
+				PID:         fmt.Sprintf("aemc:content-copy:%s-SNAPSHOT", timex.FileTimestampForNow()),
 				FilterRoots: filterRoots,
 				FilterFile:  filterFile,
 			}); err != nil {
@@ -185,8 +258,8 @@ func (c *CLI) contentCopyCmd() *cobra.Command {
 			c.Changed("content copied")
 		},
 	}
-	cmd.Flags().StringP("instance-target-url", "u", "", "Destination instance URL")
-	cmd.Flags().StringP("instance-target-id", "i", "", "Destination instance ID")
+	cmd.Flags().StringSliceP("instance-target-url", "u", []string{}, "Destination instance URL")
+	cmd.Flags().StringSliceP("instance-target-id", "i", []string{}, "Destination instance ID")
 	cmd.MarkFlagsOneRequired("instance-target-url", "instance-target-id")
 	cmd.Flags().StringSliceP("filter-roots", "r", []string{}, "Vault filter root paths")
 	cmd.Flags().StringP("filter-file", "f", "", "Vault filter file path")
@@ -195,33 +268,41 @@ func (c *CLI) contentCopyCmd() *cobra.Command {
 	return cmd
 }
 
-func determineContentTargetInstance(cmd *cobra.Command, instanceManager *pkg.InstanceManager) (*pkg.Instance, error) {
-	var instance *pkg.Instance
-	url, _ := cmd.Flags().GetString("instance-target-url")
-	if url != "" {
-		instance, _ = instanceManager.NewByIDAndURL("remote_adhoc_target", url)
+func determineContentTargetInstances(cmd *cobra.Command, instanceManager *pkg.InstanceManager) ([]pkg.Instance, error) {
+	var instances []pkg.Instance
+	urls, _ := cmd.Flags().GetStringSlice("instance-target-url")
+	for _, url := range urls {
+		instance, err := instanceManager.NewByIDAndURL("remote_adhoc_target", url)
+		if err != nil {
+			return nil, err
+		}
+		instances = append(instances, *instance)
 	}
-	id, _ := cmd.Flags().GetString("instance-target-id")
-	if id != "" {
-		instance = instanceManager.NewByID(id)
+	ids, _ := cmd.Flags().GetStringSlice("instance-target-id")
+	for _, id := range ids {
+		instance := instanceManager.NewByID(id)
+		instances = append(instances, *instance)
 	}
-	if instance == nil {
+	if instances == nil {
 		return nil, fmt.Errorf("missing 'instance-target-url' or 'instance-target-id'")
 	}
-	return instance, nil
+	return instances, nil
 }
 
 func determineContentDir(cmd *cobra.Command) (string, error) {
 	dir, _ := cmd.Flags().GetString("dir")
 	if dir != "" && !strings.Contains(dir, content.JCRRoot) {
-		return "", fmt.Errorf("content dir '%s' does not contain '%s'", dir, content.JCRRoot)
+		return "", fmt.Errorf("content directory '%s' does not contain '%s'", dir, content.JCRRoot)
+	}
+	if pathx.IsFile(dir) {
+		return "", fmt.Errorf("content directory '%s' is not a directory; consider using 'file' parameter", dir)
 	}
 	path, _ := cmd.Flags().GetString("path")
 	if path != "" && !strings.Contains(path, content.JCRRoot) {
 		return "", fmt.Errorf("content path '%s' does not contain '%s'", path, content.JCRRoot)
 	}
 	if path != "" && !pathx.Exists(path) {
-		return "", fmt.Errorf("content path does not exist: %s", path)
+		return "", fmt.Errorf("content path '%s' need to exist on file system; consider using 'dir' or 'file' parameter otherwise", path)
 	}
 	if path != "" && pathx.IsDir(path) {
 		return path, nil
@@ -234,15 +315,69 @@ func determineContentFile(cmd *cobra.Command) (string, error) {
 	if file != "" && !strings.Contains(file, content.JCRRoot) {
 		return "", fmt.Errorf("content file '%s' does not contain '%s'", file, content.JCRRoot)
 	}
+	if pathx.IsDir(file) {
+		return "", fmt.Errorf("content file '%s' is not a file; consider using 'dir' parameter", file)
+	}
 	path, _ := cmd.Flags().GetString("path")
 	if path != "" && !strings.Contains(path, content.JCRRoot) {
 		return "", fmt.Errorf("content path '%s' does not contain '%s'", path, content.JCRRoot)
 	}
 	if path != "" && !pathx.Exists(path) {
-		return "", fmt.Errorf("content path does not exist: %s", path)
+		return "", fmt.Errorf("content path '%s' need to exist on file system; consider using 'dir' or 'file' parameter otherwise", path)
 	}
 	if path != "" && pathx.IsFile(path) {
 		return path, nil
 	}
 	return file, nil
 }
+
+func determineFilterRoots(cmd *cobra.Command) []string {
+	filterRoots, _ := cmd.Flags().GetStringSlice("filter-roots")
+	if len(filterRoots) > 0 {
+		return filterRoots
+	}
+	filterFile, _ := cmd.Flags().GetString("filter-file")
+	if filterFile != "" {
+		return nil
+	}
+	dir, _ := determineContentDir(cmd)
+	if dir != "" {
+		return []string{pkg.DetermineFilterRoot(dir)}
+	}
+	file, _ := determineContentFile(cmd)
+	if file != "" {
+		return []string{pkg.DetermineFilterRoot(file)}
+	}
+	return nil
+}
+
+func determineFilterRootExcludes(cmd *cobra.Command) []string {
+	file, _ := determineContentFile(cmd)
+	if file == "" || !strings.HasSuffix(file, content.JCRContentFile) || content.IsPageContentFile(file) {
+		return nil
+	}
+
+	dir := filepath.Dir(file)
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil
+	}
+
+	var filterRootExcludes []string
+	for _, entry := range entries {
+		if entry.Name() != content.JCRContentFile {
+			jcrPath := pkg.DetermineFilterRoot(filepath.Join(dir, entry.Name()))
+			excludePattern := fmt.Sprintf("%s(/.*)?", jcrPath)
+			filterRootExcludes = append(filterRootExcludes, excludePattern)
+		}
+	}
+	return filterRootExcludes
+}
+
+func determineFilterMode(cmd *cobra.Command) string {
+	update, _ := cmd.Flags().GetBool("update")
+	if update {
+		return "update"
+	}
+	return ""
+}