Skip to content

feat: Uniform compression method #7581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ssongliu merged 1 commit into from
Dec 27, 2024
Merged

feat: Uniform compression method #7581

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions agent/app/service/backup_app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ func handleAppBackup(install *model.AppInstall, parentTask *task.Task, backupDir
}

appPath := install.GetPath()
if err := handleTar(appPath, tmpDir, "app.tar.gz", excludes, ""); err != nil {
if err := fileOp.TarGzCompressPro(true, appPath, path.Join(tmpDir, "app.tar.gz"), "", excludes); err != nil {
return err
}

Expand All @@ -170,7 +170,7 @@ func handleAppBackup(install *model.AppInstall, parentTask *task.Task, backupDir
}
}
t.LogStart(i18n.GetMsgByKey("CompressDir"))
if err := handleTar(tmpDir, backupDir, fileName, "", secret); err != nil {
if err := fileOp.TarGzCompressPro(true, tmpDir, path.Join(backupDir, fileName), secret, ""); err != nil {
return err
}
t.Log(i18n.GetWithName("CompressFileSuccess", fileName))
Expand Down Expand Up @@ -200,7 +200,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF

recoverApp := func(t *task.Task) error {
fileOp := files.NewFileOp()
if err := handleUnTar(recoverFile, path.Dir(recoverFile), secret); err != nil {
if err := fileOp.TarGzExtractPro(recoverFile, path.Dir(recoverFile), secret); err != nil {
return err
}
tmpPath := strings.ReplaceAll(recoverFile, ".tar.gz", "")
Expand Down Expand Up @@ -307,7 +307,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF

deCompressName := i18n.GetWithName("DeCompressFile", "app.tar.gz")
t.LogStart(deCompressName)
if err := handleUnTar(tmpPath+"/app.tar.gz", install.GetAppPath(), ""); err != nil {
if err := fileOp.TarGzExtractPro(tmpPath+"/app.tar.gz", install.GetAppPath(), ""); err != nil {
t.LogFailedWithErr(deCompressName, err)
_ = fileOp.DeleteDir(appDir)
_ = fileOp.Rename(backPath, appDir)
Expand Down
7 changes: 4 additions & 3 deletions agent/app/service/backup_mysql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,13 @@ func (u *BackupService) MysqlRecoverByUpload(req dto.CommonRecover) error {
if strings.HasSuffix(fileName, ".tar.gz") {
fileNameItem := time.Now().Format(constant.DateTimeSlimLayout)
dstDir := fmt.Sprintf("%s/%s", path.Dir(req.File), fileNameItem)
if _, err := os.Stat(dstDir); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(dstDir, os.ModePerm); err != nil {
fileOp := files.NewFileOp()
if !fileOp.Stat(dstDir) {
if err := fileOp.CreateDir(dstDir, os.ModePerm); err != nil {
return fmt.Errorf("mkdir %s failed, err: %v", dstDir, err)
}
}
if err := handleUnTar(req.File, dstDir, ""); err != nil {
if err := fileOp.TarGzExtractPro(req.File, dstDir, ""); err != nil {
_ = os.RemoveAll(dstDir)
return err
}
Expand Down
7 changes: 4 additions & 3 deletions agent/app/service/backup_postgresql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,13 @@ func (u *BackupService) PostgresqlRecoverByUpload(req dto.CommonRecover) error {
if strings.HasSuffix(fileName, ".tar.gz") {
fileNameItem := time.Now().Format(constant.DateTimeSlimLayout)
dstDir := fmt.Sprintf("%s/%s", path.Dir(req.File), fileNameItem)
if _, err := os.Stat(dstDir); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(dstDir, os.ModePerm); err != nil {
fileOp := files.NewFileOp()
if !fileOp.Stat(dstDir) {
if err := fileOp.CreateDir(dstDir, os.ModePerm); err != nil {
return fmt.Errorf("mkdir %s failed, err: %v", dstDir, err)
}
}
if err := handleUnTar(req.File, dstDir, ""); err != nil {
if err := fileOp.TarGzExtractPro(req.File, dstDir, ""); err != nil {
_ = os.RemoveAll(dstDir)
return err
}
Expand Down
4 changes: 2 additions & 2 deletions agent/app/service/backup_redis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func handleRedisBackup(redisInfo *repo.RootInfo, parentTask *task.Task, backupDi

if strings.HasSuffix(fileName, ".tar.gz") {
redisDataDir := fmt.Sprintf("%s/%s/%s/data/appendonlydir", constant.AppInstallDir, "redis", redisInfo.Name)
if err := handleTar(redisDataDir, backupDir, fileName, "", secret); err != nil {
if err := fileOp.TarGzCompressPro(true, redisDataDir, path.Join(backupDir, fileName), secret, ""); err != nil {
return err
}
return nil
Expand Down Expand Up @@ -201,7 +201,7 @@ func handleRedisRecover(redisInfo *repo.RootInfo, parentTask *task.Task, recover
}
if appendonly == "yes" && strings.HasPrefix(redisInfo.Version, "7.") {
redisDataDir := fmt.Sprintf("%s/%s/%s/data", constant.AppInstallDir, "redis", redisInfo.Name)
if err := handleUnTar(recoverFile, redisDataDir, secret); err != nil {
if err := fileOp.TarGzExtractPro(recoverFile, redisDataDir, secret); err != nil {
return err
}
} else {
Expand Down
8 changes: 4 additions & 4 deletions agent/app/service/backup_runtime.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ func handleRuntimeBackup(runtime *model.Runtime, backupDir, fileName string, exc
}

appPath := runtime.GetPath()
if err := handleTar(appPath, tmpDir, "runtime.tar.gz", excludes, secret); err != nil {
if err := fileOp.TarGzCompressPro(true, appPath, path.Join(tmpDir, "runtime.tar.gz"), secret, excludes); err != nil {
return err
}
if err := handleTar(tmpDir, backupDir, fileName, "", secret); err != nil {
if err := fileOp.TarGzCompressPro(true, tmpDir, path.Join(backupDir, fileName), secret, ""); err != nil {
return err
}
return nil
Expand All @@ -47,7 +47,7 @@ func handleRuntimeBackup(runtime *model.Runtime, backupDir, fileName string, exc
func handleRuntimeRecover(runtime *model.Runtime, recoverFile string, isRollback bool, secret string) error {
isOk := false
fileOp := files.NewFileOp()
if err := handleUnTar(recoverFile, path.Dir(recoverFile), secret); err != nil {
if err := fileOp.TarGzExtractPro(recoverFile, path.Dir(recoverFile), secret); err != nil {
return err
}
tmpPath := strings.ReplaceAll(recoverFile, ".tar.gz", "")
Expand Down Expand Up @@ -100,7 +100,7 @@ func handleRuntimeRecover(runtime *model.Runtime, recoverFile string, isRollback
_ = fileOp.Rename(runtimeDir, backPath)
_ = fileOp.CreateDir(runtimeDir, constant.DirPerm)

if err := handleUnTar(tmpPath+"/runtime.tar.gz", fmt.Sprintf("%s/%s", constant.RuntimeDir, runtime.Type), secret); err != nil {
if err := fileOp.TarGzExtractPro(tmpPath+"/runtime.tar.gz", fmt.Sprintf("%s/%s", constant.RuntimeDir, runtime.Type), secret); err != nil {
global.LOG.Errorf("handle recover from runtime.tar.gz failed, err: %v", err)
_ = fileOp.DeleteDir(runtimeDir)
_ = fileOp.Rename(backPath, runtimeDir)
Expand Down
8 changes: 4 additions & 4 deletions agent/app/service/backup_website.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func handleWebsiteRecover(website *model.Website, recoverFile string, isRollback
fileOp := files.NewFileOp()
tmpPath := strings.ReplaceAll(recoverFile, ".tar.gz", "")
t.Log(i18n.GetWithName("DeCompressFile", recoverFile))
if err = handleUnTar(recoverFile, path.Dir(recoverFile), secret); err != nil {
if err = fileOp.TarGzExtractPro(recoverFile, path.Dir(recoverFile), secret); err != nil {
return err
}
defer func() {
Expand Down Expand Up @@ -182,7 +182,7 @@ func handleWebsiteRecover(website *model.Website, recoverFile string, isRollback
}
taskName := i18n.GetMsgByKey("TaskRecover") + i18n.GetMsgByKey("websiteDir")
t.Log(taskName)
if err = handleUnTar(fmt.Sprintf("%s/%s.web.tar.gz", tmpPath, website.Alias), GetSitePath(*website, SiteDir), ""); err != nil {
if err = fileOp.TarGzExtractPro(fmt.Sprintf("%s/%s.web.tar.gz", tmpPath, website.Alias), GetSitePath(*website, SiteDir), ""); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}
Expand Down Expand Up @@ -263,10 +263,10 @@ func handleWebsiteBackup(website *model.Website, backupDir, fileName, excludes,

websiteDir := GetSitePath(*website, SiteDir)
t.LogStart(i18n.GetMsgByKey("CompressDir"))
if err = handleTar(websiteDir, tmpDir, fmt.Sprintf("%s.web.tar.gz", website.Alias), excludes, ""); err != nil {
if err = fileOp.TarGzCompressPro(true, websiteDir, path.Join(tmpDir, fmt.Sprintf("%s.web.tar.gz", website.Alias)), "", excludes); err != nil {
return err
}
if err = handleTar(tmpDir, backupDir, fileName, "", secret); err != nil {
if err = fileOp.TarGzCompressPro(true, tmpDir, path.Join(backupDir, fileName), secret, ""); err != nil {
return err
}
t.Log(i18n.GetWithName("CompressFileSuccess", fileName))
Expand Down
11 changes: 6 additions & 5 deletions agent/app/service/cronjob_backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,7 @@ func loadWebsForJob(cronjob model.Cronjob) []model.Website {
}

func handleBackupLogs(targetDir, fileName string, secret string) error {
fileOp := files.NewFileOp()
websites, err := websiteRepo.List()
if err != nil {
return err
Expand All @@ -338,7 +339,7 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
if len(logFiles) != 0 {
for i := 0; i < len(logFiles); i++ {
if !logFiles[i].IsDir() {
_ = common.CopyFile(path.Join(itemDir, logFiles[i].Name()), dirItem)
_ = fileOp.CopyFile(path.Join(itemDir, logFiles[i].Name()), dirItem)
}
}
}
Expand All @@ -347,7 +348,7 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
if len(logFiles2) != 0 {
for i := 0; i < len(logFiles2); i++ {
if !logFiles2[i].IsDir() {
_ = common.CopyFile(path.Join(itemDir2, logFiles2[i].Name()), dirItem)
_ = fileOp.CopyFile(path.Join(itemDir2, logFiles2[i].Name()), dirItem)
}
}
}
Expand All @@ -366,7 +367,7 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
if len(systemLogFiles) != 0 {
for i := 0; i < len(systemLogFiles); i++ {
if !systemLogFiles[i].IsDir() {
_ = common.CopyFile(path.Join(systemLogDir, systemLogFiles[i].Name()), systemDir)
_ = fileOp.CopyFile(path.Join(systemLogDir, systemLogFiles[i].Name()), systemDir)
}
}
}
Expand All @@ -382,13 +383,13 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
if len(loginLogFiles) != 0 {
for i := 0; i < len(loginLogFiles); i++ {
if !loginLogFiles[i].IsDir() && (strings.HasPrefix(loginLogFiles[i].Name(), "secure") || strings.HasPrefix(loginLogFiles[i].Name(), "auth.log")) {
_ = common.CopyFile(path.Join("/var/log", loginLogFiles[i].Name()), loginDir)
_ = fileOp.CopyFile(path.Join("/var/log", loginLogFiles[i].Name()), loginDir)
}
}
}
global.LOG.Debug("backup ssh log successful!")

if err := handleTar(targetDir, path.Dir(targetDir), fileName, "", secret); err != nil {
if err := fileOp.TarGzCompressPro(true, targetDir, path.Join(path.Dir(targetDir), fileName), secret, ""); err != nil {
return err
}
defer func() {
Expand Down
76 changes: 0 additions & 76 deletions agent/app/service/cronjob_helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
"github.com/1Panel-dev/1Panel/agent/utils/files"
"github.com/1Panel-dev/1Panel/agent/utils/ntp"
"github.com/pkg/errors"
)

func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
Expand Down Expand Up @@ -140,81 +139,6 @@ func (u *CronjobService) handleNtpSync() error {
return nil
}

func handleTar(sourceDir, targetDir, name, exclusionRules string, secret string) error {
if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(targetDir, os.ModePerm); err != nil {
return err
}
}

excludes := strings.Split(exclusionRules, ",")
excludeRules := ""
for _, exclude := range excludes {
if len(exclude) == 0 {
continue
}
excludeRules += " --exclude " + exclude
}
path := ""
if strings.Contains(sourceDir, "/") {
itemDir := strings.ReplaceAll(sourceDir[strings.LastIndex(sourceDir, "/"):], "/", "")
aheadDir := sourceDir[:strings.LastIndex(sourceDir, "/")]
if len(aheadDir) == 0 {
aheadDir = "/"
}
path += fmt.Sprintf("-C %s %s", aheadDir, itemDir)
} else {
path = sourceDir
}

commands := ""

if len(secret) != 0 {
extraCmd := "| openssl enc -aes-256-cbc -salt -k '" + secret + "' -out"
commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read --exclude-from=<(find %s -type s -print) -zcf %s %s %s %s", sourceDir, " -"+excludeRules, path, extraCmd, targetDir+"/"+name)
global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******"))
} else {
itemPrefix := pathUtils.Base(sourceDir)
if itemPrefix == "/" {
itemPrefix = ""
}
commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read --exclude-from=<(find %s -type s -printf '%s' | sed 's|^|%s/|') -zcf %s %s %s", sourceDir, "%P\n", itemPrefix, targetDir+"/"+name, excludeRules, path)
global.LOG.Debug(commands)
}
stdout, err := cmd.ExecWithTimeOut(commands, 24*time.Hour)
if err != nil {
if len(stdout) != 0 {
global.LOG.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err)
return fmt.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err)
}
}
return nil
}

func handleUnTar(sourceFile, targetDir string, secret string) error {
if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(targetDir, os.ModePerm); err != nil {
return err
}
}
commands := ""
if len(secret) != 0 {
extraCmd := "openssl enc -d -aes-256-cbc -k '" + secret + "' -in " + sourceFile + " | "
commands = fmt.Sprintf("%s tar -zxvf - -C %s", extraCmd, targetDir+" > /dev/null 2>&1")
global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******"))
} else {
commands = fmt.Sprintf("tar zxvfC %s %s", sourceFile, targetDir)
global.LOG.Debug(commands)
}

stdout, err := cmd.ExecWithTimeOut(commands, 24*time.Hour)
if err != nil {
global.LOG.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err)
return errors.New(stdout)
}
return nil
}

func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, startTime time.Time) ([]string, string, error) {
var (
err error
Copy link
Member

@wanghe-fit2cloud wanghe-fit2cloud Dec 27, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are no clear or significant code differences presented in your sample text. The provided code seems to be correctly structured and functionally correct based on its purpose, which is not specified at this point. However, since there's nothing wrong with it according to current knowledge cutoff dates of 2021-09-01 and up-to-date information, we would consider the code itself functional at that time but please refer to updated specifications to validate for accuracy regarding changes and improvements.

Expand Down
12 changes: 6 additions & 6 deletions agent/app/service/snapshot_create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,38 +248,38 @@ func snapBaseData(snap snapHelper, targetDir string) error {
snap.Task.Log("---------------------- 2 / 8 ----------------------")
snap.Task.LogStart(i18n.GetMsgByKey("SnapBaseInfo"))

err := common.CopyFile("/usr/local/bin/1panel-core", targetDir)
err := snap.FileOp.CopyFile("/usr/local/bin/1panel-core", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel-core"), err)
if err != nil {
return err
}

err = common.CopyFile("/usr/local/bin/1panel-agent", targetDir)
err = snap.FileOp.CopyFile("/usr/local/bin/1panel-agent", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel-agent"), err)
if err != nil {
return err
}

err = common.CopyFile("/usr/local/bin/1pctl", targetDir)
err = snap.FileOp.CopyFile("/usr/local/bin/1pctl", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err)
if err != nil {
return err
}

err = common.CopyFile("/etc/systemd/system/1panel.service", targetDir)
err = snap.FileOp.CopyFile("/etc/systemd/system/1panel.service", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err)
if err != nil {
return err
}

err = common.CopyFile("/etc/systemd/system/1panel-agent.service", targetDir)
err = snap.FileOp.CopyFile("/etc/systemd/system/1panel-agent.service", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel-agent.service"), err)
if err != nil {
return err
}

if snap.FileOp.Stat("/etc/docker/daemon.json") {
err = common.CopyFile("/etc/docker/daemon.json", targetDir)
err = snap.FileOp.CopyFile("/etc/docker/daemon.json", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/docker/daemon.json"), err)
if err != nil {
return err
Expand Down
14 changes: 7 additions & 7 deletions agent/init/viper/viper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/1Panel-dev/1Panel/agent/cmd/server/conf"
"github.com/1Panel-dev/1Panel/agent/configs"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/common"
"github.com/1Panel-dev/1Panel/agent/utils/files"
"github.com/1Panel-dev/1Panel/agent/utils/xpack"
"github.com/fsnotify/fsnotify"
Expand Down Expand Up @@ -65,10 +64,11 @@ func initDir() {
}
global.CONF.System.BaseDir = nodeInfo.BaseDir

_, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.BaseDir, "1panel/docker/compose/"))
global.CONF.System.DataDir, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.BaseDir, "1panel"))
global.CONF.System.Cache, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.DataDir, "cache"))
global.CONF.System.DbPath, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.DataDir, "db"))
global.CONF.System.LogPath, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.DataDir, "log"))
global.CONF.System.TmpDir, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.DataDir, "tmp"))
fileOp := files.NewFileOp()
_, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.BaseDir, "1panel/docker/compose/"))
global.CONF.System.DataDir, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.BaseDir, "1panel"))
global.CONF.System.Cache, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.DataDir, "cache"))
global.CONF.System.DbPath, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.DataDir, "db"))
global.CONF.System.LogPath, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.DataDir, "log"))
global.CONF.System.TmpDir, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.DataDir, "tmp"))
}
Loading
Loading