Skip to content

feat: 压缩和解压缩 tar.gz 文件时支持设置压缩密码 #5258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 4, 2024
Merged

feat: 压缩和解压缩 tar.gz 文件时支持设置压缩密码 #5258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5ddae12
feat: 压缩和解压缩 tar.gz 文件时支持设置压缩密码
john1298308460 Jun 3, 2024
89bad0f
fix: 解决快照加密备份失败的问题
Jun 3, 2024
30315a0
fix: 日志打印优化
john1298308460 Jun 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions backend/app/dto/backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,15 @@ type CommonBackup struct {
Type string `json:"type" validate:"required,oneof=app mysql mariadb redis website postgresql"`
Name string `json:"name"`
DetailName string `json:"detailName"`
Secret string `json:"secret"`
}
type CommonRecover struct {
Source string `json:"source" validate:"required,oneof=OSS S3 SFTP MINIO LOCAL COS KODO OneDrive WebDAV"`
Type string `json:"type" validate:"required,oneof=app mysql mariadb redis website postgresql"`
Name string `json:"name"`
DetailName string `json:"detailName"`
File string `json:"file"`
Secret string `json:"secret"`
}

type RecordSearch struct {
Expand Down
7 changes: 6 additions & 1 deletion backend/app/dto/cronjob.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package dto

import "time"
import (
"time"
)

type CronjobCreate struct {
Name string `json:"name" validate:"required"`
Expand All @@ -21,6 +23,7 @@ type CronjobCreate struct {
BackupAccounts string `json:"backupAccounts"`
DefaultDownload string `json:"defaultDownload"`
RetainCopies int `json:"retainCopies" validate:"number,min=1"`
Secret string `json:"secret"`
}

type CronjobUpdate struct {
Expand All @@ -42,6 +45,7 @@ type CronjobUpdate struct {
BackupAccounts string `json:"backupAccounts"`
DefaultDownload string `json:"defaultDownload"`
RetainCopies int `json:"retainCopies" validate:"number,min=1"`
Secret string `json:"secret"`
}

type CronjobUpdateStatus struct {
Expand Down Expand Up @@ -87,6 +91,7 @@ type CronjobInfo struct {

LastRecordTime string `json:"lastRecordTime"`
Status string `json:"status"`
Secret string `json:"secret"`
}

type SearchRecord struct {
Expand Down
8 changes: 5 additions & 3 deletions backend/app/dto/request/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,14 @@ type FileCompress struct {
Type string `json:"type" validate:"required"`
Name string `json:"name" validate:"required"`
Replace bool `json:"replace"`
Secret string `json:"secret"`
}

type FileDeCompress struct {
Dst string `json:"dst" validate:"required"`
Type string `json:"type" validate:"required"`
Path string `json:"path" validate:"required"`
Dst string `json:"dst" validate:"required"`
Type string `json:"type" validate:"required"`
Path string `json:"path" validate:"required"`
Secret string `json:"secret"`
}

type FileEdit struct {
Expand Down
8 changes: 5 additions & 3 deletions backend/app/dto/setting.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,11 +116,13 @@ type SnapshotCreate struct {
From string `json:"from" validate:"required"`
DefaultDownload string `json:"defaultDownload" validate:"required"`
Description string `json:"description" validate:"max=256"`
Secret string `json:"secret"`
}
type SnapshotRecover struct {
IsNew bool `json:"isNew"`
ReDownload bool `json:"reDownload"`
ID uint `json:"id" validate:"required"`
IsNew bool `json:"isNew"`
ReDownload bool `json:"reDownload"`
ID uint `json:"id" validate:"required"`
Secret string `json:"secret"`
}
type SnapshotBatchDelete struct {
DeleteWithFile bool `json:"deleteWithFile"`
Expand Down
5 changes: 4 additions & 1 deletion backend/app/model/cronjob.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package model

import "time"
import (
"time"
)

type Cronjob struct {
BaseModel
Expand Down Expand Up @@ -31,6 +33,7 @@ type Cronjob struct {
Status string `gorm:"type:varchar(64)" json:"status"`
EntryIDs string `gorm:"type:varchar(64)" json:"entryIDs"`
Records []JobRecords `json:"records"`
Secret string `gorm:"type:varchar(64)" json:"secret"`
}

type JobRecords struct {
Expand Down
2 changes: 1 addition & 1 deletion backend/app/service/app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -752,7 +752,7 @@ func getAppFromRepo(downloadPath string) error {
if err := fileOp.DownloadFile(downloadUrl, packagePath); err != nil {
return err
}
if err := fileOp.Decompress(packagePath, constant.ResourceDir, files.SdkZip); err != nil {
if err := fileOp.Decompress(packagePath, constant.ResourceDir, files.SdkZip, ""); err != nil {
return err
}
defer func() {
Expand Down
2 changes: 1 addition & 1 deletion backend/app/service/app_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -765,7 +765,7 @@ func downloadApp(app model.App, appDetail model.AppDetail, appInstall *model.App
global.LOG.Errorf("download app[%s] error %v", app.Name, err)
return
}
if err = fileOp.Decompress(filePath, appResourceDir, files.SdkTarGz); err != nil {
if err = fileOp.Decompress(filePath, appResourceDir, files.SdkTarGz, ""); err != nil {
global.LOG.Errorf("decompress app[%s] error %v", app.Name, err)
return
}
Expand Down
20 changes: 10 additions & 10 deletions backend/app/service/backup_app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func (u *BackupService) AppBackup(req dto.CommonBackup) (*model.BackupRecord, er
backupDir := path.Join(localDir, itemDir)

fileName := fmt.Sprintf("%s_%s.tar.gz", req.DetailName, timeNow+common.RandStrAndNum(5))
if err := handleAppBackup(&install, backupDir, fileName, ""); err != nil {
if err := handleAppBackup(&install, backupDir, fileName, "", req.Secret); err != nil {
return nil, err
}

Expand Down Expand Up @@ -78,13 +78,13 @@ func (u *BackupService) AppRecover(req dto.CommonRecover) error {
if _, err := compose.Down(install.GetComposePath()); err != nil {
return err
}
if err := handleAppRecover(&install, req.File, false); err != nil {
if err := handleAppRecover(&install, req.File, false, req.Secret); err != nil {
return err
}
return nil
}

func handleAppBackup(install *model.AppInstall, backupDir, fileName string, excludes string) error {
func handleAppBackup(install *model.AppInstall, backupDir, fileName string, excludes string, secret string) error {
fileOp := files.NewFileOp()
tmpDir := fmt.Sprintf("%s/%s", backupDir, strings.ReplaceAll(fileName, ".tar.gz", ""))
if !fileOp.Stat(tmpDir) {
Expand All @@ -103,7 +103,7 @@ func handleAppBackup(install *model.AppInstall, backupDir, fileName string, excl
}

appPath := install.GetPath()
if err := handleTar(appPath, tmpDir, "app.tar.gz", excludes); err != nil {
if err := handleTar(appPath, tmpDir, "app.tar.gz", excludes, ""); err != nil {
return err
}

Expand All @@ -129,16 +129,16 @@ func handleAppBackup(install *model.AppInstall, backupDir, fileName string, excl
}
}

if err := handleTar(tmpDir, backupDir, fileName, ""); err != nil {
if err := handleTar(tmpDir, backupDir, fileName, "", secret); err != nil {
return err
}
return nil
}

func handleAppRecover(install *model.AppInstall, recoverFile string, isRollback bool) error {
func handleAppRecover(install *model.AppInstall, recoverFile string, isRollback bool, secret string) error {
isOk := false
fileOp := files.NewFileOp()
if err := handleUnTar(recoverFile, path.Dir(recoverFile)); err != nil {
if err := handleUnTar(recoverFile, path.Dir(recoverFile), secret); err != nil {
return err
}
tmpPath := strings.ReplaceAll(recoverFile, ".tar.gz", "")
Expand All @@ -164,13 +164,13 @@ func handleAppRecover(install *model.AppInstall, recoverFile string, isRollback

if !isRollback {
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("app/%s_%s.tar.gz", install.Name, time.Now().Format("20060102150405")))
if err := handleAppBackup(install, path.Dir(rollbackFile), path.Base(rollbackFile), ""); err != nil {
if err := handleAppBackup(install, path.Dir(rollbackFile), path.Base(rollbackFile), "", ""); err != nil {
return fmt.Errorf("backup app %s for rollback before recover failed, err: %v", install.Name, err)
}
defer func() {
if !isOk {
global.LOG.Info("recover failed, start to rollback now")
if err := handleAppRecover(install, rollbackFile, true); err != nil {
if err := handleAppRecover(install, rollbackFile, true, secret); err != nil {
global.LOG.Errorf("rollback app %s from %s failed, err: %v", install.Name, rollbackFile, err)
return
}
Expand Down Expand Up @@ -251,7 +251,7 @@ func handleAppRecover(install *model.AppInstall, recoverFile string, isRollback
_ = fileOp.Rename(appDir, backPath)
_ = fileOp.CreateDir(appDir, 0755)

if err := handleUnTar(tmpPath+"/app.tar.gz", install.GetAppPath()); err != nil {
if err := handleUnTar(tmpPath+"/app.tar.gz", install.GetAppPath(), ""); err != nil {
global.LOG.Errorf("handle recover from app.tar.gz failed, err: %v", err)
_ = fileOp.DeleteDir(appDir)
_ = fileOp.Rename(backPath, appDir)
Expand Down
2 changes: 1 addition & 1 deletion backend/app/service/backup_mysql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func (u *BackupService) MysqlRecoverByUpload(req dto.CommonRecover) error {
return fmt.Errorf("mkdir %s failed, err: %v", dstDir, err)
}
}
if err := handleUnTar(req.File, dstDir); err != nil {
if err := handleUnTar(req.File, dstDir, ""); err != nil {
_ = os.RemoveAll(dstDir)
return err
}
Expand Down
2 changes: 1 addition & 1 deletion backend/app/service/backup_postgresql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func (u *BackupService) PostgresqlRecoverByUpload(req dto.CommonRecover) error {
return fmt.Errorf("mkdir %s failed, err: %v", dstDir, err)
}
}
if err := handleUnTar(req.File, dstDir); err != nil {
if err := handleUnTar(req.File, dstDir, ""); err != nil {
_ = os.RemoveAll(dstDir)
return err
}
Expand Down
16 changes: 8 additions & 8 deletions backend/app/service/backup_redis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func (u *BackupService) RedisBackup(db dto.CommonBackup) error {
}
itemDir := fmt.Sprintf("database/redis/%s", redisInfo.Name)
backupDir := path.Join(localDir, itemDir)
if err := handleRedisBackup(redisInfo, backupDir, fileName); err != nil {
if err := handleRedisBackup(redisInfo, backupDir, fileName, db.Secret); err != nil {
return err
}
record := &model.BackupRecord{
Expand All @@ -70,13 +70,13 @@ func (u *BackupService) RedisRecover(req dto.CommonRecover) error {
return err
}
global.LOG.Infof("recover redis from backup file %s", req.File)
if err := handleRedisRecover(redisInfo, req.File, false); err != nil {
if err := handleRedisRecover(redisInfo, req.File, false, req.Secret); err != nil {
return err
}
return nil
}

func handleRedisBackup(redisInfo *repo.RootInfo, backupDir, fileName string) error {
func handleRedisBackup(redisInfo *repo.RootInfo, backupDir, fileName string, secret string) error {
fileOp := files.NewFileOp()
if !fileOp.Stat(backupDir) {
if err := os.MkdirAll(backupDir, os.ModePerm); err != nil {
Expand All @@ -91,7 +91,7 @@ func handleRedisBackup(redisInfo *repo.RootInfo, backupDir, fileName string) err

if strings.HasSuffix(fileName, ".tar.gz") {
redisDataDir := fmt.Sprintf("%s/%s/%s/data/appendonlydir", constant.AppInstallDir, "redis", redisInfo.Name)
if err := handleTar(redisDataDir, backupDir, fileName, ""); err != nil {
if err := handleTar(redisDataDir, backupDir, fileName, "", secret); err != nil {
return err
}
return nil
Expand All @@ -111,7 +111,7 @@ func handleRedisBackup(redisInfo *repo.RootInfo, backupDir, fileName string) err
return nil
}

func handleRedisRecover(redisInfo *repo.RootInfo, recoverFile string, isRollback bool) error {
func handleRedisRecover(redisInfo *repo.RootInfo, recoverFile string, isRollback bool, secret string) error {
fileOp := files.NewFileOp()
if !fileOp.Stat(recoverFile) {
return buserr.WithName("ErrFileNotFound", recoverFile)
Expand Down Expand Up @@ -147,13 +147,13 @@ func handleRedisRecover(redisInfo *repo.RootInfo, recoverFile string, isRollback
}
}
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/redis/%s_%s.%s", redisInfo.Name, time.Now().Format("20060102150405"), suffix))
if err := handleRedisBackup(redisInfo, path.Dir(rollbackFile), path.Base(rollbackFile)); err != nil {
if err := handleRedisBackup(redisInfo, path.Dir(rollbackFile), path.Base(rollbackFile), secret); err != nil {
return fmt.Errorf("backup database %s for rollback before recover failed, err: %v", redisInfo.Name, err)
}
defer func() {
if !isOk {
global.LOG.Info("recover failed, start to rollback now")
if err := handleRedisRecover(redisInfo, rollbackFile, true); err != nil {
if err := handleRedisRecover(redisInfo, rollbackFile, true, secret); err != nil {
global.LOG.Errorf("rollback redis from %s failed, err: %v", rollbackFile, err)
return
}
Expand All @@ -170,7 +170,7 @@ func handleRedisRecover(redisInfo *repo.RootInfo, recoverFile string, isRollback
}
if appendonly == "yes" && strings.HasPrefix(redisInfo.Version, "7.") {
redisDataDir := fmt.Sprintf("%s/%s/%s/data", constant.AppInstallDir, "redis", redisInfo.Name)
if err := handleUnTar(recoverFile, redisDataDir); err != nil {
if err := handleUnTar(recoverFile, redisDataDir, secret); err != nil {
return err
}
} else {
Expand Down
16 changes: 8 additions & 8 deletions backend/app/service/backup_runtime.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
"github.com/1Panel-dev/1Panel/backend/utils/files"
)

func handleRuntimeBackup(runtime *model.Runtime, backupDir, fileName string, excludes string) error {
func handleRuntimeBackup(runtime *model.Runtime, backupDir, fileName string, excludes string, secret string) error {
fileOp := files.NewFileOp()
tmpDir := fmt.Sprintf("%s/%s", backupDir, strings.ReplaceAll(fileName, ".tar.gz", ""))
if !fileOp.Stat(tmpDir) {
Expand All @@ -35,19 +35,19 @@ func handleRuntimeBackup(runtime *model.Runtime, backupDir, fileName string, exc
}

appPath := runtime.GetPath()
if err := handleTar(appPath, tmpDir, "runtime.tar.gz", excludes); err != nil {
if err := handleTar(appPath, tmpDir, "runtime.tar.gz", excludes, secret); err != nil {
return err
}
if err := handleTar(tmpDir, backupDir, fileName, ""); err != nil {
if err := handleTar(tmpDir, backupDir, fileName, "", secret); err != nil {
return err
}
return nil
}

func handleRuntimeRecover(runtime *model.Runtime, recoverFile string, isRollback bool) error {
func handleRuntimeRecover(runtime *model.Runtime, recoverFile string, isRollback bool, secret string) error {
isOk := false
fileOp := files.NewFileOp()
if err := handleUnTar(recoverFile, path.Dir(recoverFile)); err != nil {
if err := handleUnTar(recoverFile, path.Dir(recoverFile), secret); err != nil {
return err
}
tmpPath := strings.ReplaceAll(recoverFile, ".tar.gz", "")
Expand All @@ -73,13 +73,13 @@ func handleRuntimeRecover(runtime *model.Runtime, recoverFile string, isRollback

if !isRollback {
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("runtime/%s_%s.tar.gz", runtime.Name, time.Now().Format("20060102150405")))
if err := handleRuntimeBackup(runtime, path.Dir(rollbackFile), path.Base(rollbackFile), ""); err != nil {
if err := handleRuntimeBackup(runtime, path.Dir(rollbackFile), path.Base(rollbackFile), "", secret); err != nil {
return fmt.Errorf("backup runtime %s for rollback before recover failed, err: %v", runtime.Name, err)
}
defer func() {
if !isOk {
global.LOG.Info("recover failed, start to rollback now")
if err := handleRuntimeRecover(runtime, rollbackFile, true); err != nil {
if err := handleRuntimeRecover(runtime, rollbackFile, true, secret); err != nil {
global.LOG.Errorf("rollback runtime %s from %s failed, err: %v", runtime.Name, rollbackFile, err)
return
}
Expand All @@ -100,7 +100,7 @@ func handleRuntimeRecover(runtime *model.Runtime, recoverFile string, isRollback
_ = fileOp.Rename(runtimeDir, backPath)
_ = fileOp.CreateDir(runtimeDir, 0755)

if err := handleUnTar(tmpPath+"/runtime.tar.gz", fmt.Sprintf("%s/%s", constant.RuntimeDir, runtime.Type)); err != nil {
if err := handleUnTar(tmpPath+"/runtime.tar.gz", fmt.Sprintf("%s/%s", constant.RuntimeDir, runtime.Type), secret); err != nil {
global.LOG.Errorf("handle recover from runtime.tar.gz failed, err: %v", err)
_ = fileOp.DeleteDir(runtimeDir)
_ = fileOp.Rename(backPath, runtimeDir)
Expand Down
Loading