Output: Support extended parsing and redirection

Also adds a status monitor when run in console mode

Author: jkellerer

config/flag.go

@@ -205,7 +205,10 @@ func stringify(value reflect.Value, onlySimplyValues bool) ([]string, bool) {
 		sort.Strings(flatMap)
 		return flatMap, len(flatMap) > 0
 
-	case reflect.Interface:
+	case reflect.Interface, reflect.Pointer:
+		if value.IsNil() {
+			return emptyStringArray, false
+		}
 		return stringify(value.Elem(), onlySimplyValues)
 
 	default:

config/flag_test.go

@@ -13,7 +13,7 @@ func TestPointerValueShouldReturnErrorMessage(t *testing.T) {
 	concrete := "test"
 	value := &concrete
 	argValue, _ := stringifyValueOf(value)
-	assert.Equal(t, []string{"ERROR: unexpected type ptr"}, argValue)
+	assert.Equal(t, []string{"test"}, argValue)
 }
 
 func TestNilValueFlag(t *testing.T) {

config/info.go

@@ -659,11 +659,10 @@ func NewProfileInfoForRestic(resticVersion string, withDefaultOptions bool) Prof
 	// Building initial set including generic sections (from data model)
 	profileSet := propertySetFromType(infoTypes.profile)
 	{
-		genericSection := propertySetFromType(infoTypes.genericSection)
 		for _, name := range infoTypes.genericSectionNames {
 			pi := new(propertyInfo)
 			pi.nested = &namedPropertySet{
-				propertySet: genericSection,
+				propertySet: propertySetFromType(infoTypes.genericSection),
 				name:        name,
 			}
 			profileSet.properties[name] = pi

config/info_customizer.go

@@ -119,6 +119,31 @@ func init() {
 		}
 	})
 
+	// Profile: "stdout-hidden" default values
+	redirectOutput, noRedirectOutput := &RedirectOutputSection{StdoutFile: []string{""}}, &RedirectOutputSection{}
+	registerPropertyInfoCustomizer(func(sectionName, propertyName string, property accessibleProperty) {
+		if propertyName == constants.ParameterStdoutHidden {
+			property.basic().addDescriptionFilter(func(desc string) string {
+				var def string
+				if redirectOutput.IsOutputHidden(sectionName, false) == HideOutput {
+					def = `Default is "true" when redirected`
+				} else if noRedirectOutput.IsOutputHidden(sectionName, false) == HideOutput {
+					def = `Default is "true"`
+				} else if noRedirectOutput.IsOutputHidden(sectionName, true) == HideOutput {
+					if sectionName == constants.CommandBackup {
+						def = `Default is "true" when "extended-status" is set`
+					} else {
+						def = `Default is "true" when "json" is requested`
+					}
+				}
+				if def != "" {
+					desc = fmt.Sprintf("%s. %s", strings.TrimSuffix(strings.TrimSpace(desc), "."), def)
+				}
+				return desc
+			})
+		}
+	})
+
 	// Profile: exclude "help" (--help flag doesn't need to be in the reference)
 	ExcludeProfileProperty("*", "help")
 }

config/info_customizer_test.go

@@ -171,6 +171,37 @@ func TestDeprecatedSection(t *testing.T) {
 	require.True(t, set.PropertyInfo("schedule").IsDeprecated())
 }
 
+func TestStdoutHiddenProperty(t *testing.T) {
+	var testType = struct {
+		RedirectOutputSection `mapstructure:",squash"`
+	}{}
+
+	var tests = []struct {
+		section  string
+		expected string
+	}{
+		{section: constants.CommandCat, expected: `Default is "true" when redirected`},
+		{section: constants.CommandDump, expected: `Default is "true" when redirected`},
+		{section: constants.CommandCopy, expected: ``},
+		{section: constants.CommandBackup, expected: ``}, // Default is "true" when "extended-status" is set
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			set := propertySetFromType(reflect.TypeOf(testType))
+			require.NotNil(t, set.PropertyInfo(constants.ParameterStdoutHidden))
+
+			base := set.PropertyInfo(constants.ParameterStdoutHidden).Description()
+			assert.NotEmpty(t, base)
+
+			customizeProperties(test.section, set.properties)
+			assert.Equal(t,
+				strings.TrimSuffix(strings.TrimSpace(fmt.Sprintf("%s. %s", base, test.expected)), "."),
+				set.PropertyInfo(constants.ParameterStdoutHidden).Description())
+		})
+	}
+}
+
 func TestHelpIsExcluded(t *testing.T) {
 	assert.True(t, isExcluded("*", "help"))
 	assert.False(t, isExcluded("*", "any-other"))

config/profile.go

@@ -42,6 +42,21 @@ type RunShellCommands interface {
 	GetRunShellCommands() *RunShellCommandsSection
 }
 
+type OutputHidden int
+
+const (
+	ShowOutput = OutputHidden(iota)
+	HideJsonOutput
+	HideOutput
+)
+
+// RedirectOutput provides access to output redirection settings
+type RedirectOutput interface {
+	IsOutputHidden(commandName string, jsonRequested bool) OutputHidden
+	IsOutputRedirected() bool
+	GetRedirectOutput() *RedirectOutputSection
+}
+
 // OtherFlags provides access to dynamic commandline flags
 type OtherFlags interface {
 	GetOtherFlags() map[string]any
@@ -106,6 +121,7 @@ type Profile struct {
 type GenericSection struct {
 	OtherFlagsSection       `mapstructure:",squash"`
 	RunShellCommandsSection `mapstructure:",squash"`
+	RedirectOutputSection   `mapstructure:",squash"`
 }
 
 func (g *GenericSection) IsEmpty() bool { return g == nil }
@@ -155,6 +171,7 @@ func (i *InitSection) getCommandFlags(profile *Profile) (flags *shell.Args) {
 type BackupSection struct {
 	SectionWithScheduleAndMonitoring `mapstructure:",squash"`
 	RunShellCommandsSection          `mapstructure:",squash"`
+	RedirectOutputSection            `mapstructure:",squash"`
 	unresolvedSource                 []string
 	CheckBefore                      bool     `mapstructure:"check-before" description:"Check the repository before starting the backup command"`
 	CheckAfter                       bool     `mapstructure:"check-after" description:"Check the repository after the backup command succeeded"`
@@ -165,7 +182,7 @@ type BackupSection struct {
 	Iexclude                         []string `mapstructure:"iexclude" argument:"iexclude" argument-type:"no-glob"`
 	ExcludeFile                      []string `mapstructure:"exclude-file" argument:"exclude-file"`
 	FilesFrom                        []string `mapstructure:"files-from" argument:"files-from"`
-	ExtendedStatus                   bool     `mapstructure:"extended-status" argument:"json"`
+	ExtendedStatus                   *bool    `mapstructure:"extended-status" argument:"json"`
 	NoErrorOnWarning                 bool     `mapstructure:"no-error-on-warning" description:"Do not fail the backup when some files could not be read"`
 }
 
@@ -176,6 +193,10 @@ func (b *BackupSection) resolve(p *Profile) {
 	if len(b.StdinCommand) > 0 {
 		b.UseStdin = true
 	}
+	// Enable ExtendedStatus if unset and required for full functionality
+	if bools.IsUndefined(b.ExtendedStatus) && len(p.StatusFile)+len(p.PrometheusSaveToFile)+len(p.PrometheusPush) != 0 {
+		b.ExtendedStatus = bools.True()
+	}
 	// Resolve source paths
 	if b.unresolvedSource == nil {
 		b.unresolvedSource = b.Source
@@ -247,6 +268,7 @@ func (s *ScheduleBaseSection) GetSchedule() *ScheduleBaseSection { return s }
 type CopySection struct {
 	SectionWithScheduleAndMonitoring `mapstructure:",squash"`
 	RunShellCommandsSection          `mapstructure:",squash"`
+	RedirectOutputSection            `mapstructure:",squash"`
 	Initialize                       bool              `mapstructure:"initialize" description:"Initialize the secondary repository if missing"`
 	InitializeCopyChunkerParams      *bool             `mapstructure:"initialize-copy-chunker-params" default:"true" description:"Copy chunker parameters when initializing the secondary repository"`
 	Repository                       ConfidentialValue `mapstructure:"repository" description:"Destination repository to copy snapshots to"`
@@ -392,6 +414,37 @@ type SendMonitoringHeader struct {
 	Value ConfidentialValue `mapstructure:"value" examples:"\"Bearer ...\";\"Basic ...\";\"no-cache\";\"attachment;; filename=stats.txt\";\"application/json\";\"text/plain\";\"text/xml\"" description:"Value of the header"`
 }
 
+// RedirectOutputSection contains settings to redirect restic command output
+type RedirectOutputSection struct {
+	StdoutHidden  *bool    `mapstructure:"stdout-hidden" description:"Toggles whether stdout is hidden in console & log"`
+	StdoutFile    []string `mapstructure:"stdout-file" description:"Redirect restic stdout to file(s)"`
+	StdoutCommand []string `mapstructure:"stdout-command" description:"Pipe restic stdout to shell command(s)"`
+}
+
+func (r *RedirectOutputSection) GetRedirectOutput() *RedirectOutputSection { return r }
+
+func (r *RedirectOutputSection) IsOutputRedirected() bool {
+	return r != nil &&
+		(len(r.StdoutFile) > 0 || len(r.StdoutCommand) > 0)
+}
+
+func (r *RedirectOutputSection) IsOutputHidden(commandName string, jsonRequested bool) (hidden OutputHidden) {
+	if r != nil {
+		if bools.IsTrue(r.StdoutHidden) {
+			hidden = HideOutput
+		} else if !bools.IsStrictlyFalse(r.StdoutHidden) {
+			if commandName == constants.CommandDump || commandName == constants.CommandCat {
+				if r.IsOutputRedirected() {
+					hidden = HideOutput
+				}
+			} else if commandName == constants.CommandBackup && jsonRequested {
+				hidden = HideJsonOutput
+			}
+		}
+	}
+	return
+}
+
 // OtherFlagsSection contains additional restic command line flags
 type OtherFlagsSection struct {
 	OtherFlags map[string]any `mapstructure:",remain"`

constants/command.go

@@ -10,6 +10,7 @@ const (
 	CommandSnapshots = "snapshots"
 	CommandUnlock    = "unlock"
 	CommandMount     = "mount"
+	CommandCat       = "cat"
 	CommandCopy      = "copy"
 	CommandDump      = "dump"
 	CommandFind      = "find"

constants/parameter.go

@@ -22,4 +22,5 @@ const (
 	ParameterPasswordFile    = "password-file"
 	ParameterPasswordCommand = "password-command"
 	ParameterKeyHint         = "key-hint"
+	ParameterStdoutHidden    = "stdout-hidden"
 )

main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/creativeprojects/resticprofile/config"
 	"github.com/creativeprojects/resticprofile/constants"
 	"github.com/creativeprojects/resticprofile/filesearch"
+	"github.com/creativeprojects/resticprofile/monitor/console"
 	"github.com/creativeprojects/resticprofile/monitor/prom"
 	"github.com/creativeprojects/resticprofile/monitor/status"
 	"github.com/creativeprojects/resticprofile/preventsleep"
@@ -403,6 +404,7 @@ func runProfile(
 	}
 
 	// add progress receivers if necessary
+	wrapper.addProgress(console.NewProgress(profile))
 	if profile.StatusFile != "" {
 		wrapper.addProgress(status.NewProgress(profile, status.NewStatus(profile.StatusFile)))
 	}