Merge pull request #1840 from himsngh/himsngh/1839

[1839] Fix: Truncate status.deploy.stdout as it can exceed etcd's limit on large clusters, causing reconciliation delay

Author: himsngh

pkg/app/app.go

@@ -40,6 +40,9 @@ type Hooks struct {
 type Opts struct {
 	DefaultSyncPeriod time.Duration
 	MinimumSyncPeriod time.Duration
+	// MaxStatusOutputBytes caps each stdout/stderr field written into App status.
+	// Zero means use defaultMaxStatusOutputBytes (1 MiB).
+	MaxStatusOutputBytes int
 }
 
 type App struct {

pkg/app/app_reconcile.go

@@ -127,6 +127,7 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult {
 		var fetchResult exec.CmdRunResult
 		assetsPath, fetchResult = a.fetch(assetsPath)
 
+		fetchResult = fetchResult.WithTruncatedStrings(a.maxOutputBytes())
 		a.app.Status.Fetch = &v1alpha1.AppStatusFetch{
 			Stderr:    fetchResult.Stderr,
 			Stdout:    fetchResult.Stdout,
@@ -178,7 +179,7 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult {
 }
 
 func (a *App) updateLastDeploy(result exec.CmdRunResult) exec.CmdRunResult {
-	result = result.WithFriendlyYAMLStrings()
+	result = result.WithFriendlyYAMLStrings().WithTruncatedStrings(a.maxOutputBytes())
 
 	a.app.Status.Deploy = &v1alpha1.AppStatusDeploy{
 		Stdout:           result.Stdout,
@@ -242,7 +243,7 @@ func (a *App) resetLastDeployStartedAt() {
 }
 
 func (a *App) reconcileInspect() error {
-	inspectResult := a.inspect().WithFriendlyYAMLStrings()
+	inspectResult := a.inspect().WithFriendlyYAMLStrings().WithTruncatedStrings(a.maxOutputBytes())
 
 	if !inspectResult.IsEmpty() {
 		a.app.Status.Inspect = &v1alpha1.AppStatusInspect{
@@ -339,6 +340,7 @@ func (a *App) removeAllConditions() {
 }
 
 func (a *App) setUsefulErrorMessage(result exec.CmdRunResult) {
+	result = result.WithTruncatedStrings(a.maxOutputBytes())
 	switch {
 	case result.Stderr != "":
 		a.app.Status.UsefulErrorMessage = result.Stderr

pkg/app/app_status_truncate.go

@@ -0,0 +1,24 @@
+// Copyright 2026 The Carvel Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package app
+
+// defaultMaxStatusOutputBytes is the default cap for each stdout/stderr string
+// written into an App status sub-field (deploy, fetch, inspect).
+// etcd rejects objects larger than its 1.5 MiB limit (often 2 MiB at the
+// Kubernetes gRPC client level); on clusters with large number of nodes (ex 250+),
+// a single kapp deploy can produce several MiB of output, causing the UpdateStatus call to
+// fail and delay reconciliation until a subsequent no-op reconcile fits.
+//
+// Keeping the tail is intentional: the most actionable content (resource
+// summary, error lines) always appears at the end of kapp output.
+const defaultMaxStatusOutputBytes = 1 * 1024 * 1024 // 1 MiB
+
+// maxOutputBytes returns the configured output cap for this App, falling back
+// to defaultMaxStatusOutputBytes when Opts.MaxStatusOutputBytes is zero.
+func (a *App) maxOutputBytes() int {
+	if a.opts.MaxStatusOutputBytes > 0 {
+		return a.opts.MaxStatusOutputBytes
+	}
+	return defaultMaxStatusOutputBytes
+}

pkg/app/app_status_truncate_test.go

@@ -0,0 +1,125 @@
+// Copyright 2026 The Carvel Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package app
+
+import (
+	"strings"
+	"testing"
+
+	"carvel.dev/kapp-controller/pkg/apis/kappctrl/v1alpha1"
+	"carvel.dev/kapp-controller/pkg/deploy"
+	"carvel.dev/kapp-controller/pkg/exec"
+	"carvel.dev/kapp-controller/pkg/fetch"
+	"carvel.dev/kapp-controller/pkg/metrics"
+	"carvel.dev/kapp-controller/pkg/template"
+	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+// newAppForTest returns a minimal *App wired with no-op hooks.
+func newAppForTest(t *testing.T, appModel v1alpha1.App, opts Opts) *App {
+	t.Helper()
+	return NewApp(appModel, Hooks{
+		BlockDeletion:   func() error { return nil },
+		UnblockDeletion: func() error { return nil },
+		UpdateStatus:    func(string) error { return nil },
+	}, fetch.Factory{}, template.Factory{}, deploy.Factory{},
+		logf.Log.WithName("test"),
+		opts,
+		metrics.NewMetrics(),
+		FakeComponentInfo{},
+	)
+}
+
+func Test_updateLastDeploy_TruncatesLargeStdout(t *testing.T) {
+	const limit = 50
+	a := newAppForTest(t, v1alpha1.App{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-app", Namespace: "default"},
+	}, Opts{MaxStatusOutputBytes: limit})
+	a.app.Status.Deploy = &v1alpha1.AppStatusDeploy{}
+
+	largeStdout := strings.Repeat("resource change line\n", 1000)
+	a.updateLastDeploy(exec.CmdRunResult{Stdout: largeStdout, Finished: true, ExitCode: 0})
+
+	got := a.app.Status.Deploy.Stdout
+	assert.True(t, strings.HasPrefix(got, exec.TruncationMarker),
+		"deploy stdout should start with truncation marker when over limit")
+	assert.LessOrEqual(t, len(got), len(exec.TruncationMarker)+limit,
+		"deploy stdout should not exceed marker + limit bytes")
+}
+
+func Test_updateLastDeploy_TruncatesLargeStderr(t *testing.T) {
+	const limit = 50
+	a := newAppForTest(t, v1alpha1.App{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-app", Namespace: "default"},
+	}, Opts{MaxStatusOutputBytes: limit})
+	a.app.Status.Deploy = &v1alpha1.AppStatusDeploy{}
+
+	largeStderr := strings.Repeat("error line\n", 1000)
+	a.updateLastDeploy(exec.CmdRunResult{Stderr: largeStderr, Finished: true, ExitCode: 1})
+
+	got := a.app.Status.Deploy.Stderr
+	assert.True(t, strings.HasPrefix(got, exec.TruncationMarker))
+	assert.LessOrEqual(t, len(got), len(exec.TruncationMarker)+limit)
+}
+
+func Test_updateLastDeploy_NoTruncation_WhenUnderLimit(t *testing.T) {
+	a := newAppForTest(t, v1alpha1.App{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-app", Namespace: "default"},
+	}, Opts{MaxStatusOutputBytes: 10000})
+	a.app.Status.Deploy = &v1alpha1.AppStatusDeploy{}
+
+	smallStdout := "Changes\n\nOp: 1 add\n"
+	a.updateLastDeploy(exec.CmdRunResult{Stdout: smallStdout, Finished: true, ExitCode: 0})
+
+	assert.NotEmpty(t, a.app.Status.Deploy.Stdout)
+	assert.False(t, strings.HasPrefix(a.app.Status.Deploy.Stdout, exec.TruncationMarker),
+		"output under the limit must not be prefixed with the truncation marker")
+}
+
+func Test_updateLastDeploy_TailIsPreserved(t *testing.T) {
+	const limit = 40
+	a := newAppForTest(t, v1alpha1.App{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-app", Namespace: "default"},
+	}, Opts{MaxStatusOutputBytes: limit})
+	a.app.Status.Deploy = &v1alpha1.AppStatusDeploy{}
+
+	// WithFriendlyYAMLStrings (called inside updateLastDeploy) strips trailing
+	// whitespace, so do not include a trailing newline in the expected assertion.
+	tail := "IMPORTANT: deploy succeeded"
+	a.updateLastDeploy(exec.CmdRunResult{
+		Stdout:   strings.Repeat("x", 500) + tail + "\n",
+		Finished: true,
+		ExitCode: 0,
+	})
+
+	assert.Contains(t, a.app.Status.Deploy.Stdout, tail,
+		"the tail (most recent output) must survive truncation")
+}
+
+func Test_setUsefulErrorMessage_TruncatesLargeStderr(t *testing.T) {
+	const limit = 50
+	a := newAppForTest(t, v1alpha1.App{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-app", Namespace: "default"},
+	}, Opts{MaxStatusOutputBytes: limit})
+
+	largeStderr := strings.Repeat("kapp: Error: resource conflict\n", 1000)
+	a.setUsefulErrorMessage(exec.CmdRunResult{Stderr: largeStderr, ExitCode: 1})
+
+	got := a.app.Status.UsefulErrorMessage
+	assert.True(t, strings.HasPrefix(got, exec.TruncationMarker))
+	assert.LessOrEqual(t, len(got), len(exec.TruncationMarker)+limit)
+}
+
+func Test_setUsefulErrorMessage_NoTruncation_WhenUnderLimit(t *testing.T) {
+	a := newAppForTest(t, v1alpha1.App{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-app", Namespace: "default"},
+	}, Opts{MaxStatusOutputBytes: 10000})
+
+	stderr := "kapp: Error: resource configmap/foo not found"
+	a.setUsefulErrorMessage(exec.CmdRunResult{Stderr: stderr, ExitCode: 1})
+
+	assert.Equal(t, stderr, a.app.Status.UsefulErrorMessage)
+}

pkg/exec/cmd_run_result.go

@@ -7,12 +7,17 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+	"unicode/utf8"
 )
 
 var (
 	trailingSpace = regexp.MustCompile("\\s+\n")
 )
 
+// TruncationMarker is prepended to any output field that was clipped.
+// Operators can detect truncation by checking for this prefix.
+const TruncationMarker = "[output truncated]\n"
+
 type CmdRunResult struct {
 	Stdout   string
 	Stderr   string
@@ -65,3 +70,31 @@ func (r CmdRunResult) WithFriendlyYAMLStrings() CmdRunResult {
 func (r CmdRunResult) IsEmpty() bool {
 	return r == (CmdRunResult{})
 }
+
+// WithTruncatedStrings returns a copy of r with Stdout and Stderr each capped
+// to maxBytes by keeping the tail (the most actionable kapp output always
+// appears last).  When a field is under the limit it is left unchanged.
+func (r CmdRunResult) WithTruncatedStrings(maxBytes int) CmdRunResult {
+	return CmdRunResult{
+		Stdout:   truncateOutput(r.Stdout, maxBytes),
+		Stderr:   truncateOutput(r.Stderr, maxBytes),
+		ExitCode: r.ExitCode,
+		Error:    r.Error,
+		Finished: r.Finished,
+	}
+}
+
+// truncateOutput returns s unchanged when len(s) <= maxBytes.  Otherwise it
+// returns TruncationMarker followed by the last maxBytes bytes of s, adjusted
+// forward to a valid UTF-8 rune boundary.
+func truncateOutput(s string, maxBytes int) string {
+	if len(s) <= maxBytes {
+		return s
+	}
+	start := len(s) - maxBytes
+	// Advance past UTF-8 continuation bytes to avoid splitting a multi-byte rune.
+	for start < len(s) && !utf8.RuneStart(s[start]) {
+		start++
+	}
+	return TruncationMarker + s[start:]
+}

pkg/exec/cmd_run_result_test.go

@@ -0,0 +1,121 @@
+// Copyright 2026 The Carvel Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package exec
+
+import (
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_truncateOutput(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		maxBytes int
+		check    func(t *testing.T, result string)
+	}{
+		{
+			name:     "under limit is unchanged",
+			input:    "small output",
+			maxBytes: 1024,
+			check: func(t *testing.T, result string) {
+				assert.Equal(t, "small output", result)
+			},
+		},
+		{
+			name:     "exactly at limit is unchanged",
+			input:    strings.Repeat("x", 100),
+			maxBytes: 100,
+			check: func(t *testing.T, result string) {
+				assert.Equal(t, strings.Repeat("x", 100), result)
+			},
+		},
+		{
+			name:     "empty string is unchanged",
+			input:    "",
+			maxBytes: 100,
+			check: func(t *testing.T, result string) {
+				assert.Equal(t, "", result)
+			},
+		},
+		{
+			name:     "over limit has marker prefix and tail suffix",
+			input:    strings.Repeat("a", 200) + strings.Repeat("z", 50),
+			maxBytes: 50,
+			check: func(t *testing.T, result string) {
+				assert.True(t, strings.HasPrefix(result, TruncationMarker))
+				assert.True(t, strings.HasSuffix(result, strings.Repeat("z", 50)))
+				// Marker does not count toward maxBytes.
+				assert.Equal(t, len(TruncationMarker)+50, len(result))
+			},
+		},
+		{
+			name: "tail content preserved for realistic kapp output",
+			// Many per-resource lines followed by an actionable summary.  maxBytes
+			// is set to hold only the last ~100 bytes so truncation is guaranteed,
+			// and the summary at the tail must survive.
+			input: strings.Repeat("op add configmap/cm-001 (v1) namespace: default\n", 100) +
+				"Op: 100 add, 0 delete\nWait to: 100 reconcile\n",
+			maxBytes: 100,
+			check: func(t *testing.T, result string) {
+				assert.True(t, strings.HasPrefix(result, TruncationMarker),
+					"large output must start with the truncation marker")
+				assert.Contains(t, result, "100 add",
+					"the actionable summary at the tail must be preserved after truncation")
+			},
+		},
+		{
+			name: "UTF-8 boundary: continuation byte is skipped",
+			// s = 98 'a's + "éé"  (102 bytes)
+			// maxBytes = 3 → start = 99 → s[99] = 0xA9 (continuation of first 'é')
+			// → advance to 100 → s[100] = 0xC3 (leading byte of second 'é') → stop
+			// kept = s[100:] = "é"
+			input:    strings.Repeat("a", 98) + "éé",
+			maxBytes: 3,
+			check: func(t *testing.T, result string) {
+				require.Equal(t, 102, len(strings.Repeat("a", 98)+"éé"))
+				assert.True(t, utf8.ValidString(result), "result must be valid UTF-8")
+				assert.Equal(t, TruncationMarker+"é", result)
+			},
+		},
+		{
+			name:     "pure multibyte runes produce valid UTF-8",
+			input:    strings.Repeat("あ", 100), // 300 bytes; each rune is 3 bytes
+			maxBytes: 100,
+			check: func(t *testing.T, result string) {
+				// 100 / 3 = 33 complete runes (99 bytes); byte 100 is a continuation
+				// byte and is skipped.
+				assert.True(t, utf8.ValidString(result))
+				assert.Equal(t, TruncationMarker+strings.Repeat("あ", 33), result)
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			tc.check(t, truncateOutput(tc.input, tc.maxBytes))
+		})
+	}
+}
+
+func Test_WithTruncatedStrings(t *testing.T) {
+	const limit = 20
+	r := CmdRunResult{
+		Stdout:   strings.Repeat("o", 100),
+		Stderr:   strings.Repeat("e", 100),
+		ExitCode: 1,
+		Finished: true,
+	}
+	got := r.WithTruncatedStrings(limit)
+
+	assert.True(t, strings.HasPrefix(got.Stdout, TruncationMarker))
+	assert.True(t, strings.HasPrefix(got.Stderr, TruncationMarker))
+	assert.Equal(t, r.ExitCode, got.ExitCode, "ExitCode must be preserved")
+	assert.Equal(t, r.Finished, got.Finished, "Finished must be preserved")
+	assert.Equal(t, r.Error, got.Error, "Error must be preserved")
+}