Heal protected worktree index.lock contention

Co-authored-by: Andrew <andrewxhill@gmail.com>

Author: andrewxhill-ai

internal/app/app_test.go

@@ -266,6 +266,64 @@ func TestNextPushReportsNextPublishedProtectedAdvance(t *testing.T) {
 	}
 }
 
+func TestRunOnceHealsStaleProtectedWorktreeIndexLockDuringIntegration(t *testing.T) {
+	repoRoot, _ := createTestRepoWithRemote(t)
+	initRepoForWorker(t, repoRoot)
+
+	featurePath := filepath.Join(t.TempDir(), "feature-stale-lock")
+	runTestCommand(t, repoRoot, "git", "worktree", "add", "-b", "feature/stale-lock", featurePath)
+	writeFileAndCommit(t, featurePath, "stale-lock.txt", "stale lock\n", "stale lock feature")
+	featureHead := trimNewline(runTestCommand(t, featurePath, "git", "rev-parse", "HEAD"))
+	submitBranch(t, featurePath)
+
+	lockPath := trimNewline(runTestCommand(t, repoRoot, "git", "rev-parse", "--path-format=absolute", "--git-path", "index.lock"))
+	if err := os.WriteFile(lockPath, []byte("stale"), 0o644); err != nil {
+		t.Fatalf("WriteFile(index.lock): %v", err)
+	}
+	staleTime := time.Now().Add(-2 * staleProtectedIndexLockAfter)
+	if err := os.Chtimes(lockPath, staleTime, staleTime); err != nil {
+		t.Fatalf("Chtimes(index.lock): %v", err)
+	}
+
+	runOnce(t, repoRoot)
+
+	if _, err := os.Stat(lockPath); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("expected stale index.lock removed, stat err=%v", err)
+	}
+	localHead := trimNewline(runTestCommand(t, repoRoot, "git", "rev-parse", "HEAD"))
+	if localHead != featureHead {
+		t.Fatalf("expected protected head %q, got %q", featureHead, localHead)
+	}
+}
+
+func TestRunOnceRetriesTransientProtectedWorktreeIndexLockDuringIntegration(t *testing.T) {
+	repoRoot, _ := createTestRepoWithRemote(t)
+	initRepoForWorker(t, repoRoot)
+
+	featurePath := filepath.Join(t.TempDir(), "feature-transient-lock")
+	runTestCommand(t, repoRoot, "git", "worktree", "add", "-b", "feature/transient-lock", featurePath)
+	writeFileAndCommit(t, featurePath, "transient-lock.txt", "transient lock\n", "transient lock feature")
+	featureHead := trimNewline(runTestCommand(t, featurePath, "git", "rev-parse", "HEAD"))
+	submitBranch(t, featurePath)
+
+	lockPath := trimNewline(runTestCommand(t, repoRoot, "git", "rev-parse", "--path-format=absolute", "--git-path", "index.lock"))
+	if err := os.WriteFile(lockPath, []byte("transient"), 0o644); err != nil {
+		t.Fatalf("WriteFile(index.lock): %v", err)
+	}
+
+	go func() {
+		time.Sleep(300 * time.Millisecond)
+		_ = os.Remove(lockPath)
+	}()
+
+	runOnce(t, repoRoot)
+
+	localHead := trimNewline(runTestCommand(t, repoRoot, "git", "rev-parse", "HEAD"))
+	if localHead != featureHead {
+		t.Fatalf("expected protected head %q, got %q", featureHead, localHead)
+	}
+}
+
 func TestGlobalJSONForwardsToRunOnceAndPublish(t *testing.T) {
 	repoRoot, _ := createTestRepoWithRemote(t)
 	initRepoForWorker(t, repoRoot)

internal/app/integration_workflow.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"path/filepath"
+	"time"
 
 	"github.com/recallnet/mainline/internal/domain"
 	"github.com/recallnet/mainline/internal/git"
@@ -175,7 +176,7 @@ func processIntegrationSubmission(ctx context.Context, store state.Store, repoRe
 	if err := applyAppTestFault("integration.fast_forward"); err != nil {
 		return failIntegrationSubmission(ctx, store, repoRecord.ID, submission, err)
 	}
-	if err := mainEngine.FastForwardCurrentBranch(cfg.Repo.MainWorktree, targetRef); err != nil {
+	if err := fastForwardProtectedBranchWithIndexLockRecovery(mainEngine, cfg.Repo.MainWorktree, targetRef); err != nil {
 		return failIntegrationSubmissionWithSync(ctx, store, repoRecord.ID, submission, syncResult, err)
 	}
 
@@ -234,6 +235,44 @@ func processIntegrationSubmission(ctx context.Context, store state.Store, repoRe
 	return fmt.Sprintf("Integrated submission %d from %s onto %s", submission.ID, submissionDisplayRef(submission), cfg.Repo.ProtectedBranch), nil
 }
 
+var integrationIndexLockRetryBackoffs = []time.Duration{
+	250 * time.Millisecond,
+	time.Second,
+	2 * time.Second,
+}
+
+const staleProtectedIndexLockAfter = 10 * time.Second
+
+func fastForwardProtectedBranchWithIndexLockRecovery(mainEngine git.Engine, worktreePath string, targetRef string) error {
+	var lastErr error
+	for attempt := 0; attempt <= len(integrationIndexLockRetryBackoffs); attempt++ {
+		err := mainEngine.FastForwardCurrentBranch(worktreePath, targetRef)
+		if err == nil {
+			return nil
+		}
+		if !git.IsIndexLockHeldError(err) {
+			return err
+		}
+		lastErr = err
+
+		lockState, inspectErr := mainEngine.InspectIndexLock(worktreePath, staleProtectedIndexLockAfter)
+		if inspectErr != nil {
+			return fmt.Errorf("protected worktree index lock contention: %w", inspectErr)
+		}
+		if lockState.Exists && lockState.IsStale {
+			if removeErr := mainEngine.RemoveIndexLock(worktreePath); removeErr != nil {
+				return fmt.Errorf("protected worktree stale index lock recovery failed: %w", removeErr)
+			}
+			continue
+		}
+		if attempt == len(integrationIndexLockRetryBackoffs) {
+			break
+		}
+		time.Sleep(integrationIndexLockRetryBackoffs[attempt])
+	}
+	return fmt.Errorf("protected worktree index lock contention exceeded retry budget: %w", lastErr)
+}
+
 type protectedSyncResult struct {
 	Synced    bool
 	Upstream  string

internal/git/git.go

@@ -25,6 +25,7 @@ var ErrRebaseEmpty = errors.New("git rebase stopped on an empty commit")
 var ErrFastForwardRejected = errors.New("fast-forward update was rejected")
 var ErrPushRejected = errors.New("git push was rejected")
 var ErrPushInterrupted = errors.New("git push was interrupted")
+var ErrIndexLockHeld = errors.New("git index.lock is held")
 
 // Engine holds repository-local Git execution context.
 type Engine struct {
@@ -86,6 +87,14 @@ type CommitInfo struct {
 	CommittedAt time.Time `json:"committed_at"`
 }
 
+type IndexLockState struct {
+	Path      string        `json:"path"`
+	Exists    bool          `json:"exists"`
+	Age       time.Duration `json:"age,omitempty"`
+	IsStale   bool          `json:"is_stale,omitempty"`
+	UpdatedAt time.Time     `json:"updated_at,omitempty"`
+}
+
 // HealthReport captures repository health relevant to Milestone 1.
 type HealthReport struct {
 	RepositoryRoot        string   `json:"repository_root"`
@@ -530,12 +539,56 @@ func (e Engine) FastForwardCurrentBranch(worktreePath string, targetRef string)
 	if err == nil {
 		return nil
 	}
+	if isIndexLockContentionOutput(output) {
+		return fmt.Errorf("%w: %s", ErrIndexLockHeld, strings.TrimSpace(output))
+	}
 	if strings.Contains(output, "Not possible to fast-forward") || strings.Contains(output, "fatal: Not possible to fast-forward") {
 		return fmt.Errorf("%w: %s", ErrFastForwardRejected, strings.TrimSpace(output))
 	}
 	return err
 }
 
+func IsIndexLockHeldError(err error) bool {
+	return errors.Is(err, ErrIndexLockHeld)
+}
+
+func (e Engine) InspectIndexLock(worktreePath string, staleAfter time.Duration) (IndexLockState, error) {
+	lockPath, err := e.gitPath(worktreePath, "index.lock")
+	if err != nil {
+		return IndexLockState{}, err
+	}
+	info, err := os.Stat(lockPath)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return IndexLockState{Path: lockPath}, nil
+		}
+		return IndexLockState{}, err
+	}
+	updatedAt := info.ModTime().UTC()
+	age := time.Since(updatedAt)
+	state := IndexLockState{
+		Path:      lockPath,
+		Exists:    true,
+		Age:       age,
+		UpdatedAt: updatedAt,
+	}
+	if staleAfter > 0 && age >= staleAfter {
+		state.IsStale = true
+	}
+	return state, nil
+}
+
+func (e Engine) RemoveIndexLock(worktreePath string) error {
+	lockPath, err := e.gitPath(worktreePath, "index.lock")
+	if err != nil {
+		return err
+	}
+	if err := os.Remove(lockPath); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	return nil
+}
+
 // PushBranch pushes a local branch ref to the configured remote branch.
 func (e Engine) PushBranch(worktreePath string, remote string, branch string, noVerify bool) error {
 	handle, err := e.StartPushBranch(worktreePath, remote, branch, noVerify)
@@ -836,6 +889,11 @@ func (e Engine) runGit(worktreePath string, args ...string) (string, error) {
 	return string(output), nil
 }
 
+func isIndexLockContentionOutput(output string) bool {
+	text := strings.TrimSpace(output)
+	return strings.Contains(text, "index.lock") && strings.Contains(text, "File exists")
+}
+
 func (e Engine) gitPath(worktreePath string, pathspec string) (string, error) {
 	output, err := e.runGit(worktreePath, "rev-parse", "--git-path", pathspec)
 	if err != nil {