Handle publish divergence recovery and guidance

Co-authored-by: Andrew <andrewxhill@gmail.com>

Author: andrewxhill-ai

.agents/skills/worktree/SKILL.md

@@ -177,6 +177,17 @@ id instead of polling by branch name. Follow the submission, not the logs.
 Do not use sleeps, branch-name polling, `mq logs`, `mq events`, or `mq watch` as the
 primary way to decide whether a queued change finished.
 
+If a publish request fails because remote `main` moved first, prefer retrying
+the publish request from the protected worktree:
+
+```bash
+mq retry --repo ~/Projects/recallnet/mainline --publish <id>
+```
+
+`mq` now attempts the safe case automatically by fetching upstream, rebasing the
+unpublished protected-branch commits onto the updated remote tip when that
+replay is clean, and retrying the push.
+
 If the branch is behind local protected `main` or a submission blocks on a
 rebase conflict, use `mq rebase` as the default repair path:
 

README.md

@@ -108,6 +108,17 @@ If the wrapper expects remote landing as part of the same job, prefer:
 mq land --json --timeout 30m
 ```
 
+If publish loses a race because remote `main` moved first, rerun the failed
+publish request instead of hand-repairing protected `main`:
+
+```bash
+mq retry --repo /path/to/protected-worktree --publish 4
+```
+
+When the unpublished protected-branch commits can be replayed cleanly, `mq`
+fetches upstream, rebases the protected branch onto the updated remote tip, and
+retries the push automatically.
+
 If a submission blocks because its topic branch is behind local protected
 `main`, use `mq rebase` instead of hand-rolling the Git repair:
 
@@ -383,6 +394,7 @@ mq watch --repo /path/to/repo-root
 mq events --repo /path/to/repo-root --follow --json --lifecycle
 mq registry prune --json
 mq retry --repo /path/to/repo-root --submission 17
+mq retry --repo /path/to/protected-worktree --publish 4
 mq cancel --repo /path/to/repo-root --publish 4
 ```
 

SPEC.md

@@ -116,6 +116,17 @@ mq wait --submission <id> --for landed --json --timeout 30m
 mq land --json --timeout 30m
 ```
 
+If a publish request fails because remote `main` advanced first, the supported
+operator recovery path is:
+
+```bash
+mq retry --repo /path/to/protected-worktree --publish <id>
+```
+
+Before retrying the push, `mq` may fetch upstream and rebase the protected
+branch onto the updated upstream tip when unpublished local commits replay
+cleanly.
+
 ### Machine-Wide Daemon Flow
 
 ```bash

docs/FLOWS.md

@@ -211,6 +211,16 @@ mq submit --allow-newer-head --wait --timeout 10m --json
 That only permits forward movement. If the queued branch rewinds or moves to a
 non-descendant tip, `mq` still fails the submission and asks for a resubmit.
 
+If a publish request fails because remote `main` advanced first, prefer:
+
+```bash
+mq retry --repo /path/to/protected-worktree --publish 4
+```
+
+That lets `mq` fetch upstream, replay unpublished protected-branch commits onto
+the updated remote tip when the rebase is clean, and retry the push without
+manual protected-worktree repair.
+
 Exit codes:
 
 - `0`: integrated

docs/JSON_CONTRACTS.md

@@ -156,6 +156,9 @@ submission record with optional blocked-state diagnostics:
 - `publish_request_id`
 - `publish_status`
 - `outcome`
+- `publish_failure_cause`
+- `publish_failure_summary`
+- `publish_failure_error`
 - `queue_position`
 - `estimated_completion_ms`
 - `estimate_basis`
@@ -321,6 +324,10 @@ Optional fields:
 - `protected_sha`
 - `publish_request_id`
 - `publish_status`
+- `publish_failure_cause`
+- `publish_failure_summary`
+- `publish_failure_error`
+- `retry_hint`
 - `last_worker_result`
 - `error`
 

internal/app/app_test.go

@@ -6,6 +6,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"slices"
@@ -911,6 +912,104 @@ func TestStatusJSONCorrelatesSubmissionToPublish(t *testing.T) {
 	}
 }
 
+func TestStatusAndWaitSurfaceRetryCommandForRejectedPublishAfterRemoteAdvance(t *testing.T) {
+	repoRoot, remoteDir := createTestRepoWithRemote(t)
+	initRepoForWorker(t, repoRoot)
+	runTestCommand(t, repoRoot, "git", "push", "origin", "main")
+	updatePublishMode(t, repoRoot, "auto")
+
+	featurePath := filepath.Join(t.TempDir(), "feature-status-publish-reject")
+	runTestCommand(t, repoRoot, "git", "worktree", "add", "-b", "feature/status-publish-reject", featurePath)
+	writeFileAndCommit(t, featurePath, "status.txt", "status\n", "status feature")
+	submitBranch(t, featurePath)
+	runOnce(t, repoRoot)
+
+	layout, err := git.DiscoverRepositoryLayout(repoRoot)
+	if err != nil {
+		t.Fatalf("DiscoverRepositoryLayout: %v", err)
+	}
+	store := state.NewStore(state.DefaultPath(layout.GitDir))
+	repoRecord, err := store.GetRepositoryByPath(context.Background(), layout.RepositoryRoot)
+	if err != nil {
+		t.Fatalf("GetRepositoryByPath: %v", err)
+	}
+	submissions, err := store.ListIntegrationSubmissions(context.Background(), repoRecord.ID)
+	if err != nil {
+		t.Fatalf("ListIntegrationSubmissions: %v", err)
+	}
+	requests, err := store.ListPublishRequests(context.Background(), repoRecord.ID)
+	if err != nil {
+		t.Fatalf("ListPublishRequests: %v", err)
+	}
+	if len(submissions) != 1 || len(requests) != 1 {
+		t.Fatalf("expected 1 submission and 1 publish request, got %+v %+v", submissions, requests)
+	}
+	if _, err := store.UpdatePublishRequestStatus(context.Background(), requests[0].ID, domain.PublishStatusFailed, sql.NullInt64{}); err != nil {
+		t.Fatalf("UpdatePublishRequestStatus: %v", err)
+	}
+	if err := appendStateEvent(context.Background(), store, state.EventRecord{
+		RepoID:    repoRecord.ID,
+		ItemType:  domain.ItemTypePublishRequest,
+		ItemID:    state.NullInt64(requests[0].ID),
+		EventType: domain.EventTypePublishFailed,
+		Payload: mustJSON(map[string]string{
+			"target_sha": requests[0].TargetSHA,
+			"error":      "git push was rejected: To github.com:recallnet/tradecore.git\n ! [rejected]            main -> main (fetch first)\nerror: failed to push some refs to 'github.com:recallnet/tradecore.git'",
+			"kind":       publishFailureKindGitPushFailed,
+			"stage":      publishStagePush,
+		}),
+	}); err != nil {
+		t.Fatalf("appendStateEvent: %v", err)
+	}
+
+	upstreamClone := filepath.Join(t.TempDir(), "upstream-clone")
+	runTestCommand(t, t.TempDir(), "git", "clone", remoteDir, upstreamClone)
+	runTestCommand(t, upstreamClone, "git", "config", "user.name", "Test User")
+	runTestCommand(t, upstreamClone, "git", "config", "user.email", "test@example.com")
+	runTestCommand(t, upstreamClone, "git", "config", "core.hooksPath", ".git/hooks")
+	writeFileAndCommit(t, upstreamClone, "upstream.txt", "upstream\n", "upstream advance")
+	runTestCommand(t, upstreamClone, "git", "push", "origin", "main")
+	runTestCommand(t, repoRoot, "git", "fetch", "origin", "main")
+
+	var waitOut bytes.Buffer
+	var waitErr bytes.Buffer
+	err = runCLI([]string{"wait", "--repo", repoRoot, "--submission", strconv.FormatInt(submissions[0].ID, 10), "--for", "landed", "--json", "--timeout", "1s"}, newStepPrinter(&waitOut), &waitErr)
+	if err == nil {
+		t.Fatalf("expected landed wait to fail")
+	}
+
+	var waitResult submissionWaitResult
+	if err := json.Unmarshal(waitOut.Bytes(), &waitResult); err != nil {
+		t.Fatalf("Unmarshal wait: %v", err)
+	}
+	if waitResult.RetryHint != "retry-publish-after-protected-reconcile" {
+		t.Fatalf("expected protected reconcile retry hint, got %+v", waitResult)
+	}
+	if !strings.Contains(waitResult.PublishFailureSummary, "mq retry --repo") || !strings.Contains(waitResult.PublishFailureSummary, fmt.Sprintf("--publish %d", requests[0].ID)) {
+		t.Fatalf("expected retry command in wait summary, got %+v", waitResult)
+	}
+
+	var statusOut bytes.Buffer
+	var statusErr bytes.Buffer
+	if err := runCLI([]string{"status", "--repo", repoRoot, "--json"}, newStepPrinter(&statusOut), &statusErr); err != nil {
+		t.Fatalf("runCLI status returned error: %v", err)
+	}
+
+	var statusResult statusResult
+	if err := json.Unmarshal(statusOut.Bytes(), &statusResult); err != nil {
+		t.Fatalf("Unmarshal status: %v", err)
+	}
+	if statusResult.LatestSubmission == nil {
+		t.Fatalf("expected latest submission, got %+v", statusResult)
+	}
+	if !strings.Contains(statusResult.LatestSubmission.PublishFailureSummary, "mq retry --repo") || !strings.Contains(statusResult.LatestSubmission.PublishFailureSummary, fmt.Sprintf("--publish %d", requests[0].ID)) {
+		t.Fatalf("expected retry command in status summary, got %+v", statusResult.LatestSubmission)
+	}
+	if len(statusResult.LatestSubmission.NextActions) == 0 || !strings.Contains(statusResult.LatestSubmission.NextActions[0].Command, "mq retry --repo") || !strings.Contains(statusResult.LatestSubmission.NextActions[0].Command, fmt.Sprintf("--publish %d", requests[0].ID)) {
+		t.Fatalf("expected retry next action, got %+v", statusResult.LatestSubmission.NextActions)
+	}
+}
+
 func TestStatusJSONReportsPublishExecutionAndProtectedWorktreeActivity(t *testing.T) {
 	repoRoot, _ := createTestRepoWithRemote(t)
 	initRepoForWorker(t, repoRoot)

internal/app/execution_estimate.go

@@ -44,7 +44,7 @@ func collectExecutionEstimate(ctx context.Context, store state.Store, repoID int
 			integrationDurations = append(integrationDurations, succeededAt.Sub(startedAt).Milliseconds())
 		}
 
-		info, err := resolveSubmissionPublishInfo(ctx, store, repoID, submission, mainEngine)
+		info, err := resolveSubmissionPublishInfo(ctx, store, repoID, submission, mainEngine, cfg.Repo.ProtectedBranch)
 		if err != nil {
 			return executionEstimate{}, err
 		}

internal/app/land.go

@@ -255,6 +255,13 @@ func waitForLandedPublish(queued queuedSubmission, timeout time.Duration, pollIn
 					result.Published = true
 					return result, nil
 				} else if result.PublishStatus == "failed" || result.PublishStatus == "cancelled" || result.PublishStatus == "superseded" {
+					if result.PublishStatus == "failed" {
+						info, infoErr := resolvePublishFailureInfo(ctx, queued.Store, queued.RepoRecord.ID, result.PublishRequestID, mainEngine, queued.Config.Repo.ProtectedBranch)
+						if infoErr == nil && info.Summary != "" {
+							result.Error = fmt.Sprintf("publish request %d failed: %s", result.PublishRequestID, info.Summary)
+							return result, fmt.Errorf("publish request %d failed: %s", result.PublishRequestID, info.Summary)
+						}
+					}
 					result.Error = fmt.Sprintf("publish request %d %s", result.PublishRequestID, result.PublishStatus)
 					return result, fmt.Errorf("publish request %d %s", result.PublishRequestID, result.PublishStatus)
 				}

internal/app/protected_root_recovery.go

@@ -38,7 +38,7 @@ func ensureProtectedRootHealthy(ctx context.Context, engine git.Engine, lockMana
 		return report, nil
 	}
 
-	repaired, _, repairErr := tryRepairCanonicalProtectedRootWithMode(ctx, engine, cfg, store, repoRecord, mode)
+	repaired, repairNote, repairErr := tryRepairCanonicalProtectedRootWithMode(ctx, engine, cfg, store, repoRecord, mode)
 	if repairErr != nil {
 		return git.HealthReport{}, repairErr
 	}
@@ -60,6 +60,9 @@ func ensureProtectedRootHealthy(ctx context.Context, engine git.Engine, lockMana
 		return git.HealthReport{}, protectedWorktreeDirtyError(cfg.Repo.MainWorktree, report.ProtectedDirtyPaths, activity)
 	}
 	if report.HasDivergedUpstream {
+		if repairNote != "" {
+			return git.HealthReport{}, fmt.Errorf("%s", repairNote)
+		}
 		return git.HealthReport{}, fmt.Errorf("protected branch %q has diverged from upstream %s", cfg.Repo.ProtectedBranch, report.UpstreamRef)
 	}
 	return report, nil

internal/app/publish_test.go

@@ -740,6 +740,69 @@ func TestDrainRepoUntilSettledSleepsThroughDelayedPublishRetry(t *testing.T) {
 	}
 }
 
+func TestRetryPublishAutoRebasesProtectedBranchAfterRemoteAdvance(t *testing.T) {
+	repoRoot, remoteDir := createTestRepoWithRemote(t)
+	initRepoForWorker(t, repoRoot)
+	runTestCommand(t, repoRoot, "git", "push", "origin", "main")
+	t.Setenv("MAINLINE_DISABLE_MUTATION_DRAIN", "1")
+
+	writeFileAndCommit(t, repoRoot, "local.txt", "local\n", "main change local")
+	queuePublish(t, repoRoot)
+	t.Setenv("MAINLINE_DISABLE_MUTATION_DRAIN", "")
+
+	layout, err := git.DiscoverRepositoryLayout(repoRoot)
+	if err != nil {
+		t.Fatalf("DiscoverRepositoryLayout: %v", err)
+	}
+	store := state.NewStore(state.DefaultPath(layout.GitDir))
+	repoRecord, err := store.GetRepositoryByPath(context.Background(), layout.RepositoryRoot)
+	if err != nil {
+		t.Fatalf("GetRepositoryByPath: %v", err)
+	}
+	requests, err := store.ListPublishRequests(context.Background(), repoRecord.ID)
+	if err != nil {
+		t.Fatalf("ListPublishRequests: %v", err)
+	}
+	if len(requests) != 1 {
+		t.Fatalf("expected 1 publish request, got %d", len(requests))
+	}
+	if _, err := store.UpdatePublishRequestStatus(context.Background(), requests[0].ID, domain.PublishStatusFailed, requests[0].SupersededBy); err != nil {
+		t.Fatalf("UpdatePublishRequestStatus: %v", err)
+	}
+
+	upstreamClone := filepath.Join(t.TempDir(), "upstream-clone")
+	runTestCommand(t, t.TempDir(), "git", "clone", remoteDir, upstreamClone)
+	runTestCommand(t, upstreamClone, "git", "config", "user.name", "Test User")
+	runTestCommand(t, upstreamClone, "git", "config", "user.email", "test@example.com")
+	runTestCommand(t, upstreamClone, "git", "config", "core.hooksPath", ".git/hooks")
+	writeFileAndCommit(t, upstreamClone, "upstream.txt", "upstream\n", "upstream advance")
+	upstreamHead := trimNewline(runTestCommand(t, upstreamClone, "git", "rev-parse", "HEAD"))
+	runTestCommand(t, upstreamClone, "git", "push", "origin", "main")
+
+	var retryOut bytes.Buffer
+	var retryErr bytes.Buffer
+	if err := runRetry([]string{"--repo", repoRoot, "--publish", strconv.FormatInt(requests[0].ID, 10)}, newStepPrinter(&retryOut), &retryErr); err != nil {
+		t.Fatalf("runRetry returned error: %v", err)
+	}
+
+	localHead := trimNewline(runTestCommand(t, repoRoot, "git", "rev-parse", "HEAD"))
+	remoteHead := trimNewline(runTestCommand(t, remoteDir, "git", "rev-parse", "refs/heads/main"))
+	if remoteHead != localHead {
+		t.Fatalf("expected remote head %q, got %q", localHead, remoteHead)
+	}
+	parent := trimNewline(runTestCommand(t, repoRoot, "git", "rev-parse", "HEAD^"))
+	if parent != upstreamHead {
+		t.Fatalf("expected rebased protected branch to sit on upstream head %q, got %q", upstreamHead, parent)
+	}
+	refreshed, err := store.GetPublishRequest(context.Background(), requests[0].ID)
+	if err != nil {
+		t.Fatalf("GetPublishRequest: %v", err)
+	}
+	if refreshed.Status != domain.PublishStatusSucceeded {
+		t.Fatalf("expected publish request succeeded after retry, got %+v", refreshed)
+	}
+}
+
 func TestIsTransientPublishErrorRecognizesGitHTTPStatusShape(t *testing.T) {
 	err := errors.New("fatal: unable to access 'https://github.com/acme/repo.git/': The requested URL returned error: 503")
 	if !isTransientPublishError(err) {