Wrapper: detect claude-code death and resume the session¶

// claudeProcess is the seam the Manager supervises. Real impl is *claudeProc
// over a PTY; tests substitute a fake whose Wait() they control.
type claudeProcess interface {
    Wait() error
    Read(p []byte) (int, error)
    Write(p []byte) (int, error)
    Close() error
}

func (c *claudeProc) Wait() error               { return c.cmd.Wait() }
func (c *claudeProc) Read(p []byte) (int, error) { return c.ptmx.Read(p) }

func (c Config) claudeArgs(resume bool) []string {
    args := []string{}
    if c.Model != "" {
        args = append(args, "--model", c.Model)
    }
    if resume {
        // Resume the most recent conversation in the workspace so a relaunched
        // session keeps its context after a crash.
        args = append(args, "--continue")
    }
    return args
}

func spawnClaude(cfg Config, resume bool) (*claudeProc, error) {
    args := cfg.claudeArgs(resume)
    ...
}

m := &Manager{cfg: cfg, ...}
m.spawn = func(cfg Config, resume bool) (claudeProcess, error) {
    p, err := spawnClaude(cfg, resume)
    if err != nil {
        return nil, err
    }
    return p, nil
}
return m

// failTurn fails the in-flight turn immediately (used when claude died and the
// restart budget is exhausted or a relaunch failed). Marks the session Dead so
// the operator respawns a fresh pod. No-op if id is no longer current.
func (mgr *Manager) failTurn(id, reason string) {
    mgr.mu.Lock()
    if mgr.current != id {
        mgr.mu.Unlock()
        return
    }
    if mgr.timer != nil {
        mgr.timer.Stop()
    }
    now := mgr.now()
    _ = mgr.store.Fail(id, reason, now)
    mgr.clearCurrentLocked(Dead)
    mgr.m.TurnsTotal.WithLabelValues("failed").Inc()
    rec, _ := mgr.store.Get(id)
    mgr.mu.Unlock()
    mgr.log.Warn("turn failed", "turn_id", id, "reason", reason)
    mgr.fireDone(rec)
}

func (mgr *Manager) watch(proc claudeProcess) {
    err := proc.Wait()
    mgr.mu.Lock()
    if mgr.stopping {
        mgr.state = Dead
        mgr.mu.Unlock()
        mgr.log.Info("claude stopped (shutdown)")
        return
    }
    inFlight := mgr.current
    mgr.restarts++
    attempt := mgr.restarts
    mgr.state = Dead // brief: Submit rejects until relaunch flips to Ready
    mgr.mu.Unlock()

    mgr.m.ClaudeRestarts.Inc()
    mgr.log.Error("claude exited unexpectedly", "err", err, "in_flight_turn", inFlight,
        "attempt", attempt, "max_restarts", mgr.cfg.MaxRestarts, "pty_tail", mgr.ring.tail(800))

    if attempt > mgr.cfg.MaxRestarts {
        mgr.log.Error("claude restart budget exhausted; operator will respawn",
            "max_restarts", mgr.cfg.MaxRestarts)
        if inFlight != "" {
            mgr.failTurn(inFlight, fmt.Sprintf("claude died; restart budget (%d) exhausted", mgr.cfg.MaxRestarts))
        }
        return // state stays Dead
    }

    if rerr := mgr.relaunch(); rerr != nil {
        mgr.mu.Lock()
        mgr.state = Dead
        mgr.mu.Unlock()
        mgr.log.Error("claude relaunch failed; operator will respawn", "err", rerr)
        if inFlight != "" {
            mgr.failTurn(inFlight, fmt.Sprintf("claude relaunch failed: %v", rerr))
        }
        return
    }
    mgr.log.Info("claude relaunched after exit", "attempt", attempt, "resumed_turn", inFlight)
    if inFlight != "" {
        mgr.resumeTurn(inFlight)
    }
}

// relaunch spawns a fresh claude (with --continue when a conversation exists),
// rewires the PTY, restarts the reader+watcher, and waits for boot. The new
// watch goroutine handles the next death (restarts persists across relaunches).
func (mgr *Manager) relaunch() error {
    proc, err := mgr.spawn(mgr.cfg, mgr.shouldResume())
    if err != nil {
        return err
    }
    mgr.mu.Lock()
    mgr.proc, mgr.w = proc, proc
    mgr.state = Booting
    mgr.mu.Unlock()
    go mgr.readPTY(proc)
    go mgr.watch(proc)
    mgr.bootWait() // flips Booting -> Ready
    return nil
}

// shouldResume reports whether a prior conversation exists to --continue. A
// death during the very first boot (no turn ever submitted, none completed)
// relaunches fresh; anything later resumes.
func (mgr *Manager) shouldResume() bool {
    mgr.mu.Lock()
    defer mgr.mu.Unlock()
    return mgr.current != "" || mgr.turnsCompleted > 0 || mgr.transcriptPath != ""
}

// resumeTurn re-submits the in-flight turn's prompt to the restored session,
// keeping the same turn id (so the eventual Stop hook still correlates) and the
// original timeout timer. No-op if the turn was resolved during relaunch.
func (mgr *Manager) resumeTurn(id string) {
    mgr.mu.Lock()
    if mgr.current != id || mgr.state != Ready {
        mgr.mu.Unlock()
        return
    }
    rec, ok := mgr.store.Get(id)
    if !ok || mgr.w == nil {
        mgr.mu.Unlock()
        return
    }
    seq := mgr.cfg.SubmitSeq
    if _, err := mgr.w.Write([]byte(seq.PasteStart + rec.Text + seq.PasteEnd)); err != nil {
        mgr.mu.Unlock()
        mgr.failTurn(id, fmt.Sprintf("resume write paste: %v", err))
        return
    }
    time.Sleep(mgr.cfg.SubmitDelay)
    if _, err := mgr.w.Write([]byte(seq.Submit)); err != nil {
        mgr.mu.Unlock()
        mgr.failTurn(id, fmt.Sprintf("resume write submit: %v", err))
        return
    }
    mgr.state = Busy
    mgr.mu.Unlock()
    mgr.log.Info("resumed in-flight turn after relaunch", "turn_id", id)
}