/**
 * Pipeline Engine — DAG-based workflow orchestration on top of the Task system.
 *
 * Workflow YAML → Pipeline instance → Nodes executed as Tasks
 * Supports: dependencies, output passing, conditional routing, parallel execution, notifications.
 */

import { randomUUID } from 'node:crypto';
import { existsSync, readdirSync, readFileSync, writeFileSync, mkdirSync, statSync, unlinkSync, rmSync } from 'node:fs';
import { execSync } from 'node:child_process';
import { join } from 'node:path';
import YAML from 'yaml';
import { createTask, getTask, onTaskEvent, taskModelOverrides, taskAppendSystemPromptOverrides, cancelTask } from './task-manager';
import { getProjectInfo, resolveOrCloneProject, getProjectWorktreeRoot } from './projects';
import { loadSettings } from './settings';
import { getAgent, listAgents } from './agents';
import type { Task } from '../src/types';
import { getDataDir } from './dirs';

const PIPELINES_DIR = join(getDataDir(), 'pipelines');
const WORKFLOWS_DIR = join(getDataDir(), 'flows');

// Track pipeline task IDs so terminal notifications can skip them (persists across hot-reloads)
const pipelineTaskKey = Symbol.for('mw-pipeline-task-ids');
const gPipeline = globalThis as any;
if (!gPipeline[pipelineTaskKey]) gPipeline[pipelineTaskKey] = new Set<string>();
export const pipelineTaskIds: Set<string> = gPipeline[pipelineTaskKey];

// ─── Types ────────────────────────────────────────────────

export interface WorkflowNode {
  id: string;
  project: string;
  prompt: string;
  mode?: 'claude' | 'shell' | 'plugin';  // default: 'claude', 'shell' runs command, 'plugin' runs plugin action
  agent?: string;              // agent ID (default: from settings)
  branch?: string;             // auto checkout this branch before running (supports templates)
  worktree?: boolean;          // default: true. Set false to skip worktree isolation (run in project dir directly)
  workdir?: string;            // explicit cwd override (template). Wins over auto-worktree. Use when one node
                               // produces a worktree path that a later node needs to execute inside —
                               // e.g. worktree-setup → fix-code via `workdir: "{{nodes.worktree-setup.outputs.wt}}"`.
  // Plugin mode fields
  plugin?: string;             // plugin ID (e.g., 'jenkins', 'docker')
  pluginAction?: string;       // action name (e.g., 'trigger', 'build'), defaults to plugin's defaultAction
  pluginParams?: Record<string, any>;  // per-use parameters
  pluginWait?: boolean;        // auto-run 'wait' action after main action
  dependsOn: string[];
  outputs: { name: string; extract: 'result' | 'git_diff' | 'stdout' | 'plugin' }[];
  routes: { condition: string; next: string }[];
  maxIterations: number;
  /** Auto-retry the node on transient failure. Default 0 (fail-fast,
   *  matches old behavior). E.g. retries: 2 → 1 initial + 2 retries =
   *  3 total attempts before the node is marked failed. Retries spawn
   *  a fresh task each time; output state is wiped between attempts.
   *  Use for nodes prone to transient errors: rate-limited API calls,
   *  flaky network, race-condition-prone shell ops. */
  retries?: number;
  /** Milliseconds to wait before each retry. Default 0 (immediate).
   *  Use 5000+ for downstream rate-limit recovery. */
  retryDelayMs?: number;
}

// ─── Conversation Mode Types ──────────────────────────────

export interface ConversationAgent {
  id: string;           // logical ID within this conversation (e.g., 'architect', 'implementer')
  agent: string;        // agent registry ID (e.g., 'claude', 'codex', 'aider')
  role: string;         // system prompt / role description
  project?: string;     // project context (optional, defaults to workflow input.project)
}

export interface ConversationMessage {
  round: number;
  agentId: string;      // logical ID from ConversationAgent
  agentName: string;    // display name (resolved from registry)
  content: string;
  timestamp: string;
  taskId?: string;      // backing task ID
  status: 'pending' | 'running' | 'done' | 'failed';
}

export interface ConversationConfig {
  agents: ConversationAgent[];
  maxRounds: number;           // max back-and-forth rounds
  stopCondition?: string;      // e.g., "all agents say DONE", "any agent says DONE"
  initialPrompt: string;       // the seed prompt to kick off the conversation
  contextStrategy?: 'full' | 'window' | 'summary';  // how to pass history, default: 'summary'
  contextWindow?: number;      // for 'window'/'summary': how many recent messages to include in full (default: 4)
  maxContentLength?: number;   // truncate each message to this length (default: 3000)
}

// ─── Workflow ─────────────────────────────────────────────

/** Type tag for an extended pipeline input field. */
export type WorkflowInputType = 'string' | 'integer' | 'number' | 'boolean' | 'enum';

/** Extended input field spec. Used when the yaml's `input:` block
 *  declares an object instead of a plain description string. */
export interface WorkflowInputFieldSpec {
  description?: string;
  label?: string;
  type?: WorkflowInputType;
  enum?: string[];           // for type: 'enum'
  required?: boolean;        // overrides the description-heuristic
  default?: string | number | boolean;
  multiline?: boolean;       // forces textarea regardless of description heuristic
}

/** A workflow input value is either a legacy description string
 *  (`input: { bug_id: "Mantis bug id" }`) or a full field spec
 *  (`input: { bug_id: { description: "…", type: integer, required: true } }`).
 *  Schedule UI / pipeline-schema endpoint normalize this into a uniform
 *  field record before rendering. */
export type WorkflowInputSpec = string | WorkflowInputFieldSpec;

/**
 * Workflow-level loop spec. Declaring this on a `dag` workflow turns the
 * pipeline into "run the whole DAG N times, once per item in `source`".
 * Each iteration's per-node state is fully reset between rounds, but the
 * pipeline_run id stays the same — i.e. it's ONE run, M iterations, not
 * M sibling runs. Designed for batch use cases (list of bug ids, list
 * of MR ids, …) where each item flows through the same node chain.
 *
 * Resolved + validated at startPipeline time:
 *   - `source` is templated against `input` / `vars` and parsed to an array
 *   - empty array → pipeline immediately settles to done (0 iterations)
 *
 * Inside the run, node prompts reference the current item via
 * `{{<asName>}}` (defaults to `{{item}}`) and current position via
 * `{{loop.index}}` / `{{loop.total}}`.
 *
 * Not supported (kept Non-Goals — see forge-pipeline-foreach-design.md):
 *   - nested for_each
 *   - parallel iterations (always sequential in v1)
 *   - dynamic source from an upstream node's output
 *   - cross-iteration output access
 */
export interface ForEachSpec {
  /** Templated string ("{{input.bug_ids}}") or literal array; resolved at
   *  startPipeline time (no `before:`) or after setup phase finishes (with
   *  `before:` — then source may reference `{{nodes.<id>.outputs.<name>}}`). */
  source: string | unknown[];
  /** Separator when `source` resolves to a string. Default ",". */
  split?: string;
  /** Variable name exposed inside nodes via `{{<asName>}}`. Default "item". */
  as?: string;
  /** Iteration-failure policy: "continue" (next iteration runs anyway, pipeline ends `failed` if any iter failed) or "stop" (first failure halts). Default "continue". */
  on_failure?: 'continue' | 'stop';
  /** Node ids that run ONCE before the loop body — for resolving items
   *  dynamically from upstream (e.g. a list-iids shell node). These nodes:
   *    - are scheduled first in isolation (loop-body nodes wait)
   *    - keep `done` status across all iterations
   *    - are excluded from per-iter snapshots + per-iter reset
   *  The for_each.source template can reference their outputs. */
  before?: string[];
}

export interface Workflow {
  name: string;
  type?: 'dag' | 'conversation';  // default: 'dag'
  description?: string;
  vars: Record<string, string>;
  input: Record<string, WorkflowInputSpec>;
  nodes: Record<string, WorkflowNode>;
  /** Loop over a list — each iteration runs the full DAG once. See ForEachSpec. */
  for_each?: ForEachSpec;
  // Conversation mode fields (only when type === 'conversation')
  conversation?: ConversationConfig;
}

/** Extract a description string from either input shape. */
export function inputDescription(spec: WorkflowInputSpec | undefined): string {
  if (!spec) return '';
  if (typeof spec === 'string') return spec;
  return spec.description || '';
}

/** Normalize any input spec into a uniform field record. */
export function normalizeInputField(name: string, spec: WorkflowInputSpec | undefined): {
  name: string;
  description: string;
  label: string;
  type: WorkflowInputType;
  enum: string[] | null;
  required: boolean;
  default: string | number | boolean | null;
  multiline: boolean;
} {
  if (!spec || typeof spec === 'string') {
    const description = typeof spec === 'string' ? spec : '';
    // Legacy heuristic: descriptions containing "optional" mean non-required.
    const required = !!description && !/optional|leave blank|leave empty/i.test(description);
    const multiline = /multi-line|multiline|prompt|description|body|template/i.test(description);
    return { name, description, label: '', type: 'string', enum: null, required, default: null, multiline };
  }
  const t: WorkflowInputType = (spec.type || 'string') as WorkflowInputType;
  return {
    name,
    description: spec.description || '',
    label: spec.label || '',
    type: t,
    enum: Array.isArray(spec.enum) ? spec.enum.map(String) : null,
    required: spec.required ?? true,
    default: spec.default ?? null,
    multiline: spec.multiline ?? false,
  };
}

export type PipelineNodeStatus = 'pending' | 'running' | 'done' | 'failed' | 'skipped' | 'cancelled';

export interface PipelineNodeState {
  status: PipelineNodeStatus;
  taskId?: string;
  outputs: Record<string, string>;
  /** Number of times THIS node's task has been launched. 1 = first
   *  attempt; 2,3,... = retries. Bumped each time we (re)create the
   *  task. Capped by WorkflowNode.retries (default 0 = no retry). */
  attempts?: number;
  iterations: number;
  startedAt?: string;
  completedAt?: string;
  error?: string;
}

export interface Pipeline {
  id: string;
  workflowName: string;
  type?: 'dag' | 'conversation';  // default: 'dag'
  status: 'running' | 'done' | 'failed' | 'cancelled';
  input: Record<string, string>;
  vars: Record<string, string>;
  nodes: Record<string, PipelineNodeState>;
  nodeOrder: string[];  // for UI display
  createdAt: string;
  completedAt?: string;
  /**
   * Skill names from the forge-skills registry to make available to
   * every task this pipeline spawns. Composed into the task's
   * --append-system-prompt. Carried in pipeline state so retries +
   * recovery use the same set as the original run.
   */
  skills?: string[];
  /**
   * Absolute path to this run's scratch dir, served to YAML nodes as
   * `{{run.tmp_dir}}`. Layout: `<project_dir>/.forge/worktrees/pipeline-<id>/`.
   * Resolved from `input.project`; empty / undefined when project lookup
   * failed (the template then renders as empty string).
   * `done` runs wipe this immediately; `failed`/`cancelled` keep it for
   * GC (see lib/pipeline-gc.ts).
   */
  tmpDir?: string;
  /**
   * Loop state when this pipeline run was started against a workflow that
   * declared `for_each:`. Absent on plain DAG runs (the orchestrator
   * branches on `pipeline.forEach !== undefined`).
   *
   * Lifecycle: `currentIndex` starts at 0; one iteration = one full DAG
   * pass. When all nodes settle, `iterations` gets a snapshot of the
   * round's node states, then everything resets and `currentIndex` ticks
   * up. The pipeline stays `running` until `currentIndex >= total`.
   */
  forEach?: {
    /** The resolved item list. Populated at startPipeline (no `before:`) or
     *  at the setup→loop transition (with `before:`). Empty during setup phase. */
    items: unknown[];
    /** 0-based index of the currently-executing iteration. */
    currentIndex: number;
    /** Cached items.length; redundant but cheap and makes UI math easy. */
    total: number;
    /** Variable name nodes use via `{{<asName>}}`. */
    asName: string;
    /** Set by startPipeline from workflow.for_each.on_failure. Default "continue". */
    onFailure: 'continue' | 'stop';
    /** Mirror of spec.before — frozen at start. Loop-setup nodes that run
     *  ONCE, never reset, never snapshot. Undefined or empty = no setup phase. */
    before?: string[];
    /** Phase flag: false during setup (before nodes still running or pending);
     *  true once items[] is final + loop can start. For backward compat,
     *  pipelines without `before:` are created with itemsResolved=true. */
    itemsResolved: boolean;
    /** Per-iteration history snapshots (oldest first). Appended at iteration boundary in checkPipelineCompletion. */
    iterations: Array<{
      index: number;
      status: 'done' | 'failed' | 'cancelled';
      startedAt: string;
      completedAt: string;
      /** Compact snapshot. `taskId` is retained so the UI can re-open
       *  the original task drawer for any past iteration's node. */
      nodes: Record<string, {
        status: PipelineNodeStatus;
        outputs: Record<string, string>;
        error?: string;
        taskId?: string;
      }>;
    }>;
  };
  // Conversation mode state
  conversation?: {
    config: ConversationConfig;
    messages: ConversationMessage[];
    currentRound: number;
    currentAgentIndex: number;  // index into config.agents
  };
}

// ─── Workflow Loading ─────────────────────────────────────

// All pipelines now live in the marketplace (`forge-workflow` repo).
// Empty BUILTIN_WORKFLOWS preserves the constant export so any external
// caller doesn't break, but no pipelines are shipped baked-in any more.
// To use the previously-builtin ones, install from marketplace:
//   - issue-fix-and-review
//   - multi-agent-collaboration
//   - review-mr
export const BUILTIN_WORKFLOWS: Record<string, string> = {};


export interface WorkflowWithMeta extends Workflow {
  builtin?: boolean;
}

export function listWorkflows(): WorkflowWithMeta[] {
  // User workflows
  const userWorkflows: WorkflowWithMeta[] = [];
  if (existsSync(WORKFLOWS_DIR)) {
    for (const f of readdirSync(WORKFLOWS_DIR).filter(f => f.endsWith('.yaml') || f.endsWith('.yml'))) {
      const fullPath = join(WORKFLOWS_DIR, f);
      try {
        userWorkflows.push({ ...parseWorkflow(readFileSync(fullPath, 'utf-8')), builtin: false });
      } catch (e) {
        // A broken yaml shouldn't crash the listing, but the user needs
        // to know WHY their pipeline isn't showing up. Common offender:
        // `outputs:` written as a map (`{name: "$STDOUT"}`) when the
        // parser expects an array (`- name: x; extract: stdout`) and
        // therefore .map() throws.
        console.warn(`[listWorkflows] skip ${fullPath}: ${(e as Error).message}`);
      }
    }
  }

  // Built-in workflows (don't override user ones with same name)
  const userNames = new Set(userWorkflows.map(w => w.name));
  const builtins: WorkflowWithMeta[] = [];
  for (const [key, yaml] of Object.entries(BUILTIN_WORKFLOWS)) {
    try {
      const w = parseWorkflow(yaml);
      if (!userNames.has(w.name)) {
        builtins.push({ ...w, builtin: true });
      }
    } catch (e) {
      console.warn(`[listWorkflows] builtin ${key} failed to parse: ${(e as Error).message}`);
    }
  }

  return [...builtins, ...userWorkflows];
}

export function getWorkflow(name: string): WorkflowWithMeta | null {
  return listWorkflows().find(w => w.name === name) || null;
}

/**
 * Reserved template namespaces — these names already mean something in
 * `{{...}}` substitution (input.x / vars.x / nodes.X.outputs.Y /
 * run.tmp_dir / raw: prefix / loop.index|total). Picking one of them as
 * the `for_each.as` variable would shadow it and silently break templates.
 */
const FOREACH_RESERVED_NAMES = new Set(['input', 'vars', 'nodes', 'run', 'raw', 'loop']);

/**
 * Parse + validate a workflow's `for_each:` block. Returns undefined when
 * the workflow doesn't declare it (the common case). Throws on malformed
 * input so authors find out at workflow load, not at first fire.
 */
function parseForEach(raw: any, workflowType: string): ForEachSpec | undefined {
  if (raw === undefined || raw === null) return undefined;
  if (workflowType !== 'dag') {
    throw new Error(`workflow.for_each is only supported on type='dag' workflows (got '${workflowType}')`);
  }
  if (typeof raw !== 'object' || Array.isArray(raw)) {
    throw new Error("workflow.for_each must be an object with at least { source: ... }");
  }
  const source = raw.source;
  if (source === undefined || source === null || source === '') {
    throw new Error("workflow.for_each.source is required (template like '{{input.bug_ids}}' or literal array)");
  }
  if (typeof source !== 'string' && !Array.isArray(source)) {
    throw new Error("workflow.for_each.source must be a template string or array literal");
  }
  const asName = raw.as ?? 'item';
  if (typeof asName !== 'string' || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(asName)) {
    throw new Error("workflow.for_each.as must be a valid identifier (letters / digits / underscore, starting with letter or _)");
  }
  if (FOREACH_RESERVED_NAMES.has(asName)) {
    throw new Error(`workflow.for_each.as='${asName}' collides with reserved template namespace (input / vars / nodes / run / raw / loop)`);
  }
  const onFailure = raw.on_failure ?? raw.onFailure ?? 'continue';
  if (onFailure !== 'continue' && onFailure !== 'stop') {
    throw new Error(`workflow.for_each.on_failure must be 'continue' or 'stop' (got '${onFailure}')`);
  }
  const split = raw.split;
  if (split !== undefined && typeof split !== 'string') {
    throw new Error("workflow.for_each.split must be a string");
  }
  let before: string[] | undefined;
  if (raw.before !== undefined && raw.before !== null) {
    if (!Array.isArray(raw.before) || raw.before.some((s: any) => typeof s !== 'string' || !s)) {
      throw new Error("workflow.for_each.before must be an array of node id strings");
    }
    // node-id existence validated in parseWorkflow once nodes are built.
    before = [...raw.before];
  }
  return {
    source,
    split,
    as: asName,
    on_failure: onFailure,
    before,
  };
}

export function parseWorkflow(raw: string): Workflow {
  const parsed = YAML.parse(raw);
  const workflowType = parsed.type || 'dag';
  const nodes: Record<string, WorkflowNode> = {};

  for (const [id, def] of Object.entries(parsed.nodes || {})) {
    const n = def as any;
    nodes[id] = {
      id,
      project: n.project || '',
      prompt: n.prompt || '',
      mode: n.mode || (n.plugin ? 'plugin' : 'claude'),
      agent: n.agent || undefined,
      branch: n.branch || undefined,
      worktree: n.worktree !== undefined ? n.worktree : undefined,
      workdir: n.workdir || undefined,
      plugin: n.plugin || undefined,
      pluginAction: n.plugin_action || n.pluginAction || undefined,
      pluginParams: n.plugin_params || n.pluginParams || n.params || undefined,
      pluginWait: n.plugin_wait || n.pluginWait || n.wait || false,
      dependsOn: n.depends_on || n.dependsOn || [],
      outputs: (n.outputs || []).map((o: any) => ({
        name: o.name,
        extract: o.extract || 'result',
      })),
      routes: (n.routes || []).map((r: any) => ({
        condition: r.condition || 'default',
        next: r.next,
      })),
      maxIterations: n.max_iterations || n.maxIterations || 3,
      retries: Number.isFinite(Number(n.retries)) ? Math.max(0, Math.trunc(Number(n.retries))) : 0,
      retryDelayMs: Number.isFinite(Number(n.retry_delay_ms ?? n.retryDelayMs))
        ? Math.max(0, Math.trunc(Number(n.retry_delay_ms ?? n.retryDelayMs)))
        : 0,
    };
  }

  // Parse conversation config
  let conversation: ConversationConfig | undefined;
  if (workflowType === 'conversation' && parsed.agents) {
    conversation = {
      agents: (parsed.agents as any[]).map((a: any) => ({
        id: a.id,
        agent: a.agent || 'claude',
        role: a.role || '',
        project: a.project || undefined,
      })),
      maxRounds: parsed.max_rounds || parsed.maxRounds || 10,
      stopCondition: parsed.stop_condition || parsed.stopCondition || undefined,
      initialPrompt: parsed.initial_prompt || parsed.initialPrompt || '',
      contextStrategy: parsed.context_strategy || parsed.contextStrategy || 'summary',
      contextWindow: parsed.context_window || parsed.contextWindow || 4,
      maxContentLength: parsed.max_content_length || parsed.maxContentLength || 3000,
    };
  }

  const for_each = parseForEach(parsed.for_each, workflowType);
  if (for_each?.before) {
    for (const id of for_each.before) {
      if (!nodes[id]) {
        throw new Error(`workflow.for_each.before references unknown node id '${id}' — must match one of: ${Object.keys(nodes).join(', ') || '(no nodes)'}`);
      }
    }
  }

  return {
    name: parsed.name || 'unnamed',
    type: workflowType,
    description: parsed.description,
    vars: parsed.vars || {},
    input: parsed.input || {},
    nodes,
    for_each,
    conversation,
  };
}

// ─── Pipeline Persistence ─────────────────────────────────

function ensureDir() {
  if (!existsSync(PIPELINES_DIR)) mkdirSync(PIPELINES_DIR, { recursive: true });
}

function savePipeline(pipeline: Pipeline) {
  ensureDir();
  writeFileSync(join(PIPELINES_DIR, `${pipeline.id}.json`), JSON.stringify(pipeline, null, 2));
}

export function getPipeline(id: string): Pipeline | null {
  try {
    return JSON.parse(readFileSync(join(PIPELINES_DIR, `${id}.json`), 'utf-8'));
  } catch {
    return null;
  }
}

export function deletePipeline(id: string): boolean {
  const filePath = join(PIPELINES_DIR, `${id}.json`);
  try {
    if (existsSync(filePath)) {
      unlinkSync(filePath);
      __pipelineCache.delete(filePath);
      return true;
    }
  } catch (e) {
    console.warn(`[pipeline] deletePipeline ${id} failed: ${(e as Error).message}`);
  }
  return false;
}

/** Bulk-delete terminal pipelines created before `beforeIso`. Skips
 *  anything still in 'running' or 'pending' so we never blow away
 *  state from a live run. Returns count removed. */
export interface BulkDeletePipelinesFilter {
  before: string;  // ISO 8601
  statuses?: Array<'done' | 'failed' | 'cancelled'>;
}

export function bulkDeletePipelines(filter: BulkDeletePipelinesFilter): number {
  const wanted = new Set(filter.statuses && filter.statuses.length
    ? filter.statuses
    : ['done', 'failed', 'cancelled']);
  const cutoff = new Date(filter.before).getTime();
  if (!Number.isFinite(cutoff)) return 0;

  let removed = 0;
  for (const p of listPipelines()) {
    if (!wanted.has(p.status as any)) continue;
    const createdMs = new Date(p.createdAt).getTime();
    if (!Number.isFinite(createdMs) || createdMs >= cutoff) continue;
    if (deletePipeline(p.id)) removed++;
  }
  return removed;
}

// Parsed-pipeline cache keyed by absolute path → { mtime, pipeline }.
// Pipeline files only change when a pipeline run advances (savePipeline) —
// re-reading + JSON.parse on every list call was burning 1-3s with 200
// runs accumulated (largest file 136KB, conversation history + node
// outputs blow up). With this cache, subsequent listPipelines() calls
// only re-parse files whose mtime changed since the last read.
const __pipelineCache = new Map<string, { mtimeMs: number; pipeline: Pipeline }>();

export function listPipelines(): Pipeline[] {
  ensureDir();
  const files = readdirSync(PIPELINES_DIR).filter(f => f.endsWith('.json'));
  const out: Pipeline[] = [];
  for (const f of files) {
    const path = join(PIPELINES_DIR, f);
    try {
      const stat = statSync(path);
      const cached = __pipelineCache.get(path);
      if (cached && cached.mtimeMs === stat.mtimeMs) {
        out.push(cached.pipeline);
        continue;
      }
      const pipeline = JSON.parse(readFileSync(path, 'utf-8')) as Pipeline;
      __pipelineCache.set(path, { mtimeMs: stat.mtimeMs, pipeline });
      out.push(pipeline);
    } catch { /* skip unreadable */ }
  }
  return out;
}

/**
 * Light summary for the pipeline LIST UI — strips the heavy fields
 * (node outputs, conversation message bodies, errors). The full pipeline
 * is fetched via GET /api/pipelines/:id when the user opens a specific
 * run. Cuts /api/pipelines payload from ~1.6MB to <50KB with 200 runs.
 */
export interface PipelineSummary {
  id: string;
  workflowName: string;
  status: Pipeline['status'];
  type?: Pipeline['type'];
  createdAt: string;
  completedAt?: string;
  nodeOrder: string[];
  nodes: Record<string, { status: PipelineNodeState['status']; iterations?: number }>;
  conversation?: { currentRound?: number; config?: { maxRounds?: number } };
  input?: Record<string, string>;
}

export interface ListPipelinesOptions {
  /** Newest-first limit. Defaults to 100. Hard ceiling 500 (the heavy
   *  pipeline detail view fetches one run at a time anyway, so a
   *  five-hundred-deep list covers every practical workflow grouping). */
  limit?: number;
  /** Filter to a single workflow. Combine with a higher limit when the
   *  user expands one workflow and wants its full history. */
  workflow?: string;
  /** Cursor for "Load more" — return runs with createdAt < before. */
  before?: string;
}

export function listPipelinesSummary(opts: ListPipelinesOptions = {}): PipelineSummary[] {
  const limit = Math.min(Math.max(1, opts.limit ?? 100), 500);
  // listPipelines already builds parsed objects (cached). Sort by
  // createdAt desc, optionally filter, then slice. Avoids serializing the
  // hundreds of older runs the UI never shows.
  let all = listPipelines();
  if (opts.workflow) {
    const target = opts.workflow;
    all = all.filter(p => p.workflowName === target);
  }
  all.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
  if (opts.before) all = all.filter(p => p.createdAt < opts.before!);
  const slice = all.slice(0, limit);
  return slice.map(p => ({
    id: p.id,
    workflowName: p.workflowName,
    status: p.status,
    type: p.type,
    createdAt: p.createdAt,
    completedAt: p.completedAt,
    nodeOrder: p.nodeOrder,
    nodes: Object.fromEntries(
      Object.entries(p.nodes).map(([k, v]) => [k, { status: v.status, iterations: v.iterations }]),
    ),
    conversation: p.conversation
      ? { currentRound: p.conversation.currentRound, config: { maxRounds: p.conversation.config?.maxRounds } }
      : undefined,
    input: p.input,
  }));
}

// ─── Template Resolution ──────────────────────────────────

/** Escape a string for safe embedding in single-quoted shell strings */
function shellEscape(s: string): string {
  // Replace single quotes with '\'' (end quote, escaped quote, start quote)
  return s.replace(/'/g, "'\\''");
}

/** Escape a string for safe embedding in $'...' shell strings (ANSI-C quoting) */
function shellEscapeAnsiC(s: string): string {
  return s
    .replace(/\\/g, '\\\\')
    .replace(/'/g, "\\'")
    .replace(/\n/g, '\\n')
    .replace(/\r/g, '\\r')
    .replace(/\t/g, '\\t');
}

/** Stringify a for_each item for template substitution. Scalars become
 *  their natural string; objects/arrays JSON-stringified (callers usually
 *  pull out a sub-field with `{{<asName>.foo}}` instead). */
function stringifyForeachItem(v: unknown): string {
  if (v === null || v === undefined) return '';
  if (typeof v === 'string') return v;
  if (typeof v === 'number' || typeof v === 'boolean') return String(v);
  try { return JSON.stringify(v); } catch { return String(v); }
}

export function resolveTemplate(template: string, ctx: {
  input: Record<string, string>;
  vars: Record<string, string>;
  nodes: Record<string, PipelineNodeState>;
  tmpDir?: string;
  forEach?: {
    asName: string;
    item: unknown;
    index: number;
    total: number;
  };
}, shellMode?: boolean): string {
  return template.replace(/\{\{(.*?)\}\}/g, (_, expr) => {
    let path = expr.trim();

    // `{{raw:…}}` opts out of shell-mode ANSI-C escaping so the value
    // is substituted verbatim. Use this when the value lands in a
    // quoted heredoc, a file written by `cat <<EOF`, or stdin to jq —
    // anywhere the default `\n → \\n` escaping would surface as literal
    // backslash-n in the final output (LLM-generated multi-line text
    // posted as comments / chat seeds, etc.).
    let raw = false;
    if (path.startsWith('raw:')) {
      raw = true;
      path = path.slice(4).trim();
    }

    let value = '';

    // {{input.xxx}}
    if (path.startsWith('input.')) value = ctx.input[path.slice(6)] || '';
    // {{vars.xxx}}
    else if (path.startsWith('vars.')) value = ctx.vars[path.slice(5)] || '';
    // {{run.tmp_dir}} — absolute path to this pipeline run's scratch dir
    // (`<project_dir>/.forge/worktrees/pipeline-<id>/`). Empty if project
    // lookup failed at startPipeline time. Yaml is responsible for mkdir-ing
    // any sub-paths like `{{run.tmp_dir}}/mr-${MR_IID}`.
    else if (path === 'run.tmp_dir') value = ctx.tmpDir || '';
    // {{loop.index}} — current iteration (0-based), only set when this
    // pipeline run was started against a for_each workflow.
    else if (ctx.forEach && path === 'loop.index') value = String(ctx.forEach.index);
    // {{loop.total}} — total iterations the pipeline will run.
    else if (ctx.forEach && path === 'loop.total') value = String(ctx.forEach.total);
    // {{<asName>}} — current item itself (scalar → str, object → JSON).
    else if (ctx.forEach && path === ctx.forEach.asName) {
      value = stringifyForeachItem(ctx.forEach.item);
    }
    // {{<asName>.foo}} — sub-field of object item. Returns empty string
    // (not literal placeholder) on missing field so node prompts can
    // safely conditionalize on whether the field was set.
    else if (ctx.forEach && path.startsWith(ctx.forEach.asName + '.')) {
      const field = path.slice(ctx.forEach.asName.length + 1);
      const item = ctx.forEach.item;
      const v = (item !== null && typeof item === 'object') ? (item as any)[field] : undefined;
      value = stringifyForeachItem(v);
    }
    // {{nodes.xxx.outputs.yyy}}
    else {
      const nodeMatch = path.match(/^nodes\.([\w-]+)\.outputs\.([\w-]+)$/);
      if (nodeMatch) {
        const [, nodeId, outputName] = nodeMatch;
        value = ctx.nodes[nodeId]?.outputs[outputName] || '';
      } else {
        return `{{${raw ? 'raw:' : ''}${path}}}`;
      }
    }

    return shellMode && !raw ? shellEscapeAnsiC(value) : value;
  });
}

// ─── Per-run scratch dir (`{{run.tmp_dir}}`) ────────────────

/**
 * Compute the absolute scratch-dir path for a pipeline run. Returns
 * empty string if input.project doesn't resolve. Does NOT mkdir.
 */
function computePipelineTmpDir(pipeline: Pipeline): string {
  const name = pipeline.input?.project;
  if (!name) return '';
  const proj = getProjectInfo(name);
  if (!proj) return '';
  // Worktree root differs for scratch vs real projects — see getProjectWorktreeRoot.
  return join(getProjectWorktreeRoot(proj), `pipeline-${pipeline.id}`);
}

/**
 * Create the scratch dir at pipeline start. Idempotent. Failure is
 * non-fatal — `tmpDir` stays unset and `{{run.tmp_dir}}` renders empty.
 */
function ensurePipelineTmpDir(pipeline: Pipeline): void {
  const dir = computePipelineTmpDir(pipeline);
  if (!dir) return;
  try {
    mkdirSync(dir, { recursive: true });
    pipeline.tmpDir = dir;
  } catch (e) {
    console.warn(`[pipeline] could not create tmp_dir ${dir}: ${(e as Error).message}`);
  }
}

/**
 * Remove the scratch dir. Called on `done` (if settings allow) and
 * by the GC sweep for expired failed/cancelled runs.
 */
export function cleanupPipelineTmpDir(pipeline: Pipeline): void {
  const dir = pipeline.tmpDir || computePipelineTmpDir(pipeline);
  if (!dir || !existsSync(dir)) return;
  try {
    rmSync(dir, { recursive: true, force: true });
  } catch (e) {
    console.warn(`[pipeline] cleanupPipelineTmpDir(${dir}) failed: ${(e as Error).message}`);
  }
}

// ─── for_each: source resolution ──────────────────────────

/**
 * Resolve a workflow's `for_each.source` into a concrete array of items.
 * Called once at startPipeline time — the items list is frozen for the
 * lifetime of the run (no dynamic re-resolution per iteration).
 *
 * Rules:
 *   - source is array → used directly
 *   - source is string → resolveTemplate against (input, vars, nodes), then split
 *     by `spec.split` (default ",") + trim + drop empties
 *   - anything else → []
 *
 * `nodes` enables `{{nodes.<id>.outputs.<name>}}` references — required when
 * the workflow uses `for_each.before:` to compute items dynamically. Optional
 * for the simple case (template references only input/vars).
 */
export function resolveForEachSource(
  spec: ForEachSpec,
  input: Record<string, string>,
  vars: Record<string, string>,
  nodes?: Record<string, PipelineNodeState>,
): unknown[] {
  let raw: unknown = spec.source;
  if (typeof raw === 'string') {
    raw = resolveTemplate(raw, { input, vars, nodes: nodes || {} }, false);
  }
  if (Array.isArray(raw)) {
    return raw;
  }
  if (typeof raw === 'string') {
    const sep = spec.split ?? ',';
    return raw.split(sep).map((s) => s.trim()).filter((s) => s.length > 0);
  }
  return [];
}

// ─── Project-level pipeline lock ─────────────────────────
const projectPipelineLocks = new Map<string, string>(); // projectPath → pipelineId

function acquireProjectLock(projectPath: string, pipelineId: string): boolean {
  const existing = projectPipelineLocks.get(projectPath);
  if (existing && existing !== pipelineId) {
    // Check if the existing pipeline is still running
    const p = getPipeline(existing);
    if (p && p.status === 'running') return false;
    // Stale lock, clear it
  }
  projectPipelineLocks.set(projectPath, pipelineId);
  return true;
}

function releaseProjectLock(projectPath: string, pipelineId: string) {
  if (projectPipelineLocks.get(projectPath) === pipelineId) {
    projectPipelineLocks.delete(projectPath);
  }
}

// ─── Pipeline Execution ───────────────────────────────────

/**
 * Render the skills list into an --append-system-prompt block. Empty
 * input → empty string (caller passes undefined to skip the flag).
 */
function renderSkillsAppendPrompt(skills: string[] | undefined): string {
  if (!skills || skills.length === 0) return '';
  const lines = skills.map((s) => `  /${s}`).join('\n');
  return [
    'For this task, the following Forge skills are available and should be used as appropriate:',
    lines,
    "Read each skill's SKILL.md before invoking. Prefer invoking these skills (with /<name>) over reimplementing their workflows inline.",
  ].join('\n');
}

export function startPipeline(
  workflowName: string,
  input: Record<string, string>,
  opts: { skills?: string[] } = {},
): Pipeline {
  const workflow = getWorkflow(workflowName);
  if (!workflow) throw new Error(`Workflow not found: ${workflowName}`);

  // Apply declared defaults for any field the caller didn't pass. UI callers
  // (Schedule form, Fire dialog) already pre-fill all fields with their
  // defaults at form-load time, so for them this merge is a no-op. Sparse-input
  // callers (chat trigger_pipeline, MCP, direct API) only send what the user
  // explicitly named; without this merge multi-line defaults like
  // mr_body_template / user_prompt come through as empty → "{{input.X}}"
  // renders to '' → MR body empty / triage prompt empty.
  //
  // Semantics: ONLY fill when the key is missing from the input map. An
  // empty-string value is respected as "intentionally cleared" so schedule
  // users who clear a field in the form keep that behaviour.
  for (const [key, spec] of Object.entries(workflow.input || {})) {
    if (Object.prototype.hasOwnProperty.call(input, key)) continue;
    if (!spec || typeof spec !== 'object') continue;
    const def = (spec as WorkflowInputFieldSpec).default;
    if (def == null) continue;
    input = { ...input, [key]: String(def) };
  }

  // Conversation mode — separate execution path
  if (workflow.type === 'conversation' && workflow.conversation) {
    return startConversationPipeline(workflow, input);
  }

  const id = randomUUID().slice(0, 8);
  const nodes: Record<string, PipelineNodeState> = {};
  const nodeOrder = topologicalSort(workflow.nodes);

  for (const nodeId of nodeOrder) {
    nodes[nodeId] = {
      status: 'pending',
      outputs: {},
      iterations: 0,
    };
  }

  // Resolve for_each (if declared). Two paths:
  //   - No `before:` → items resolved immediately from input/vars (frozen).
  //   - With `before:` → defer; setup-phase nodes run first, then
  //     checkPipelineCompletion resolves items using their outputs and flips
  //     itemsResolved=true to enter the loop body.
  let forEachState: Pipeline['forEach'];
  if (workflow.for_each) {
    const beforeIds = workflow.for_each.before;
    if (beforeIds && beforeIds.length > 0) {
      forEachState = {
        items: [],
        currentIndex: 0,
        total: 0,
        asName: workflow.for_each.as || 'item',
        onFailure: workflow.for_each.on_failure || 'continue',
        before: [...beforeIds],
        itemsResolved: false,
        iterations: [],
      };
    } else {
      const items = resolveForEachSource(workflow.for_each, input, workflow.vars);
      forEachState = {
        items,
        currentIndex: 0,
        total: items.length,
        asName: workflow.for_each.as || 'item',
        onFailure: workflow.for_each.on_failure || 'continue',
        itemsResolved: true,
        iterations: [],
      };
    }
  }

  const pipeline: Pipeline = {
    id,
    workflowName,
    status: 'running',
    input,
    vars: { ...workflow.vars },
    nodes,
    nodeOrder,
    createdAt: new Date().toISOString(),
    skills: opts.skills && opts.skills.length ? [...opts.skills] : undefined,
    forEach: forEachState,
  };

  ensurePipelineTmpDir(pipeline);

  // Empty for_each source → nothing to iterate; settle done immediately.
  // (E.g. user submitted `bug_ids: ""` or an empty array; not an error.)
  // Deferred (with `before:`) skips this — items get resolved post-setup.
  if (forEachState && forEachState.itemsResolved && forEachState.total === 0) {
    pipeline.status = 'done';
    pipeline.completedAt = new Date().toISOString();
    savePipeline(pipeline);
    // Mirror checkPipelineCompletion's done-side cleanup so an empty
    // for_each behaves consistently with a normal done pipeline.
    if (loadSettings().pipelineTmpCleanDoneImmediate !== false) {
      cleanupPipelineTmpDir(pipeline);
    }
    setupTaskListener(pipeline.id);
    return pipeline;
  }

  savePipeline(pipeline);

  // Start nodes that have no dependencies
  scheduleReadyNodes(pipeline, workflow);

  // Listen for task completions
  setupTaskListener(pipeline.id);

  return pipeline;
}

// ─── Conversation State Type (extracted to avoid Turbopack parse issues) ──
type ConversationState = {
  config: ConversationConfig;
  messages: ConversationMessage[];
  currentRound: number;
  currentAgentIndex: number;
};

// ─── Conversation Mode Execution ──────────────────────────

function startConversationPipeline(workflow: Workflow, input: Record<string, string>): Pipeline {
  const conv = workflow.conversation!;
  const id = randomUUID().slice(0, 8);

  // Resolve agent display names
  const agentNames: Record<string, string> = {};
  const allAgents = listAgents();
  for (const ca of conv.agents) {
    const found = allAgents.find(a => a.id === ca.agent);
    agentNames[ca.id] = found?.name || ca.agent;
  }

  const pipeline: Pipeline = {
    id,
    workflowName: workflow.name,
    type: 'conversation',
    status: 'running',
    input,
    vars: { ...workflow.vars },
    nodes: {},
    nodeOrder: [],
    createdAt: new Date().toISOString(),
    conversation: {
      config: {
        ...conv,
        // Store resolved initial prompt so buildConversationContext uses it
        initialPrompt: conv.initialPrompt.replace(/\{\{input\.(\w+)\}\}/g, (_, key) => input[key] || ''),
      },
      messages: [],
      currentRound: 1,
      currentAgentIndex: 0,
    },
  };

  savePipeline(pipeline);

  const resolvedPrompt = pipeline.conversation!.config.initialPrompt;

  // Start the first round
  scheduleNextConversationTurn(pipeline, resolvedPrompt, agentNames);

  return pipeline;
}

function scheduleNextConversationTurn(pipeline: Pipeline, contextForAgent: string, agentNames?: Record<string, string>) {
  const conv = pipeline.conversation!;
  const config = conv.config;
  const agentDef = config.agents[conv.currentAgentIndex];

  if (!agentDef) {
    pipeline.status = 'failed';
    pipeline.completedAt = new Date().toISOString();
    savePipeline(pipeline);
    return;
  }

  // Resolve project — three-tier fallback in lib/projects.ts:
  //   1. existing project by name
  //   2. auto-clone GitLab default_project_path → projectRoots[0] / ~/IdeaProjects
  //   3. scratch (<dataDir>/scratch, always writable)
  // Never fails. The scratch fallback means a pipeline that needs no real
  // repo still runs (connector-only flows, talk-only agents); pipelines that
  // truly need source files will fail later inside their own nodes with a
  // domain-specific error, not a "Project not found".
  const projectName = agentDef.project || pipeline.input.project || '';
  const resolved = resolveOrCloneProject(projectName);
  const projectInfo = resolved.project;
  if (resolved.source !== 'existing') {
    console.log(`[pipeline ${pipeline.id.slice(0,8)}] project resolved via ${resolved.source}: ${projectInfo.path}`);
  }

  // Build the prompt: role context + conversation history + new message
  const rolePrefix = agentDef.role ? `[Your role: ${agentDef.role}]\n\n` : '';
  const fullPrompt = `${rolePrefix}${contextForAgent}`;

  // Create a task for this agent's turn
  const task = createTask({
    projectName: projectInfo.name,
    projectPath: projectInfo.path,
    prompt: fullPrompt,
    mode: 'prompt',
    agent: agentDef.agent,
    conversationId: '', // fresh session — no resume for conversation mode
  });
  pipelineTaskIds.add(task.id);
  {
    const skillsAppend = renderSkillsAppendPrompt(pipeline.skills);
    if (skillsAppend) taskAppendSystemPromptOverrides.set(task.id, skillsAppend);
  }

  // Add pending message
  const names = agentNames || resolveAgentNames(config.agents);
  conv.messages.push({
    round: conv.currentRound,
    agentId: agentDef.id,
    agentName: names[agentDef.id] || agentDef.agent,
    content: '',
    timestamp: new Date().toISOString(),
    taskId: task.id,
    status: 'running',
  });

  savePipeline(pipeline);

  // Listen for this task to complete
  setupConversationTaskListener(pipeline.id, task.id);
}

function resolveAgentNames(agents: ConversationAgent[]): Record<string, string> {
  const allAgents = listAgents();
  const names: Record<string, string> = {};
  for (const ca of agents) {
    const found = allAgents.find(a => a.id === ca.agent);
    names[ca.id] = found?.name || ca.agent;
  }
  return names;
}

function setupConversationTaskListener(pipelineId: string, taskId: string) {
  const cleanup = onTaskEvent((evtTaskId, event, data) => {
    if (evtTaskId !== taskId) return;
    if (event !== 'status') return;
    if (data !== 'done' && data !== 'failed') return;

    cleanup(); // one-shot listener

    const pipeline = getPipeline(pipelineId);
    if (!pipeline || pipeline.status !== 'running' || !pipeline.conversation) return;

    const conv = pipeline.conversation;
    const config = conv.config;
    const msgIndex = conv.messages.findIndex(m => m.taskId === taskId);
    if (msgIndex < 0) return;

    const task = getTask(taskId);

    if (data === 'failed' || !task) {
      conv.messages[msgIndex].status = 'failed';
      conv.messages[msgIndex].content = task?.error || 'Task failed';
      pipeline.status = 'failed';
      pipeline.completedAt = new Date().toISOString();
      savePipeline(pipeline);
      notifyPipelineComplete(pipeline);
      return;
    }

    // Task completed — extract response
    const response = task.resultSummary || '';
    conv.messages[msgIndex].status = 'done';
    conv.messages[msgIndex].content = response;

    // Check stop condition
    if (checkConversationStopCondition(conv, response)) {
      finishConversation(pipeline, 'done');
      return;
    }

    // Move to next agent in round, or next round
    conv.currentAgentIndex++;
    if (conv.currentAgentIndex >= config.agents.length) {
      // Completed a full round
      conv.currentAgentIndex = 0;
      conv.currentRound++;

      if (conv.currentRound > config.maxRounds) {
        finishConversation(pipeline, 'done');
        return;
      }
    }

    savePipeline(pipeline);

    // Build context for next agent: accumulate conversation history
    const contextForNext = buildConversationContext(conv);
    scheduleNextConversationTurn(pipeline, contextForNext);
  });
}

/**
 * Build context string for the next agent in conversation.
 *
 * Three strategies:
 *   - 'full'    : pass ALL history as-is (token-heavy, good for short convos)
 *   - 'window'  : pass only the last N messages in full, drop older ones
 *   - 'summary' : pass older messages as one-line summaries + last N in full (default)
 */
function buildConversationContext(conv: ConversationState): string {
  const config = conv.config;
  const strategy = config.contextStrategy || 'summary';
  const windowSize = config.contextWindow || 4;
  const maxLen = config.maxContentLength || 3000;

  const doneMessages = conv.messages.filter(m => m.status === 'done' && m.content);

  let context = `[Conversation — Round ${conv.currentRound}]\n\n`;
  context += `Task: ${config.initialPrompt}\n\n`;

  if (doneMessages.length === 0) {
    context += `--- Your Turn ---\nYou are the first to respond. Please address the task above. If you believe the task is complete, include "DONE" in your response.`;
    return context;
  }

  context += `--- Conversation History ---\n\n`;

  if (strategy === 'full') {
    // Full: all messages, truncated per maxLen
    for (const msg of doneMessages) {
      context += formatMessage(msg, config, maxLen);
    }
  } else if (strategy === 'window') {
    // Window: only last N messages
    const recent = doneMessages.slice(-windowSize);
    if (doneMessages.length > windowSize) {
      context += `[... ${doneMessages.length - windowSize} earlier messages omitted ...]\n\n`;
    }
    for (const msg of recent) {
      context += formatMessage(msg, config, maxLen);
    }
  } else {
    // Summary (default): older messages as one-line summaries, recent in full
    const cutoff = doneMessages.length - windowSize;
    if (cutoff > 0) {
      context += `[Previous rounds summary]\n`;
      for (let i = 0; i < cutoff; i++) {
        const msg = doneMessages[i];
        const summary = extractSummaryLine(msg.content);
        context += `  R${msg.round} ${msg.agentName}: ${summary}\n`;
      }
      context += `\n`;
    }
    // Recent messages in full
    const recent = doneMessages.slice(Math.max(0, cutoff));
    for (const msg of recent) {
      context += formatMessage(msg, config, maxLen);
    }
  }

  context += `--- Your Turn ---\nRespond based on the conversation above. If you believe the task is complete, include "DONE" in your response.`;
  return context;
}

function formatMessage(msg: ConversationMessage, config: ConversationConfig, maxLen: number): string {
  const agentDef = config.agents.find(a => a.id === msg.agentId);
  const label = `${msg.agentName} (${agentDef?.id || '?'})`;
  const content = msg.content.length > maxLen
    ? msg.content.slice(0, maxLen) + '\n[... truncated]'
    : msg.content;
  return `[${label} — Round ${msg.round}]:\n${content}\n\n`;
}

/** Extract a one-line summary from agent output (first meaningful line or first 120 chars) */
function extractSummaryLine(content: string): string {
  const lines = content.split('\n').map(l => l.trim()).filter(l => l.length > 10);
  const first = lines[0] || content.slice(0, 120);
  return first.length > 120 ? first.slice(0, 117) + '...' : first;
}

function checkConversationStopCondition(conv: ConversationState, latestResponse: string): boolean {
  const condition = conv.config.stopCondition;
  if (!condition) return false;

  const lower = condition.toLowerCase();

  // "any agent says DONE"
  if (lower.includes('any') && lower.includes('done')) {
    return latestResponse.toUpperCase().includes('DONE');
  }

  // "all agents say DONE" / "both agents say DONE"
  if ((lower.includes('all') || lower.includes('both')) && lower.includes('done')) {
    // Only check messages from the CURRENT round — don't mix rounds
    const currentRound = conv.currentRound;
    const agentIds = conv.config.agents.map(a => a.id);
    const roundMessages = new Map<string, string>();
    for (const msg of conv.messages) {
      if (msg.status === 'done' && msg.round === currentRound && msg.agentId !== 'user') {
        roundMessages.set(msg.agentId, msg.content);
      }
    }
    // All agents in this round must have responded AND said DONE
    return agentIds.every(id => {
      const content = roundMessages.get(id);
      return content && content.toUpperCase().includes('DONE');
    });
  }

  // Default: check if latest response contains DONE
  return latestResponse.toUpperCase().includes('DONE');
}

/** Cleanly finish a conversation — cancel any still-running tasks, mark messages */
function finishConversation(pipeline: Pipeline, status: 'done' | 'failed') {
  const conv = pipeline.conversation!;
  for (const msg of conv.messages) {
    if (msg.status === 'running' && msg.taskId) {
      // Cancel the running task — may already be finished, ignore those
      try { cancelTask(msg.taskId); } catch (e) {
        console.warn(`[pipeline] cancelTask(${msg.taskId}) in finishConversation: ${(e as Error).message}`);
      }
      msg.status = status === 'done' ? 'done' : 'failed';
      if (!msg.content) msg.content = status === 'done' ? '(conversation ended)' : '(conversation failed)';
    }
    if (msg.status === 'pending') {
      msg.status = 'failed';
    }
  }
  pipeline.status = status;
  pipeline.completedAt = new Date().toISOString();
  savePipeline(pipeline);
  notifyPipelineComplete(pipeline);
}

/** Cancel a conversation pipeline */
export function cancelConversation(pipelineId: string): boolean {
  const pipeline = getPipeline(pipelineId);
  if (!pipeline || pipeline.status !== 'running' || !pipeline.conversation) return false;

  // Cancel any running task
  for (const msg of pipeline.conversation.messages) {
    if (msg.status === 'running' && msg.taskId) {
      cancelTask(msg.taskId);
    }
    if (msg.status === 'pending') msg.status = 'failed';
  }

  pipeline.status = 'cancelled';
  pipeline.completedAt = new Date().toISOString();
  savePipeline(pipeline);
  return true;
}

/**
 * Inject a user message into a running conversation.
 * Waits for current agent to finish, then sends the injected message
 * as additional context to the specified agent on the next turn.
 */
export function injectConversationMessage(pipelineId: string, targetAgentId: string, message: string): boolean {
  const pipeline = getPipeline(pipelineId);
  if (!pipeline || pipeline.status !== 'running' || !pipeline.conversation) {
    throw new Error('Pipeline not running or not a conversation');
  }

  const conv = pipeline.conversation;
  const agentDef = conv.config.agents.find(a => a.id === targetAgentId);
  if (!agentDef) throw new Error(`Agent not found: ${targetAgentId}`);

  // Add a "user" message to the conversation
  conv.messages.push({
    round: conv.currentRound,
    agentId: 'user',
    agentName: 'Operator',
    content: `[@${targetAgentId}] ${message}`,
    timestamp: new Date().toISOString(),
    status: 'done',
  });

  savePipeline(pipeline);

  // If no agent is currently running, immediately schedule the target agent
  const hasRunning = conv.messages.some(m => m.status === 'running');
  if (!hasRunning) {
    // Point to the target agent for next turn
    const targetIdx = conv.config.agents.findIndex(a => a.id === targetAgentId);
    if (targetIdx >= 0) {
      conv.currentAgentIndex = targetIdx;
      savePipeline(pipeline);
      const context = buildConversationContext(conv);
      scheduleNextConversationTurn(pipeline, context);
    }
  }
  // If an agent IS running, the injected message will be included in the next context build

  return true;
}

// ─── Conversation Recovery ────────────────────────────────

function recoverConversationPipeline(pipeline: Pipeline) {
  const conv = pipeline.conversation!;
  const runningMsg = conv.messages.find(m => m.status === 'running');
  if (!runningMsg || !runningMsg.taskId) return;

  const task = getTask(runningMsg.taskId);
  if (!task) {
    // Task gone — mark as done with empty content, try next turn
    runningMsg.status = 'done';
    runningMsg.content = '(no response — task was cleaned up)';
    savePipeline(pipeline);
    advanceConversation(pipeline);
    return;
  }
  if (task.status === 'done') {
    runningMsg.status = 'done';
    runningMsg.content = task.resultSummary || '';
    savePipeline(pipeline);
    advanceConversation(pipeline);
  } else if (task.status === 'failed' || task.status === 'cancelled') {
    runningMsg.status = 'failed';
    runningMsg.content = task.error || 'Task failed';
    pipeline.status = 'failed';
    pipeline.completedAt = new Date().toISOString();
    savePipeline(pipeline);
  } else {
    // Still running — re-attach listener
    setupConversationTaskListener(pipeline.id, runningMsg.taskId);
  }
}

function advanceConversation(pipeline: Pipeline) {
  const conv = pipeline.conversation!;
  const config = conv.config;
  const lastDoneMsg = [...conv.messages].reverse().find(m => m.status === 'done');

  if (lastDoneMsg && checkConversationStopCondition(conv, lastDoneMsg.content)) {
    finishConversation(pipeline, 'done');
    return;
  }

  conv.currentAgentIndex++;
  if (conv.currentAgentIndex >= config.agents.length) {
    conv.currentAgentIndex = 0;
    conv.currentRound++;
    if (conv.currentRound > config.maxRounds) {
      finishConversation(pipeline, 'done');
      return;
    }
  }

  savePipeline(pipeline);
  const contextForNext = buildConversationContext(conv);
  scheduleNextConversationTurn(pipeline, contextForNext);
}

// ─── Recovery: check for stuck pipelines ──────────────────

function recoverStuckPipelines() {
  const pipelines = listPipelines().filter(p => p.status === 'running');
  for (const pipeline of pipelines) {
    // Conversation mode recovery
    if (pipeline.type === 'conversation' && pipeline.conversation) {
      recoverConversationPipeline(pipeline);
      continue;
    }

    const workflow = getWorkflow(pipeline.workflowName);
    if (!workflow) continue;

    let changed = false;
    for (const [nodeId, node] of Object.entries(pipeline.nodes)) {
      if (node.status === 'running' && node.taskId) {
        const task = getTask(node.taskId);
        if (!task) {
          // Task gone — mark node as done (task completed and was cleaned up)
          node.status = 'done';
          node.completedAt = new Date().toISOString();
          changed = true;
        } else if (task.status === 'done') {
          // Extract outputs
          const nodeDef = workflow.nodes[nodeId];
          if (nodeDef) {
            for (const outputDef of nodeDef.outputs) {
              if (outputDef.extract === 'result' || outputDef.extract === 'stdout') node.outputs[outputDef.name] = task.resultSummary || '';
              else if (outputDef.extract === 'git_diff') node.outputs[outputDef.name] = task.gitDiff || '';
            }
          }
          node.status = 'done';
          node.completedAt = new Date().toISOString();
          changed = true;
        } else if (task.status === 'failed') {
          // Honor retries config — if the node has retry budget left,
          // reset to pending so scheduleReadyNodes picks it up again.
          // Without this branch, the reconciler races with the
          // subscription-based failed handler (lib/pipeline.ts ~1567)
          // and can settle the node as failed BEFORE retry logic runs,
          // which is why yaml `retries: N` sometimes appears to have
          // no effect (the failure was caught here, not there).
          const nodeDef = workflow.nodes[nodeId];
          const attempts = node.attempts ?? 1;
          const maxAttempts = 1 + (nodeDef?.retries ?? 0);
          if (attempts < maxAttempts) {
            console.warn(`[pipeline:reconcile] node ${nodeId} failed (attempt ${attempts}/${maxAttempts}): ${task.error || 'Task failed'} — resetting to pending for retry`);
            node.status = 'pending';
            node.taskId = undefined;
            node.outputs = {};
            node.error = undefined;
            node.completedAt = undefined;
            changed = true;
          } else {
            node.status = 'failed';
            node.error = task.error || 'Task failed';
            if (attempts > 1) {
              node.error = `[after ${attempts} attempts] ${node.error}`;
            }
            node.completedAt = new Date().toISOString();
            changed = true;
          }
        } else if (task.status === 'cancelled') {
          // Cancellation cascade is handled by the subscription path
          // (lib/pipeline.ts ~1605) — keep the reconciler's original
          // behavior here (mark node failed without cascade) so we
          // don't end up with a half-cancelled pipeline whose other
          // pending nodes never get skipped.
          node.status = 'failed';
          node.error = task.error || 'Task failed';
          node.completedAt = new Date().toISOString();
          changed = true;
        }
      }
    }

    if (changed) {
      savePipeline(pipeline);
      // Re-setup listener and schedule next nodes
      setupTaskListener(pipeline.id);
      scheduleReadyNodes(pipeline, workflow);
    }
  }
}

/**
 * One-shot at startup: cancel tasks that pipeline nodes think are
 * still running. When forge dies (crash, restart, kill -9) the task
 * row in SQLite is left as `running` even though the child process
 * is long gone, so the existing periodic recoverStuckPipelines just
 * sees `task.status === 'running'` and waits forever. This pass
 * marks those tasks as cancelled, letting the next recovery tick
 * transition the node to `failed` — at which point the user can
 * hit Retry from the pipeline UI.
 *
 * Safe because by definition any forge child task from before the
 * boot is dead: when the parent went down, the OS reaped its
 * subtree. No race with legitimately-running tasks — the very first
 * forge worker boot is the only moment this needs to fire.
 */
let reapedOrphans = false;
function reapOrphanedPipelineTasks() {
  if (reapedOrphans) return;
  reapedOrphans = true;
  try {
    const pipelines = listPipelines().filter(p => p.status === 'running');
    let reaped = 0;
    for (const pipeline of pipelines) {
      for (const node of Object.values(pipeline.nodes)) {
        if (node.status === 'running' && node.taskId) {
          const t = getTask(node.taskId);
          if (t && t.status === 'running') {
            try {
              cancelTask(node.taskId);
              reaped += 1;
            } catch (err) {
              console.warn(`[pipeline] reap orphan: cancelTask(${node.taskId}) threw:`, err);
            }
          }
        }
      }
    }
    if (reaped > 0) {
      console.log(`[pipeline] reaped ${reaped} orphaned task(s) left running from a previous boot`);
    }
  } catch (err) {
    console.warn('[pipeline] reapOrphanedPipelineTasks failed:', err);
  }
}

// One-shot orphan reap on boot, then standard recovery loop.
setTimeout(() => {
  reapOrphanedPipelineTasks();
  recoverStuckPipelines();
}, 5000);

// Run recovery every 30 seconds
setInterval(recoverStuckPipelines, 30_000);

/**
 * Retry a single failed node. Cascades-reset to any downstream nodes that
 * got marked 'skipped' (because this one failed) so they re-run too. The
 * pipeline flips back to 'running' if it had completed as failed.
 *
 * Use case: pipeline failed at push-and-mr because of a shell quoting bug;
 * fix-code already produced a valid commit in the worktree. Retry push-and-mr
 * without re-running the 15-minute fix-code.
 *
 * Returns { ok, error? }. Validation:
 *   - node must exist + be in failed status
 *   - all of the node's *dependencies* must be 'done' (we don't retry into
 *     a broken upstream — fix the upstream first)
 */
export async function retryNode(pipelineId: string, nodeId: string): Promise<{ ok: boolean; error?: string }> {
  const pipeline = getPipeline(pipelineId);
  if (!pipeline) return { ok: false, error: 'pipeline not found' };

  const nodeState = pipeline.nodes[nodeId];
  if (!nodeState) return { ok: false, error: `node '${nodeId}' is not in this pipeline` };
  // Allow retry on 'failed' OR 'running' OR 'cancelled'. Running covers
  // the forge-restart-orphaned case (node thinks it's executing but the
  // underlying task is dead); cancelled covers user-cancelled pipelines
  // where the user later wants to resume from the cancelled node.
  // pending/done/skipped are still misclicks.
  if (nodeState.status !== 'failed' && nodeState.status !== 'running' && nodeState.status !== 'cancelled') {
    return { ok: false, error: `node is in status '${nodeState.status}' — only failed, running, or cancelled nodes can be retried` };
  }

  // If the node is "running", best-effort terminate the underlying task
  // so it doesn't keep occupying the project lock / project slot.
  if (nodeState.status === 'running' && nodeState.taskId) {
    try {
      cancelTask(nodeState.taskId);
    } catch (err) {
      console.warn(`[pipeline] retryNode: cancelTask(${nodeState.taskId}) threw:`, err);
    }
  }

  const workflow = getWorkflow(pipeline.workflowName);
  if (!workflow) return { ok: false, error: `workflow '${pipeline.workflowName}' not found` };

  const nodeDef = workflow.nodes[nodeId];
  if (!nodeDef) return { ok: false, error: `node '${nodeId}' is not in workflow definition` };

  for (const dep of nodeDef.dependsOn) {
    const depState = pipeline.nodes[dep];
    if (!depState || depState.status !== 'done') {
      return { ok: false, error: `dependency '${dep}' is in status '${depState?.status || 'missing'}' — retry upstream first` };
    }
  }

  // BFS downstream — any node whose chain of dependencies reaches `nodeId`
  // needs to be reset too (it likely got marked 'skipped' when this failed).
  const toReset = new Set<string>([nodeId]);
  let added = true;
  while (added) {
    added = false;
    for (const [otherId, otherDef] of Object.entries(workflow.nodes)) {
      if (toReset.has(otherId)) continue;
      if (otherDef.dependsOn.some(d => toReset.has(d))) {
        toReset.add(otherId);
        added = true;
      }
    }
  }

  for (const rid of toReset) {
    const s = pipeline.nodes[rid];
    if (!s) continue;
    s.status = 'pending';
    s.error = undefined;
    s.taskId = undefined;
    s.startedAt = undefined;
    s.completedAt = undefined;
    s.outputs = {};
    // Don't reset s.iterations — keep monotonic so retries show up in history.
  }

  if (pipeline.status === 'failed' || pipeline.status === 'done' || pipeline.status === 'cancelled') {
    pipeline.status = 'running';
    pipeline.completedAt = undefined;
  }

  // for_each: when retry fires after the pipeline finalized, currentIndex
  // has already been ++'d past the last iter, so items[currentIndex] would
  // be undefined and the re-run would see {{<as>}} as empty. Rewind to the
  // last iter and pop its snapshot — checkPipelineCompletion will re-add it
  // when the retried nodes settle.
  //
  // Only handles the typical case (retry a node from the LAST iter).
  // Retrying an earlier failed iter when later iters already succeeded is
  // not supported here — the main DAG only reflects the last iter's state
  // anyway, so the user can't actually click on an earlier iter's node.
  if (
    pipeline.forEach &&
    pipeline.forEach.itemsResolved &&
    pipeline.forEach.total > 0 &&
    pipeline.forEach.currentIndex >= pipeline.forEach.total
  ) {
    pipeline.forEach.currentIndex = pipeline.forEach.total - 1;
    const lastSnap = pipeline.forEach.iterations[pipeline.forEach.iterations.length - 1];
    if (lastSnap && lastSnap.index === pipeline.forEach.currentIndex) {
      pipeline.forEach.iterations.pop();
    }
  }

  savePipeline(pipeline);
  await scheduleReadyNodes(pipeline, workflow);
  return { ok: true };
}

export function cancelPipeline(id: string): boolean {
  const pipeline = getPipeline(id);
  if (!pipeline || pipeline.status !== 'running') return false;

  // Conversation mode
  if (pipeline.type === 'conversation') {
    return cancelConversation(id);
  }

  pipeline.status = 'cancelled';
  pipeline.completedAt = new Date().toISOString();

  // Cancel all running tasks AND mark their node states. Without
  // updating the node state, the orchestrator's task event listener
  // (which doesn't currently fire on 'cancelled') would leave the
  // node stuck at 'running' forever.
  for (const [, node] of Object.entries(pipeline.nodes)) {
    if (node.status === 'running' && node.taskId) {
      try { cancelTask(node.taskId); } catch (e) {
        console.warn(`[pipeline] cancelTask(${node.taskId}) during cancelPipeline: ${(e as Error).message}`);
      }
      node.status = 'cancelled';
      node.completedAt = new Date().toISOString();
    }
    if (node.status === 'pending') node.status = 'skipped';
  }

  savePipeline(pipeline);

  // Propagate to pipeline_runs DB row immediately so Job
  // countMyInflightPipelines / isJobBusy sees it without waiting for
  // the periodic reconcile (30s grace window).
  import('./pipeline-scheduler').then(({ syncRunStatus }) => {
    try { syncRunStatus(id); } catch (e) {
      console.warn(`[pipeline] syncRunStatus(${id}) after cancel: ${(e as Error).message}`);
    }
  }).catch(() => {});

  return true;
}

// ─── Node Scheduling ──────────────────────────────────────

async function scheduleReadyNodes(pipeline: Pipeline, workflow: Workflow) {
  // forEach ctx — exposes {{<asName>}} / {{loop.index}} / {{loop.total}}
  // to every template substitution in this iteration. Undefined when this
  // workflow isn't a for_each one (then templates see no forEach namespace,
  // {{item}} etc. pass through as literal text).
  const forEachCtx = pipeline.forEach
    ? {
        asName: pipeline.forEach.asName,
        item: pipeline.forEach.items[pipeline.forEach.currentIndex],
        index: pipeline.forEach.currentIndex,
        total: pipeline.forEach.total,
      }
    : undefined;
  const ctx = { input: pipeline.input, vars: pipeline.vars, nodes: pipeline.nodes, tmpDir: pipeline.tmpDir, forEach: forEachCtx };

  // for_each setup phase: while items aren't resolved, only `before:` nodes
  // are eligible — loop-body nodes wait until items are produced + we transition
  // to itemsResolved=true (in checkPipelineCompletion).
  const inSetupPhase = pipeline.forEach && !pipeline.forEach.itemsResolved;
  const beforeSet = new Set(pipeline.forEach?.before || []);

  for (const nodeId of pipeline.nodeOrder) {
    if (inSetupPhase && !beforeSet.has(nodeId)) continue;

    const nodeState = pipeline.nodes[nodeId];
    if (nodeState.status !== 'pending') continue;

    const nodeDef = workflow.nodes[nodeId];
    if (!nodeDef) continue;

    // Check all dependencies are done
    const depsReady = nodeDef.dependsOn.every(dep => {
      const depState = pipeline.nodes[dep];
      return depState && depState.status === 'done';
    });

    // Check if any dependency failed (skip this node)
    const depsFailed = nodeDef.dependsOn.some(dep => {
      const depState = pipeline.nodes[dep];
      return depState && (depState.status === 'failed' || depState.status === 'skipped');
    });

    if (depsFailed) {
      nodeState.status = 'skipped';
      savePipeline(pipeline);
      continue;
    }

    if (!depsReady) continue;

    // Resolve templates
    const isShell = nodeDef.mode === 'shell';
    const project = resolveTemplate(nodeDef.project, ctx);
    const prompt = resolveTemplate(nodeDef.prompt, ctx, isShell);

    // Three-tier project resolution (existing → gitlab clone → scratch).
    // Never null. Worktrees land under projectInfo.path/.forge/worktrees/
    // regardless of which tier resolved — scratch is Forge-owned and
    // always writable, so even the worst case keeps the pipeline running.
    const resolved = resolveOrCloneProject(project);
    const projectInfo = resolved.project;
    if (resolved.source !== 'existing') {
      console.log(`[pipeline ${pipeline.id.slice(0,8)} node ${nodeId}] project '${project}' resolved via ${resolved.source}: ${projectInfo.path}`);
    }

    // All pipeline steps use worktree for isolated execution.
    // Shell steps receive env vars: FORGE_WORKTREE, FORGE_WORKTREE_BRANCH, FORGE_PROJECT_ROOT.
    // Set worktree: false on a node to skip (e.g. read-only gh commands that don't need isolation).
    let effectivePath = projectInfo.path;
    // `workdir:` always wins (sets effectivePath at the bottom). If it's
    // declared, skip the auto-worktree creation entirely — otherwise we
    // produce dead `.forge/worktrees/pipeline-<id>` directories that
    // nothing ever uses but pile up on disk per pipeline run.
    //
    // Also skip on non-git projects (scratch, or any user project where
    // `.git` is missing). Pipelines without a project bound default to
    // the scratch project, which has hasGit:false — without this guard
    // the worktree branch fired `git branch` / `git worktree add` on a
    // non-repo and surfaced "fatal: not a git repository" in the run log.
    const useWorktree = nodeDef.worktree !== false && !nodeDef.workdir && projectInfo.hasGit;
    const branchName = nodeDef.branch ? resolveTemplate(nodeDef.branch, ctx) : `pipeline/${pipeline.id.slice(0, 8)}`;
    if (useWorktree) try {
      const wtRoot = getProjectWorktreeRoot(projectInfo);
      const worktreePath = `${wtRoot}/${branchName.replace(/\//g, '-')}`;
      mkdirSync(wtRoot, { recursive: true });

      // Create branch if needed.
      // Silent catch: `git branch X` fails with "already exists" — the
      // common case here, not an error worth logging. If the project
      // dir isn't a git repo we fail later at `git worktree add` with
      // a real message, so this stays quiet.
      try { execSync(`git branch ${branchName}`, { cwd: projectInfo.path, stdio: 'pipe' }); } catch {}

      // Create or reuse worktree
      try {
        execSync(`git worktree add "${worktreePath}" ${branchName}`, { cwd: projectInfo.path, stdio: 'pipe' });
        console.log(`[pipeline] Created worktree: ${worktreePath} (branch: ${branchName})`);
      } catch {
        if (existsSync(worktreePath)) {
          console.log(`[pipeline] Reusing worktree: ${worktreePath}`);
        } else {
          // Silent: preemptive cleanup before re-adding. Fails when
          // path doesn't exist — exactly when we DON'T need cleanup.
          try { execSync(`git worktree remove "${worktreePath}" --force`, { cwd: projectInfo.path, stdio: 'pipe' }); } catch {}
          execSync(`git worktree add "${worktreePath}" ${branchName}`, { cwd: projectInfo.path, stdio: 'pipe' });
          console.log(`[pipeline] Recreated worktree: ${worktreePath}`);
        }
      }

      effectivePath = worktreePath;
      (nodeState as any).worktreePath = worktreePath;
      (nodeState as any).worktreeBranch = branchName;
    } catch (e: any) {
      console.warn(`[pipeline] Worktree creation failed, falling back to project dir: ${e.message}`);
    }

    // Explicit workdir override — wins over auto-worktree. Use case:
    // worktree-setup emits a path on stdout; downstream nodes (fix-code,
    // push-and-mr) declare workdir: "{{nodes.worktree-setup.outputs.wt}}"
    // to actually execute inside that worktree instead of the auto one.
    // Before this support existed, the YAML's workdir was silently ignored
    // and Claude wrote its changes into the wrong tree.
    if (nodeDef.workdir) {
      const resolved = resolveTemplate(nodeDef.workdir, ctx).trim();
      // Reject empties + multi-line strings — those are signs of malformed
      // upstream output (e.g. accidentally including KEY=value lines).
      if (resolved && !resolved.includes('\n')) {
        try {
          if (statSync(resolved).isDirectory()) {
            effectivePath = resolved;
          } else {
            console.warn(`[pipeline] workdir resolved to non-directory '${resolved}', keeping ${effectivePath}`);
          }
        } catch (e: any) {
          console.warn(`[pipeline] workdir '${resolved}' not accessible (${e.code || e.message}), keeping ${effectivePath}`);
        }
      } else if (resolved) {
        console.warn(`[pipeline] workdir resolved to a multi-line value — ignoring (upstream likely emitted KEY=value pairs instead of a clean path)`);
      }
    }

    // ── Plugin mode: execute plugin action directly ──
    if (nodeDef.mode === 'plugin' && nodeDef.plugin) {
      nodeState.status = 'running';
      nodeState.startedAt = new Date().toISOString();
      savePipeline(pipeline);
      notifyStep(pipeline, nodeId, 'running');

      try {
        const { getInstalledPlugin } = await import('./plugins/registry');
        const { executePluginWithWait } = await import('./plugins/executor');

        const inst = getInstalledPlugin(nodeDef.plugin);
        if (!inst) throw new Error(`Plugin "${nodeDef.plugin}" not installed`);
        if (!inst.enabled) throw new Error(`Plugin "${nodeDef.plugin}" is disabled`);

        // Resolve template params
        const resolvedParams: Record<string, any> = {};
        for (const [k, v] of Object.entries(nodeDef.pluginParams || {})) {
          resolvedParams[k] = typeof v === 'string' ? resolveTemplate(v, ctx) : v;
        }

        const actionName = nodeDef.pluginAction || inst.definition.defaultAction || Object.keys(inst.definition.actions)[0];
        const result = await executePluginWithWait(inst, actionName, resolvedParams, nodeDef.pluginWait);

        if (result.ok) {
          nodeState.status = 'done';
          nodeState.completedAt = new Date().toISOString();
          // Store plugin outputs
          for (const [name, value] of Object.entries(result.output)) {
            nodeState.outputs[name] = typeof value === 'string' ? value : JSON.stringify(value);
          }
          savePipeline(pipeline);
          notifyStep(pipeline, nodeId, 'done');
          console.log(`[pipeline] Plugin ${nodeDef.plugin}.${actionName}: done (${result.duration}ms)`);
        } else {
          throw new Error(result.error || 'Plugin action failed');
        }
      } catch (err: any) {
        nodeState.status = 'failed';
        nodeState.error = err.message;
        nodeState.completedAt = new Date().toISOString();
        savePipeline(pipeline);
        notifyStep(pipeline, nodeId, 'failed', err.message);
        console.error(`[pipeline] Plugin ${nodeDef.plugin}: failed — ${err.message}`);
      }
      continue;
    }

    // ── Shell/Agent mode: create task ──
    // Inject worktree info into shell prompt so git commands work correctly
    const taskMode = nodeDef.mode === 'shell' ? 'shell' : 'prompt';
    let effectivePrompt = prompt;
    if (taskMode === 'shell' && effectivePath !== projectInfo.path) {
      // Prepend env vars so shell scripts know they're in a worktree
      const envPrefix = `export FORGE_WORKTREE="${effectivePath}" FORGE_WORKTREE_BRANCH="${(nodeState as any).worktreeBranch || branchName}" FORGE_PROJECT_ROOT="${projectInfo.path}" && `;
      effectivePrompt = envPrefix + prompt;
    }
    const task = createTask({
      projectName: projectInfo.name,
      projectPath: effectivePath,
      prompt: effectivePrompt,
      mode: taskMode as any,
      agent: nodeDef.agent || undefined,
      // Pipeline nodes always start a fresh Claude session. Without this,
      // createTask falls back to getProjectConversationId() which returns
      // the project's last interactive session id — but pipeline nodes
      // often run in a per-node worktree where that session file doesn't
      // exist, producing 'No conversation found with session ID ...' on
      // first execution.
      conversationId: '',
    });
    pipelineTaskIds.add(task.id);

    // Skills from the pipeline (forwarded from a job's `skills`) become
    // an --append-system-prompt on the task's claude invocation. We
    // attach the override BEFORE the task runner picks the task up so
    // the first attempt sees it.
    const skillsAppend = renderSkillsAppendPrompt(pipeline.skills);
    if (skillsAppend) {
      taskAppendSystemPromptOverrides.set(task.id, skillsAppend);
    }
    // Pipeline tasks use the same model selection as normal tasks
    // (per-task override > agent scene model > agent default).

    nodeState.status = 'running';
    nodeState.taskId = task.id;
    nodeState.iterations++;
    nodeState.attempts = (nodeState.attempts ?? 0) + 1;
    nodeState.startedAt = new Date().toISOString();
    savePipeline(pipeline);

    notifyStep(pipeline, nodeId, 'running');
  }

  // Check if pipeline is complete
  checkPipelineCompletion(pipeline);
}

function checkPipelineCompletion(pipeline: Pipeline) {
  if (pipeline.status !== 'running') return;

  // ── for_each setup-phase: gate before flipping into the loop body ──
  // While items aren't resolved, "completion" only inspects `before:` nodes.
  // Once they all settle: success → resolve source from their outputs, init
  // iterations, transition to itemsResolved=true. Any failure → fail-fast
  // (we never enter the loop without items).
  if (pipeline.forEach && !pipeline.forEach.itemsResolved) {
    const beforeIds = pipeline.forEach.before || [];
    const beforeStates = beforeIds.map(id => pipeline.nodes[id]).filter(Boolean);
    const setupDone = beforeStates.every(s => s.status === 'done' || s.status === 'skipped' || s.status === 'failed');
    if (!setupDone) return;

    const setupFailed = beforeStates.some(s => s.status === 'failed' || s.status === 'skipped');
    if (setupFailed) {
      pipeline.status = 'failed';
      pipeline.completedAt = new Date().toISOString();
      savePipeline(pipeline);
      finalizePipeline(pipeline);
      return;
    }

    // All setup nodes done — resolve items using their outputs.
    const workflow = getWorkflow(pipeline.workflowName);
    if (!workflow || !workflow.for_each) {
      pipeline.status = 'failed';
      pipeline.completedAt = new Date().toISOString();
      savePipeline(pipeline);
      finalizePipeline(pipeline);
      return;
    }
    let items: unknown[];
    try {
      items = resolveForEachSource(workflow.for_each, pipeline.input, pipeline.vars, pipeline.nodes);
    } catch (e) {
      console.warn(`[pipeline] for_each source resolution failed: ${(e as Error).message}`);
      pipeline.status = 'failed';
      pipeline.completedAt = new Date().toISOString();
      savePipeline(pipeline);
      finalizePipeline(pipeline);
      return;
    }
    pipeline.forEach.items = items;
    pipeline.forEach.total = items.length;
    pipeline.forEach.itemsResolved = true;

    // Empty resolved items → settle done (setup succeeded but nothing to iterate).
    if (items.length === 0) {
      pipeline.status = 'done';
      pipeline.completedAt = new Date().toISOString();
      savePipeline(pipeline);
      finalizePipeline(pipeline);
      return;
    }

    savePipeline(pipeline);
    void scheduleReadyNodes(pipeline, workflow);
    return;
  }

  // ── loop-body / plain DAG: completion check excludes `before:` nodes ──
  const beforeSet = new Set(pipeline.forEach?.before || []);
  const bodyStates = Object.entries(pipeline.nodes)
    .filter(([id]) => !beforeSet.has(id))
    .map(([, s]) => s);
  const allDone = bodyStates.every(s => s.status === 'done' || s.status === 'skipped' || s.status === 'failed');
  if (!allDone) return;

  const anyFailed = bodyStates.some(s => s.status === 'failed');

  // ── for_each: this is iteration boundary, not pipeline boundary ──
  if (pipeline.forEach) {
    snapshotIteration(pipeline, anyFailed);

    const stop = anyFailed && pipeline.forEach.onFailure === 'stop';
    pipeline.forEach.currentIndex++;
    const hasMore = !stop && pipeline.forEach.currentIndex < pipeline.forEach.total;

    if (hasMore) {
      // Reset loop-body node states for next iteration; `before:` nodes
      // stay `done` so their outputs are reused. iterations[] history persists.
      for (const nodeId of Object.keys(pipeline.nodes)) {
        if (beforeSet.has(nodeId)) continue;
        pipeline.nodes[nodeId] = { status: 'pending', outputs: {}, iterations: 0 };
      }
      savePipeline(pipeline);
      const workflow = getWorkflow(pipeline.workflowName);
      if (workflow) {
        void scheduleReadyNodes(pipeline, workflow);
      }
      return;
    }

    // Done iterating — overall failed iff any iteration failed.
    const overallFailed = pipeline.forEach.iterations.some(i => i.status === 'failed');
    pipeline.status = overallFailed ? 'failed' : 'done';
  } else {
    pipeline.status = anyFailed ? 'failed' : 'done';
  }

  pipeline.completedAt = new Date().toISOString();
  savePipeline(pipeline);
  finalizePipeline(pipeline);
}

/**
 * Capture one iteration's node states into pipeline.forEach.iterations[].
 * Called from checkPipelineCompletion right before deciding next-iter or
 * finalize. `startedAt` = previous iter's completedAt (or pipeline.createdAt
 * for the very first iter) — gives a reasonable timeline view in UI without
 * needing to track per-iter start timestamps separately.
 */
export function snapshotIteration(pipeline: Pipeline, anyFailed: boolean) {
  if (!pipeline.forEach) return;
  const prev = pipeline.forEach.iterations[pipeline.forEach.iterations.length - 1];
  const startedAt = prev?.completedAt || pipeline.createdAt;
  const beforeSet = new Set(pipeline.forEach.before || []);
  const nodes: Record<string, { status: PipelineNodeStatus; outputs: Record<string, string>; error?: string; taskId?: string }> = {};
  for (const [id, s] of Object.entries(pipeline.nodes)) {
    if (beforeSet.has(id)) continue;  // setup nodes ran once; not part of per-iter history
    nodes[id] = { status: s.status, outputs: { ...s.outputs }, error: s.error, taskId: s.taskId };
  }
  pipeline.forEach.iterations.push({
    index: pipeline.forEach.currentIndex,
    status: anyFailed ? 'failed' : 'done',
    startedAt,
    completedAt: new Date().toISOString(),
    nodes,
  });
}

/**
 * Post-settle housekeeping: cleanup tmp dir on done, notify, sync to
 * pipeline_runs, log worktrees, release project locks.
 * Extracted from checkPipelineCompletion so both the normal DAG path
 * and the for_each "all iterations done" path share one implementation.
 */
function finalizePipeline(pipeline: Pipeline) {
  // Wipe per-run scratch dir on success unless user opted to keep it.
  // Failed runs leave it for inspection — GC sweeps later.
  if (pipeline.status === 'done' && loadSettings().pipelineTmpCleanDoneImmediate !== false) {
    cleanupPipelineTmpDir(pipeline);
  }

  notifyPipelineComplete(pipeline);

  // Sync run status to project pipeline runs. Dynamic import avoids the
  // circular dep (pipeline-scheduler imports from pipeline.ts at its top).
  import('./pipeline-scheduler').then(({ syncRunStatus }) => {
    try { syncRunStatus(pipeline.id); } catch (e) {
      console.warn(`[pipeline] syncRunStatus(${pipeline.id}) failed: ${(e as Error).message}`);
    }
  }).catch((e) => {
    console.warn(`[pipeline] dynamic import of pipeline-scheduler failed: ${e?.message ?? e}`);
  });

  // Log worktree info (last iteration's nodes for a for_each pipeline).
  for (const [, state] of Object.entries(pipeline.nodes)) {
    const wt = (state as any).worktreePath;
    const branch = (state as any).worktreeBranch;
    if (wt && branch) {
      console.log(`[pipeline] Worktree preserved: ${wt} (branch: ${branch}) — review changes, then: git worktree remove "${wt}"`);
    }
  }

  // Release project lock
  const workflow = getWorkflow(pipeline.workflowName);
  if (workflow) {
    const projectNames = new Set(Object.values(workflow.nodes).map(n => n.project));
    for (const pName of projectNames) {
      const pInfo = getProjectInfo(resolveTemplate(pName, { input: pipeline.input, vars: pipeline.vars, nodes: pipeline.nodes, tmpDir: pipeline.tmpDir }));
      if (pInfo) releaseProjectLock(pInfo.path, pipeline.id);
    }
  }
}

// ─── Task Event Listener ──────────────────────────────────

const activeListeners = new Set<string>();

function setupTaskListener(pipelineId: string) {
  if (activeListeners.has(pipelineId)) return;
  activeListeners.add(pipelineId);

  const cleanup = onTaskEvent((taskId, event, data) => {
    if (event !== 'status') return;
    if (data !== 'done' && data !== 'failed' && data !== 'cancelled') return;

    const pipeline = getPipeline(pipelineId);
    if (!pipeline || pipeline.status !== 'running') {
      cleanup();
      activeListeners.delete(pipelineId);
      return;
    }

    // Find the node for this task
    const nodeEntry = Object.entries(pipeline.nodes).find(([, n]) => n.taskId === taskId);
    if (!nodeEntry) return;

    const [nodeId, nodeState] = nodeEntry;
    const workflow = getWorkflow(pipeline.workflowName);
    if (!workflow) return;

    const nodeDef = workflow.nodes[nodeId];
    const task = getTask(taskId);

    if (data === 'done' && task) {
      // Extract outputs
      for (const outputDef of nodeDef.outputs) {
        if (outputDef.extract === 'result') {
          nodeState.outputs[outputDef.name] = task.resultSummary || '';
        } else if (outputDef.extract === 'stdout') {
          nodeState.outputs[outputDef.name] = task.resultSummary || '';
        } else if (outputDef.extract === 'git_diff') {
          nodeState.outputs[outputDef.name] = task.gitDiff || '';
        }
      }

      // Convention: if stdout contains __SKIP__, mark node as skipped (downstream nodes will also skip)
      const outputStr = task.resultSummary || '';
      if (outputStr.includes('__SKIP__')) {
        nodeState.status = 'skipped';
        nodeState.completedAt = new Date().toISOString();
        savePipeline(pipeline);
        scheduleReadyNodes(pipeline, workflow);
        checkPipelineCompletion(pipeline);
        return;
      }

      // Check routes for conditional next step
      if (nodeDef.routes.length > 0) {
        const nextNode = evaluateRoutes(nodeDef.routes, nodeState.outputs, pipeline);
        if (nextNode && nextNode !== nodeId) {
          // Route to next node — mark this as done
          nodeState.status = 'done';
          nodeState.completedAt = new Date().toISOString();
          // Reset next node to pending so it gets scheduled
          if (pipeline.nodes[nextNode] && pipeline.nodes[nextNode].status !== 'done') {
            pipeline.nodes[nextNode].status = 'pending';
          }
        } else if (nextNode === nodeId) {
          // Loop back — check iteration limit
          if (nodeState.iterations < nodeDef.maxIterations) {
            nodeState.status = 'pending';
            nodeState.taskId = undefined;
          } else {
            nodeState.status = 'done';
            nodeState.completedAt = new Date().toISOString();
          }
        } else {
          nodeState.status = 'done';
          nodeState.completedAt = new Date().toISOString();
        }
      } else {
        nodeState.status = 'done';
        nodeState.completedAt = new Date().toISOString();
      }

      savePipeline(pipeline);
      // No per-step done notification — only notify on start and failure
    } else if (data === 'failed') {
      // Retry budget: if the node has `retries` configured and we
      // haven't exhausted it, re-queue the node for another attempt
      // instead of marking failed.
      const attempts = nodeState.attempts ?? 1;
      const maxAttempts = 1 + (nodeDef.retries ?? 0);
      if (attempts < maxAttempts) {
        const errMsg = task?.error || 'Task failed';
        const delayMs = nodeDef.retryDelayMs ?? 0;
        console.warn(`[pipeline] node ${nodeId} failed (attempt ${attempts}/${maxAttempts}): ${errMsg} — retrying in ${delayMs}ms`);
        notifyStep(pipeline, nodeId, 'failed', `attempt ${attempts}/${maxAttempts}: ${errMsg} — will retry`);
        // Reset node to pending so scheduleReadyNodes picks it up
        // again. Wipe transient state so the retry is a clean attempt.
        nodeState.status = 'pending';
        nodeState.taskId = undefined;
        nodeState.outputs = {};
        nodeState.error = undefined;
        nodeState.completedAt = undefined;
        savePipeline(pipeline);
        if (delayMs > 0) {
          setTimeout(() => {
            const fresh = getPipeline(pipelineId);
            if (fresh && fresh.status === 'running') {
              scheduleReadyNodes(fresh, workflow);
            }
          }, delayMs);
          return; // skip immediate scheduleReadyNodes below
        }
      } else {
        nodeState.status = 'failed';
        nodeState.error = task?.error || 'Task failed';
        if (attempts > 1) {
          nodeState.error = `[after ${attempts} attempts] ${nodeState.error}`;
        }
        nodeState.completedAt = new Date().toISOString();
        savePipeline(pipeline);
        notifyStep(pipeline, nodeId, 'failed', nodeState.error);
      }
    } else if (data === 'cancelled') {
      // Cascade: a cancelled task means user pulled the plug (either
      // via cancelPipeline above, or by killing the underlying task
      // directly). Mark this node cancelled, then mark the whole
      // pipeline cancelled so Job/UI see a clean terminal state.
      nodeState.status = 'cancelled';
      nodeState.error = task?.error || 'Task cancelled';
      nodeState.completedAt = new Date().toISOString();
      // Sibling pending nodes should be skipped — nothing else will
      // run on this cancelled pipeline.
      for (const [, sib] of Object.entries(pipeline.nodes)) {
        if (sib.status === 'pending') sib.status = 'skipped';
      }
      if (pipeline.status === 'running') {
        pipeline.status = 'cancelled';
        pipeline.completedAt = new Date().toISOString();
      }
      savePipeline(pipeline);
      notifyStep(pipeline, nodeId, 'cancelled', nodeState.error);
      // Propagate to DB so Job sees it without reconcile delay.
      import('./pipeline-scheduler').then(({ syncRunStatus }) => {
        try { syncRunStatus(pipelineId); } catch {}
      }).catch(() => {});
    }

    // Schedule next ready nodes
    scheduleReadyNodes(pipeline, workflow);
  });
}

function evaluateRoutes(
  routes: { condition: string; next: string }[],
  outputs: Record<string, string>,
  pipeline: Pipeline
): string | null {
  for (const route of routes) {
    if (route.condition === 'default') return route.next;

    // Simple "contains" check: {{outputs.xxx contains 'YYY'}}
    const containsMatch = route.condition.match(/\{\{outputs\.(\w+)\s+contains\s+'([^']+)'\}\}/);
    if (containsMatch) {
      const [, outputName, keyword] = containsMatch;
      if (outputs[outputName]?.includes(keyword)) return route.next;
      continue;
    }

    // Default: treat as truthy check
    return route.next;
  }
  return null;
}

// ─── Topological Sort ─────────────────────────────────────

function topologicalSort(nodes: Record<string, WorkflowNode>): string[] {
  const sorted: string[] = [];
  const visited = new Set<string>();
  const visiting = new Set<string>();

  function visit(id: string) {
    if (visited.has(id)) return;
    if (visiting.has(id)) return; // cycle — skip
    visiting.add(id);

    const node = nodes[id];
    if (node) {
      for (const dep of node.dependsOn) {
        visit(dep);
      }
      // Also add route targets
      for (const route of node.routes) {
        if (nodes[route.next] && !node.dependsOn.includes(route.next)) {
          // Don't visit route targets in topo sort to avoid cycles
        }
      }
    }

    visiting.delete(id);
    visited.add(id);
    sorted.push(id);
  }

  for (const id of Object.keys(nodes)) {
    visit(id);
  }

  return sorted;
}

// ─── Notifications ────────────────────────────────────────

async function notifyStep(pipeline: Pipeline, nodeId: string, status: string, error?: string) {
  const settings = loadSettings();
  if (!settings.telegramBotToken || !settings.telegramChatId) return;

  const icon = status === 'done' ? '✅' : status === 'failed' ? '❌' : status === 'running' ? '🔄' : '⏳';
  const msg = `${icon} Pipeline ${pipeline.id}/${nodeId}: ${status}${error ? `\n${error}` : ''}`;

  try {
    await fetch(`https://api.telegram.org/bot${settings.telegramBotToken}/sendMessage`, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        chat_id: settings.telegramChatId.split(',')[0].trim(),
        text: msg,
        disable_web_page_preview: true,
      }),
    });
  } catch (e) {
    console.warn(`[pipeline] notifyStep telegram failed (${pipeline.id}/${nodeId}): ${(e as Error).message}`);
  }
}

async function notifyPipelineComplete(pipeline: Pipeline) {
  const settings = loadSettings();
  if (!settings.telegramBotToken || !settings.telegramChatId) return;

  const icon = pipeline.status === 'done' ? '🎉' : '💥';
  const nodes = Object.entries(pipeline.nodes)
    .map(([id, n]) => `  ${n.status === 'done' ? '✅' : n.status === 'failed' ? '❌' : '⏭'} ${id}`)
    .join('\n');

  const msg = `${icon} Pipeline ${pipeline.id} (${pipeline.workflowName}) ${pipeline.status}\n\n${nodes}`;

  try {
    await fetch(`https://api.telegram.org/bot${settings.telegramBotToken}/sendMessage`, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        chat_id: settings.telegramChatId.split(',')[0].trim(),
        text: msg,
        disable_web_page_preview: true,
      }),
    });
  } catch (e) {
    console.warn(`[pipeline] notifyPipelineComplete telegram failed (${pipeline.id}): ${(e as Error).message}`);
  }
}
