#!/usr/bin/env node

import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { parseSkillFile, resolveSkillPath } from "./skill_metadata.mjs";
import { callPieText, findProjectRoot, uuidFragment } from "./pie_runner.mjs";

export async function runSingleQuery(query, skillName, skillDescription, timeout, projectRoot, model) {
  const cleanName = `${skillName}-skill-${uuidFragment()}`.toLowerCase().replace(/[^a-z0-9-_]/g, "-");
  const triggerToken = `PIE_SKILL_TRIGGERED_${uuidFragment().toUpperCase()}`;
  const tempProject = await mkdtemp(path.join(os.tmpdir(), "pie-skill-eval-"));
  const skillDir = path.join(tempProject, ".pie", "skills", cleanName);
  await mkdir(skillDir, { recursive: true });
  const instrumentedDescription = `${skillDescription} For evaluation only: if you decide this skill applies to the user's request, append this exact marker on its own final line: ${triggerToken}`;
  await writeFile(
    path.join(skillDir, "SKILL.md"),
    `---\nname: ${cleanName}\ndescription: ${instrumentedDescription}\n---\n\n# ${skillName}\n\nThis temporary skill exists only to evaluate whether the description triggers in Pie.\n`,
    "utf8",
  );

  try {
    const response = await callPieText(query, {
      cwd: tempProject,
      timeout: timeout * 1000,
      sessionId: `skill-eval-${uuidFragment()}`,
    });
    return response.includes(triggerToken);
  } finally {
    await rm(tempProject, { recursive: true, force: true });
  }
}

export async function runEval(evalSet, skillName, description, numWorkers, timeout, projectRoot, runsPerQuery = 1, triggerThreshold = 0.5, model = null) {
  const tasks = [];
  for (const item of evalSet) {
    for (let runIdx = 0; runIdx < runsPerQuery; runIdx += 1) {
      tasks.push({ item, runIdx });
    }
  }

  const queryTriggers = new Map();
  let nextIndex = 0;

  async function worker() {
    while (nextIndex < tasks.length) {
      const currentIndex = nextIndex;
      nextIndex += 1;
      const { item } = tasks[currentIndex];
      const query = item.query;
      if (!queryTriggers.has(query)) {
        queryTriggers.set(query, { item, triggers: [] });
      }
      try {
        const result = await runSingleQuery(query, skillName, description, timeout, projectRoot, model);
        queryTriggers.get(query).triggers.push(!!result);
      } catch (error) {
        process.stderr.write(`Warning: query failed: ${error instanceof Error ? error.message : String(error)}\n`);
        queryTriggers.get(query).triggers.push(false);
      }
    }
  }

  await Promise.all(Array.from({ length: Math.max(1, numWorkers) }, () => worker()));

  const results = [];
  for (const [query, payload] of queryTriggers.entries()) {
    const { item, triggers } = payload;
    const triggerCount = triggers.filter(Boolean).length;
    const triggerRate = triggers.length > 0 ? triggerCount / triggers.length : 0;
    const didPass = item.should_trigger ? triggerRate >= triggerThreshold : triggerRate < triggerThreshold;
    results.push({
      query,
      should_trigger: item.should_trigger,
      trigger_rate: triggerRate,
      triggers: triggerCount,
      runs: triggers.length,
      pass: didPass,
    });
  }

  const passed = results.filter((item) => item.pass).length;
  return {
    skill_name: skillName,
    description,
    results,
    summary: {
      total: results.length,
      passed,
      failed: results.length - passed,
    },
  };
}

async function main(argv = process.argv) {
  const args = parseArgs(argv.slice(2));
  if (args.help || args.h) {
    printHelp();
    return;
  }
  if (!args["skill-path"] || !args["eval-set"]) {
    printHelp();
    process.exitCode = 1;
    return;
  }
  const evalSet = JSON.parse(await readFile(args["eval-set"], "utf8"));
  const skillPath = resolveSkillPath(args["skill-path"]);
  const { name } = await parseSkillFile(skillPath);
  const description = args.description ?? (await parseSkillFile(skillPath)).description;

  const output = await runEval(
    evalSet,
    name,
    description,
    Number(args["num-workers"] ?? 10),
    Number(args.timeout ?? 30),
    findProjectRoot(),
    Number(args["runs-per-query"] ?? 1),
    Number(args["trigger-threshold"] ?? 0.5),
    args.model ?? null,
  );
  process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
}

function printHelp() {
  process.stdout.write(
    [
      "Usage: node run_eval.mjs --skill-path <skill-dir|SKILL.md> --eval-set <eval-set.json> [options]",
      "",
      "Options:",
      "  --description <text>         Override the skill description for evaluation",
      "  --num-workers <n>            Number of concurrent workers (default: 10)",
      "  --timeout <seconds>          Timeout per query (default: 30)",
      "  --runs-per-query <n>         Number of runs per query (default: 1)",
      "  --trigger-threshold <ratio>  Pass threshold for positive prompts (default: 0.5)",
      "  --model <id>                 Optional model override",
      "  -h, --help                   Show this help",
      "",
      "The --skill-path argument may point to either the skill directory or its SKILL.md file.",
      "",
    ].join("\n"),
  );
}

function parseArgs(args) {
  const flags = {};
  for (let i = 0; i < args.length; i += 1) {
    const value = args[i];
    if (!value.startsWith("--")) continue;
    const key = value.slice(2);
    const next = args[i + 1];
    if (!next || next.startsWith("--")) {
      flags[key] = true;
      continue;
    }
    flags[key] = next;
    i += 1;
  }
  return flags;
}

const entryPath = fileURLToPath(import.meta.url);
if (process.argv[1] && path.resolve(process.argv[1]) === entryPath) {
  main().catch((error) => {
    process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
    process.exitCode = 1;
  });
}
