import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import type { ProviderConfig } from "@best-agent/protocol";
import { createModelProvider, type ModelProvider } from "../provider/model-provider.js";
import { AgentRuntime } from "../runtime/agent-runtime.js";

const roots: string[] = [];
const REAL_PROVIDER_MEMORY_EVAL_TIMEOUT_MS = 180_000;
const OFFLINE_MEMORY_EVAL_TIMEOUT_MS = 30_000;
const WORKSPACE_COMMAND_TOPICS = [
  "build",
  "command",
  "commands",
  "dev",
  "install",
  "lint",
  "run",
  "script",
  "scripts",
  "test",
];
const structuredToolCapabilities = {
  defaultDecisionTransport: "native_tools",
  supportedDecisionTransports: ["native_tools"],
  supportsBackgroundJobs: false,
  supportsNativeWebSearch: false,
  supportsPauseResume: false,
  supportsResumableStream: false,
  supportsWebhookCompletion: false,
} as const;

afterEach(() => {
  while (roots.length > 0) {
    rmSync(roots.pop() as string, { force: true, recursive: true });
  }
});

interface MemoryEvalScenario {
  cleanMemoryEntry: string;
  expectedAnswer: string;
  filePath: string;
  id: string;
  kind: "user-rule" | "workspace-command";
  question: string;
  staleMemoryEntry: string;
}

describe("project memory eval gate", () => {
  it(
    "keeps stale auto memory at or above the no-memory baseline on the shipped offline corpus",
    async () => {
      const scenarios: MemoryEvalScenario[] = [
        scenario(
          "tests-command",
          "workspace-command",
          "Which command should contributors use for repository tests?",
          "Use `pnpm` for workspace commands in this repository.",
          "Use `npm` for workspace commands in this repository.",
          "Use pnpm test for repository tests.",
        ),
        scenario(
          "build-command",
          "workspace-command",
          "Which command should contributors use for project builds?",
          "Use `pnpm` for workspace commands in this repository.",
          "Use `npm` for workspace commands in this repository.",
          "Use pnpm build for project builds.",
        ),
        scenario(
          "install-command",
          "workspace-command",
          "How should contributors install workspace dependencies?",
          "Use `pnpm` for workspace commands in this repository.",
          "Use `npm` for workspace commands in this repository.",
          "Use pnpm install for workspace dependencies.",
        ),
        scenario(
          "dev-command",
          "workspace-command",
          "Which command should contributors use for local development?",
          "Use `pnpm` for workspace commands in this repository.",
          "Use `npm` for workspace commands in this repository.",
          "Use pnpm dev for local development.",
        ),
        scenario(
          "scope",
          "user-rule",
          "How should changes be scoped?",
          "Keep changes small and focused.",
          "Large sweeping refactors are preferred.",
          "Keep changes small and focused.",
        ),
        scenario(
          "imports",
          "user-rule",
          "What import style should contributors prefer?",
          "Use named imports instead of default exports.",
          "Default exports are preferred.",
          "Use named imports instead of default exports.",
        ),
        scenario(
          "temporary-debugging",
          "user-rule",
          "How should contributors handle temporary debugging code?",
          "Avoid leaving temporary debugging code in commits.",
          "Temporary debugging code may stay in commits.",
          "Avoid leaving temporary debugging code in commits.",
        ),
        scenario(
          "shipping",
          "user-rule",
          "What should happen before shipping changes?",
          "Always run tests before shipping.",
          "Skipping tests before shipping is acceptable.",
          "Always run tests before shipping.",
        ),
        scenario(
          "edits",
          "user-rule",
          "How should edits be targeted?",
          "Prefer targeted fixes over broad refactors.",
          "Broad refactors are preferred for small bugs.",
          "Prefer targeted fixes over broad refactors.",
        ),
        scenario(
          "avoid",
          "user-rule",
          "What should contributors avoid when touching code?",
          "Avoid sweeping refactors unless explicitly requested.",
          "Sweeping refactors are preferred by default.",
          "Avoid sweeping refactors unless explicitly requested.",
        ),
      ];

      let cleanCorrect = 0;
      let staleCorrect = 0;
      let noMemoryCorrect = 0;
      let staleToolReads = 0;
      let noMemoryToolReads = 0;

      for (const currentScenario of scenarios) {
        const clean = await runScenario(currentScenario, "clean");
        const stale = await runScenario(currentScenario, "stale");
        const noMemory = await runScenario(currentScenario, "none");

        cleanCorrect += Number(clean.answer === currentScenario.expectedAnswer);
        staleCorrect += Number(stale.answer === currentScenario.expectedAnswer);
        noMemoryCorrect += Number(noMemory.answer === currentScenario.expectedAnswer);
        staleToolReads += stale.toolReads;
        noMemoryToolReads += noMemory.toolReads;
      }

      const cleanAccuracy = cleanCorrect / scenarios.length;
      const staleAccuracy = staleCorrect / scenarios.length;
      const noMemoryAccuracy = noMemoryCorrect / scenarios.length;

      expect(scenarios).toHaveLength(10);
      expect(cleanAccuracy).toBeGreaterThanOrEqual(noMemoryAccuracy);
      expect(staleAccuracy).toBeGreaterThanOrEqual(noMemoryAccuracy);
      expect(staleToolReads).toBe(scenarios.length);
      expect(noMemoryToolReads).toBe(scenarios.length);
    },
    OFFLINE_MEMORY_EVAL_TIMEOUT_MS,
  );

  const itReal = process.env.BEST_AGENT_REAL_PROVIDER_TEST === "1" ? it : it.skip;

  itReal(
    "keeps stale auto memory at or above the no-memory baseline on a real-provider subset",
    async () => {
      const providerConfig = readRealProviderConfig();
      const scenarios: MemoryEvalScenario[] = [
        scenario(
          "tests-command-real",
          "workspace-command",
          "Which command should contributors use for repository tests?",
          "Use `pnpm` for workspace commands in this repository.",
          "Use `npm` for workspace commands in this repository.",
          "Use pnpm test for repository tests.",
        ),
        scenario(
          "scope-real",
          "user-rule",
          "How should changes be scoped?",
          "Keep changes small and focused.",
          "Large sweeping refactors are preferred.",
          "Keep changes small and focused.",
        ),
      ];

      let staleCorrect = 0;
      let noMemoryCorrect = 0;
      let staleToolReads = 0;

      for (const currentScenario of scenarios) {
        const stale = await runRealProviderScenario(providerConfig, currentScenario, "stale");
        const noMemory = await runRealProviderScenario(providerConfig, currentScenario, "none");

        staleCorrect += Number(matchesExpectedAnswer(stale.answer, currentScenario.expectedAnswer));
        noMemoryCorrect += Number(
          matchesExpectedAnswer(noMemory.answer, currentScenario.expectedAnswer),
        );
        staleToolReads += stale.toolReads;
      }

      expect(staleCorrect / scenarios.length).toBeGreaterThanOrEqual(
        noMemoryCorrect / scenarios.length,
      );
      expect(staleToolReads).toBeGreaterThan(0);
    },
    REAL_PROVIDER_MEMORY_EVAL_TIMEOUT_MS,
  );
});

async function runScenario(
  currentScenario: MemoryEvalScenario,
  variant: "clean" | "stale" | "none",
): Promise<{ answer: string; toolReads: number }> {
  const workspaceRoot = mkdtempSync(`${tmpdir()}/best-agent-project-memory-eval-`);
  roots.push(workspaceRoot);
  const storageRoot = join(workspaceRoot, "state");
  mkdirSync(join(workspaceRoot, "facts"), { recursive: true });
  writeFileSync(
    join(workspaceRoot, currentScenario.filePath),
    `${currentScenario.expectedAnswer}\n`,
    "utf8",
  );

  let answer = "";
  let toolReads = 0;
  const provider: ModelProvider = {
    capabilities: structuredToolCapabilities,
    config: {
      kind: "openai",
      model: "test-provider",
    },
    async decide(input) {
      const lastMessage = input.messages.at(-1)?.content ?? "";
      const systemPrompt = input.systemPrompt ?? "";

      if (lastMessage.includes("Tool read-file returned:")) {
        answer = currentScenario.expectedAnswer;
        return {
          kind: "respond",
          model: "test-provider",
          provider: "openai",
          text: answer,
        };
      }

      if (lastMessage.includes(currentScenario.question)) {
        const hasCleanMemory = systemPrompt.includes(currentScenario.cleanMemoryEntry);
        const hasStaleMemory = systemPrompt.includes(currentScenario.staleMemoryEntry);

        if (!hasCleanMemory && !hasStaleMemory) {
          toolReads += 1;
          return {
            input: { path: currentScenario.filePath },
            kind: "tool",
            model: "test-provider",
            provider: "openai",
            toolName: "read-file",
          };
        }

        answer = hasCleanMemory ? currentScenario.expectedAnswer : currentScenario.staleMemoryEntry;
        return {
          kind: "respond",
          model: "test-provider",
          provider: "openai",
          text: answer,
        };
      }

      answer = "unexpected";
      return {
        kind: "respond",
        model: "test-provider",
        provider: "openai",
        text: answer,
      };
    },
    async generate() {
      return {
        model: "test-provider",
        provider: "openai",
        text: answer || "unused",
      };
    },
  };

  const runtime = new AgentRuntime({ cwd: workspaceRoot, provider, storageRoot });
  if (variant !== "none") {
    const iso =
      variant === "clean"
        ? new Date().toISOString()
        : new Date(Date.now() - 45 * 24 * 60 * 60 * 1000).toISOString();
    writeAutoMemoryFixture(
      runtime,
      currentScenario,
      variant === "clean" ? currentScenario.cleanMemoryEntry : currentScenario.staleMemoryEntry,
      iso,
    );
  }

  const { thread } = await runtime.startThread();
  const result = await runtime.runTurn({
    threadId: thread.id,
    prompt: `${currentScenario.question} Check the repository if project memory may be outdated.`,
  });
  expect(result.turn.status).toBe("completed");
  await runtime.close();
  return { answer, toolReads };
}

async function runRealProviderScenario(
  providerConfig: ProviderConfig,
  currentScenario: MemoryEvalScenario,
  variant: "clean" | "stale" | "none",
): Promise<{ answer: string; toolReads: number }> {
  const workspaceRoot = mkdtempSync(`${tmpdir()}/best-agent-project-memory-real-eval-`);
  roots.push(workspaceRoot);
  const storageRoot = join(workspaceRoot, "state");
  mkdirSync(join(workspaceRoot, "facts"), { recursive: true });
  writeFileSync(
    join(workspaceRoot, currentScenario.filePath),
    `${currentScenario.expectedAnswer}\n`,
    "utf8",
  );

  const runtime = new AgentRuntime({
    cwd: workspaceRoot,
    provider: createModelProvider(providerConfig),
    storageRoot,
  });
  if (variant !== "none") {
    const iso =
      variant === "clean"
        ? new Date().toISOString()
        : new Date(Date.now() - 45 * 24 * 60 * 60 * 1000).toISOString();
    writeAutoMemoryFixture(
      runtime,
      currentScenario,
      variant === "clean" ? currentScenario.cleanMemoryEntry : currentScenario.staleMemoryEntry,
      iso,
    );
  }

  const { thread } = await runtime.startThread();
  const result = await runtime.runTurn({
    threadId: thread.id,
    prompt: [
      currentScenario.question,
      "Check the repository if project memory may be outdated.",
      "If you inspect a file, answer with exactly the sentence you find there.",
      "Reply with only the answer sentence.",
    ].join(" "),
  });
  expect(result.turn.status).toBe("completed");

  const history = await runtime.getThreadHistory({ threadId: thread.id });
  const answerItem = [...history.items]
    .reverse()
    .find(
      (item) => item.kind === "message" && item.role === "agent" && item.turnId === result.turn.id,
    );
  const answer =
    answerItem && "text" in answerItem && typeof answerItem.text === "string"
      ? answerItem.text
      : "";
  const toolReads = history.items.filter(
    (item) =>
      item.kind === "tool-call" && item.turnId === result.turn.id && item.toolName === "read-file",
  ).length;

  await runtime.close();
  return { answer, toolReads };
}

function writeAutoMemoryFixture(
  runtime: AgentRuntime,
  scenario: MemoryEvalScenario,
  entry: string,
  createdAt: string,
): void {
  const memory = runtime.getProjectMemory().memory;
  const path = memory.auto?.path;
  if (!path) {
    throw new Error("Missing auto-memory path.");
  }
  mkdirSync(dirname(path), { recursive: true });
  writeFileSync(
    path,
    `${JSON.stringify(
      {
        entries: [
          {
            content: entry,
            createdAt,
            evidence: [
              {
                kind: scenario.kind === "user-rule" ? "user-message" : "shell-command",
                observedAt: createdAt,
                sourceSessionId: `fixture-${scenario.id}`,
                text: scenario.kind === "workspace-command" ? extractCommand(entry) : entry,
              },
            ],
            key: `fixture:${scenario.id}`,
            kind: scenario.kind,
            lastReferencedAt: createdAt,
            observationCount: scenario.kind === "workspace-command" ? 2 : 1,
            referenceCount: 0,
            topics: buildTopics(scenario.kind, entry),
            updatedAt: createdAt,
          },
        ],
        lastConsolidatedAt: createdAt,
        pending: [],
        projectHash: memory.projectHash,
        version: 1,
      },
      null,
      2,
    )}\n`,
    "utf8",
  );
}

function buildTopics(kind: MemoryEvalScenario["kind"], entry: string): string[] {
  return kind === "workspace-command"
    ? Array.from(new Set([...WORKSPACE_COMMAND_TOPICS, ...tokenize(entry)]))
    : Array.from(new Set(["preference", "rule", ...tokenize(entry)]));
}

function extractCommand(entry: string): string {
  return entry.match(/`([^`]+)`/u)?.[1] ?? entry;
}

function tokenize(value: string): string[] {
  return value
    .toLowerCase()
    .split(/[^a-z0-9._/-]+/u)
    .map((token) => token.trim())
    .filter((token) => token.length >= 2);
}

function scenario(
  id: string,
  kind: MemoryEvalScenario["kind"],
  question: string,
  cleanMemoryEntry: string,
  staleMemoryEntry: string,
  expectedAnswer: string,
): MemoryEvalScenario {
  return {
    cleanMemoryEntry,
    expectedAnswer,
    filePath: `facts/${id}.txt`,
    id,
    kind,
    question,
    staleMemoryEntry,
  };
}

function matchesExpectedAnswer(answer: string, expectedAnswer: string): boolean {
  return normalizeForComparison(answer).includes(normalizeForComparison(expectedAnswer));
}

function normalizeForComparison(value: string): string {
  return value.replace(/\s+/gu, " ").trim().toLowerCase();
}

function readRealProviderConfig(): ProviderConfig {
  const kind = process.env.BEST_AGENT_PROVIDER_KIND?.trim() || "openai";
  const genericModel = process.env.BEST_AGENT_PROVIDER_MODEL?.trim();
  const genericApiKey = process.env.BEST_AGENT_PROVIDER_API_KEY?.trim();
  const genericBaseURL = process.env.BEST_AGENT_PROVIDER_BASE_URL?.trim();
  const genericCompatibilityMode = process.env.BEST_AGENT_PROVIDER_COMPATIBILITY_MODE?.trim();

  if (kind === "openai") {
    const model = genericModel || process.env.OPENAI_MODEL?.trim();
    const apiKey = genericApiKey || process.env.OPENAI_API_KEY?.trim();
    if (model && apiKey) {
      return {
        apiKey,
        baseURL: genericBaseURL || process.env.OPENAI_BASE_URL?.trim(),
        compatibilityMode: (genericCompatibilityMode ||
          process.env.OPENAI_COMPATIBILITY_MODE?.trim() ||
          "auto") as ProviderConfig["compatibilityMode"],
        kind,
        model,
      };
    }
  } else if (genericModel && genericApiKey) {
    return {
      apiKey: genericApiKey,
      baseURL: genericBaseURL,
      compatibilityMode: genericCompatibilityMode,
      kind,
      model: genericModel,
    } as ProviderConfig;
  }

  const cliStatePath = resolve(process.cwd(), ".best-agent", "cli-state.json");
  if (!existsSync(cliStatePath)) {
    throw new Error(
      "Missing provider env vars and .best-agent/cli-state.json for real project-memory eval.",
    );
  }
  const parsed = JSON.parse(readFileSync(cliStatePath, "utf8")) as {
    providerConfig?: ProviderConfig;
  };
  if (!parsed.providerConfig) {
    throw new Error("cli-state.json does not contain providerConfig");
  }
  return parsed.providerConfig;
}
