// cli/src/ingest-sharded.ts
// 把全部岗位按年份分片：data/positions/<year>.json.gz + data/positions/meta.json
// 原因：全量(国考2020-26 + 31省省考多年 + 事业单位)约87万岗，单一JSON达479MB，
// 超过 node 单字符串上限(~512MB)无法 JSON.parse。按年分片后每片可解析、查询懒加载。
//
// 数据源：
//   旧 positions-index.json.gz   → 取 guokao(2020-26)+beijing(2024)
//   raw/shengkao-full/<省>.csv    → 省考当年全量(29省,各省最新年)
//   raw/shengkao-hist/<省>-<年>.csv → 省考历史全量(含四川/黑龙江全量;与full按省-年去重)
//   raw/shengkao/<省>.csv         → 已审核退役清单(按省-年匹配回贴退役士兵标签;未匹配补 supplementary)
//   raw/shengkao/<省>-shiye.csv   → 事业单位退役专项
//
// Usage: NODE_OPTIONS=--max-old-space-size=8192 npx tsx src/ingest-sharded.ts
import { readFileSync, writeFileSync, readdirSync, existsSync, mkdirSync, rmSync } from "node:fs";
import { gzipSync, gunzipSync } from "node:zlib";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
const __dirname = dirname(fileURLToPath(import.meta.url));
const RAW = join(__dirname, "..", "raw");
const FULLDIR = join(RAW, "shengkao-full");
const HISTDIR = join(RAW, "shengkao-hist");
const CURDIR = join(RAW, "shengkao");
const OUTDIR = join(__dirname, "..", "data", "positions");
const LEGACY_IDX = join(__dirname, "..", "data", "positions-index.json.gz"); // 首次构建时的旧单一索引(若存在)
const GRASSROOTS_RE = /村官|三支一扶|特岗教师|特设岗位|西部计划|服务基层项目|基层服务项目|[一二三四五六七两](类服务人员|类基层项目人员|项目人员|类服务|类基层服务)|四项目人员/;
function curatedTags(text) { return GRASSROOTS_RE.test(text) ? ["退役士兵", "基层项目"] : ["退役士兵"]; }
const norm = (s) => (s || "").replace(/\s|（[^）]*）|\([^)]*\)/g, "");
const SLUG2CN = { beijing: "北京", tianjin: "天津", hebei: "河北", shanxi: "山西", neimenggu: "内蒙古", liaoning: "辽宁", jilin: "吉林", heilongjiang: "黑龙江", shanghai: "上海", jiangsu: "江苏", zhejiang: "浙江", anhui: "安徽", fujian: "福建", jiangxi: "江西", shandong: "山东", henan: "河南", hubei: "湖北", hunan: "湖南", guangdong: "广东", guangxi: "广西", hainan: "海南", chongqing: "重庆", sichuan: "四川", guizhou: "贵州", yunnan: "云南", xizang: "西藏", shaanxi: "陕西", gansu: "甘肃", qinghai: "青海", ningxia: "宁夏", xinjiang: "新疆" };
function cnProv(s) { const t = (s || "").trim(); if (SLUG2CN[t.toLowerCase()])
    return SLUG2CN[t.toLowerCase()]; return t.replace(/(省|市|自治区|维吾尔自治区|壮族自治区|回族自治区)$/, ""); }
function parseCSV(content) {
    const rows = [];
    let cur = [];
    let field = "";
    let q = false;
    for (let i = 0; i < content.length; i++) {
        const ch = content[i];
        if (q) {
            if (ch === '"' && content[i + 1] === '"') {
                field += '"';
                i++;
            }
            else if (ch === '"')
                q = false;
            else
                field += ch;
        }
        else {
            if (ch === '"')
                q = true;
            else if (ch === ",") {
                cur.push(field);
                field = "";
            }
            else if (ch === "\n" || ch === "\r") {
                if (ch === "\r" && content[i + 1] === "\n")
                    i++;
                cur.push(field);
                if (cur.some((x) => x !== ""))
                    rows.push(cur);
                cur = [];
                field = "";
            }
            else
                field += ch;
        }
    }
    if (field !== "" || cur.length) {
        cur.push(field);
        if (cur.some((x) => x !== ""))
            rows.push(cur);
    }
    return rows;
}
function readCSV(fp) {
    if (!existsSync(fp))
        return [];
    const rows = parseCSV(readFileSync(fp, "utf-8"));
    if (rows.length < 2)
        return [];
    const h = rows[0].map((x) => x.replace(/^﻿/, "").trim());
    return rows.slice(1).map((r) => { const o = {}; h.forEach((k, i) => (o[k] = (r[i] ?? "").trim())); return o; });
}
// 这些省抓取时把"类别码"当成了职位代码(唯一率仅1-6%,如海南12码/1098行)，代码不可靠 → 清空,
// 按无代码省处理(岗位本身真实,可按单位/职位/省份检索)。
const GARBAGE_CODE_PROVINCES = new Set(["黑龙江", "吉林", "海南"]);
function toPos(row, exam, year) {
    const channel = row.channel || "省考";
    const provName = cnProv(row.source_province);
    const rawCode = (row.position_code || "").trim();
    const code = GARBAGE_CODE_PROVINCES.has(provName) ? "" : rawCode;
    // 个别省源表无独立"职位名称"列(如西藏部分行)，回退到职位简介，再回退到单位名，避免空名
    const pname = (row.position_name || "").trim() || (row.position_desc || "").trim() || ((row.dept_name || "").trim() ? `${row.dept_name.trim()}岗位` : "");
    return { id: code, year, exam, dept_code: "", dept_name: row.dept_name || "", bureau: row.bureau || "", inst_type: "", inst_level: "",
        position_name: pname, position_attr: "", position_dist: "", position_desc: row.position_desc || "", exam_category: channel,
        headcount: Number(row.headcount) || 1, education: row.education || "", degree: row.degree || "", major: row.major || "不限", political: row.political || "不限",
        grassroots_years: "无限制", work_location: row.work_location?.trim() ? row.work_location.trim() : cnProv(row.source_province),
        remarks: row.remarks || "", programs: [], province: cnProv(row.source_province), channel, service_year: row.service_year || "",
        source: (row.source_url || "").trim() };
}
const tagsOf = (r) => curatedTags(`${r.position_name} ${r.position_desc} ${r.remarks}`);
function main() {
    mkdirSync(OUTDIR, { recursive: true });
    const byYear = new Map();
    const push = (p) => { if (!p.year)
        return; (byYear.get(p.year) ?? byYear.set(p.year, []).get(p.year)).push(p); };
    // 1) 国考 + 北京基底：优先从已有分片读取(可重复运行)；否则回退到旧单一索引(首次构建)。
    // 这两类来自项目原始官方职位表(国家公务员局 bm.scs.gov.cn / 北京市人社局)，补默认 source 溯源。
    const SRC_GUOKAO = "国家公务员局职位表 bm.scs.gov.cn (华图镜像 ah.huatu.com 归档)";
    const SRC_BEIJING = "北京市各级机关考试录用公务员职位表 beijing.gov.cn";
    let gbCount = 0;
    const takeGB = (p) => { if (p.exam !== "guokao" && p.exam !== "beijing")
        return; if (!p.source)
        p.source = p.exam === "guokao" ? SRC_GUOKAO : SRC_BEIJING; push(p); gbCount++; };
    if (existsSync(join(OUTDIR, "meta.json"))) {
        const m = JSON.parse(readFileSync(join(OUTDIR, "meta.json"), "utf-8"));
        for (const y of m.years) {
            const sh = JSON.parse(gunzipSync(readFileSync(join(OUTDIR, `${y}.json.gz`))).toString());
            for (const p of sh.positions)
                takeGB(p);
        }
    }
    else if (existsSync(LEGACY_IDX)) {
        const old = JSON.parse(gunzipSync(readFileSync(LEGACY_IDX)).toString());
        for (const p of old.positions)
            takeGB(p);
    }
    else {
        throw new Error("找不到国考/北京基底数据(既无分片也无旧索引)");
    }
    console.log(`国考+北京基底: ${gbCount}`);
    const coverage = {};
    // 2) 省考当年全量(shengkao-full) — 记录(省,年)已覆盖
    const covered = new Set(); // `${slug}|${year}`
    const fullPos = [];
    for (const f of readdirSync(FULLDIR)) {
        if (!f.endsWith(".csv"))
            continue;
        const slug = f.replace(".csv", "");
        for (const r of readCSV(join(FULLDIR, f))) {
            const y = Number(r.year) || 0;
            if (!y)
                continue;
            covered.add(`${slug}|${y}`);
            const p = toPos(r, "shengkao", y);
            fullPos.push(p);
            push(p);
        }
    }
    // 3) 省考历史全量(shengkao-hist) — 跳过已被 full 覆盖的(省,年)
    for (const f of readdirSync(HISTDIR)) {
        const m = f.match(/^(.+)-(\d{4})\.csv$/);
        if (!m)
            continue;
        const slug = m[1], y = Number(m[2]);
        if (covered.has(`${slug}|${y}`))
            continue;
        for (const r of readCSV(join(HISTDIR, f))) {
            const p = toPos(r, "shengkao", y);
            fullPos.push(p);
            push(p);
        }
        covered.add(`${slug}|${y}`);
    }
    // 4) 退役标签：按(省slug,年)匹配审核清单到全量行；未匹配补 supplementary
    const curBySlugYear = {};
    for (const f of readdirSync(CURDIR)) {
        if (!f.endsWith(".csv") || f.endsWith("-shiye.csv"))
            continue;
        const slug = f.replace(".csv", "");
        for (const r of readCSV(join(CURDIR, f))) {
            const y = Number(r.year) || 0;
            const k = `${slug}|${y}`;
            (curBySlugYear[k] ?? (curBySlugYear[k] = [])).push(r);
        }
    }
    // 全量行按(省中文,年)索引，用于匹配
    const fullBySlugYear = new Map();
    for (const p of fullPos) {
        const slug = Object.keys(SLUG2CN).find((s) => SLUG2CN[s] === p.province);
        if (!slug)
            continue;
        const k = `${slug}|${p.year}`;
        (fullBySlugYear.get(k) ?? fullBySlugYear.set(k, []).get(k)).push(p);
    }
    const matched = new Set();
    let tagged = 0;
    for (const [k, curRows] of Object.entries(curBySlugYear)) {
        const fulls = fullBySlugYear.get(k) || [];
        const nameCount = new Map();
        for (const p of fulls) {
            const nk = norm(p.position_name) + "|" + norm(p.dept_name);
            nameCount.set(nk, (nameCount.get(nk) || 0) + 1);
        }
        const byCode = new Map();
        const byName = new Map();
        for (const p of fulls) {
            if (p.id)
                (byCode.get(p.id) ?? byCode.set(p.id, []).get(p.id)).push(p);
            const nk = norm(p.position_name) + "|" + norm(p.dept_name);
            (byName.get(nk) ?? byName.set(nk, []).get(nk)).push(p);
        }
        for (const r of curRows) {
            const code = (r.position_code || "").trim();
            const nk = norm(r.position_name) + "|" + norm(r.dept_name);
            let target;
            if (code && byCode.has(code))
                target = byCode.get(code)[0];
            else if (byName.has(nk) && nameCount.get(nk) === 1)
                target = byName.get(nk)[0];
            if (target) {
                target.programs = tagsOf(r);
                if (r.service_year)
                    target.service_year = r.service_year;
                if (!/退役|服役|士兵|安置/.test(target.remarks) && /退役|服役|士兵|安置/.test(r.remarks || ""))
                    target.remarks = target.remarks ? `${r.remarks} | ${target.remarks}` : (r.remarks || "");
                matched.add(r);
                tagged++;
            }
        }
    }
    // 未匹配审核退役 → supplementary
    let supp = 0;
    for (const curRows of Object.values(curBySlugYear))
        for (const r of curRows) {
            if (matched.has(r))
                continue;
            const y = Number(r.year) || 0;
            const p = toPos(r, "shengkao", y);
            p.programs = tagsOf(r);
            p.supplementary = true;
            push(p);
            supp++;
        }
    // 5) 事业单位
    let shiye = 0;
    for (const f of readdirSync(CURDIR)) {
        if (!f.endsWith("-shiye.csv"))
            continue;
        for (const r of readCSV(join(CURDIR, f))) {
            const y = Number(r.year) || 2025;
            const p = toPos(r, "shiye", y);
            p.programs = tagsOf(r);
            push(p);
            shiye++;
        }
    }
    // 覆盖标注
    for (const p of fullPos) {
        const prov = p.province;
        if (!prov)
            continue;
        const c = coverage[prov] ?? (coverage[prov] = { year: 0, full_positions: 0, completeness: "full" });
        c.full_positions++;
        c.year = Math.max(c.year, p.year);
    }
    // 去重：仅删"有职位代码且全字段完全相同"的真重复(如海南2026源表自带480条重复行)。
    // 无代码省(重庆/湖南等)的同名同属性行是源表真实独立槽位,保留不删(否则少算招录)。
    let deduped = 0;
    for (const [y, arr] of byYear) {
        const seen = new Set();
        const kept = [];
        for (const p of arr) {
            if (!(p.id || "").trim() || p.supplementary) {
                kept.push(p);
                continue;
            }
            const k = [p.exam, p.id, p.dept_name, p.position_name, p.headcount, p.work_location, p.major, p.education, p.remarks].join("|");
            if (seen.has(k)) {
                deduped++;
                continue;
            }
            seen.add(k);
            kept.push(p);
        }
        byYear.set(y, kept);
    }
    console.log(`去重(同代码全字段真重复): ${deduped}`);
    // 写分片 + meta
    if (existsSync(OUTDIR))
        for (const f of readdirSync(OUTDIR))
            rmSync(join(OUTDIR, f));
    const years = [...byYear.keys()].sort((a, b) => a - b);
    const perYear = {};
    let total = 0, retired = 0;
    for (const y of years) {
        const arr = byYear.get(y);
        perYear[y] = arr.length;
        total += arr.length;
        retired += arr.filter((p) => p.programs.includes("退役士兵")).length;
        writeFileSync(join(OUTDIR, `${y}.json.gz`), gzipSync(JSON.stringify({ year: y, positions: arr })));
    }
    const exams = [...new Set([...byYear.values()].flat().map((p) => p.exam))].sort();
    const meta = { version: "0.9.1", built_at: new Date().toISOString(), years, exams, total_positions: total, per_year: perYear, shengkao_coverage: coverage };
    writeFileSync(join(OUTDIR, "meta.json"), JSON.stringify(meta, null, 1));
    console.log(`分片: ${years.length}年 (${years[0]}-${years[years.length - 1]})`);
    console.log(`各年:`, JSON.stringify(perYear));
    console.log(`总岗位 ${total}；退役士兵标签 ${retired}(matched ${tagged}+supp ${supp})；事业单位 ${shiye}`);
    console.log(`exams: ${exams.join(",")}`);
}
main();
