// cli/src/ingest-full.ts
// 构建全量索引：国考/北京(保留现有) + 全量省考职位表(raw/shengkao-full/) + 审核退役标签回贴 + 事业单位。
//
// 数据流：
//   1. 载入现有 positions-index.json.gz，保留 exam∈{guokao,beijing}。
//   2. 读 raw/shengkao-full/<省>.csv（各省最新全量省考职位表，所有岗位）→ exam=shengkao。
//   3. 读 raw/shengkao/<省>.csv（已审核的退役专项清单）逐省匹配回贴"退役士兵"标签：
//      - 优先按 position_code 精确匹配；无代码省按"职位名+单位"唯一匹配（避免一对多过度打标）。
//      - 匹配上的全量行继承审核行的 programs(退役士兵[+基层项目]) 与 service_year。
//      - 未匹配的审核退役行(含四川/黑龙江全量表不可靠的省) 作为独立记录补入，保证退役零丢失。
//   4. 读 raw/shengkao/<省>-shiye.csv（事业单位退役专项）→ exam=shiye。
//   5. province 字段统一规整为中文省名(各省抓取agent写入 slug/中文 不一致)。
//
// Usage: npx tsx src/ingest-full.ts
import { readFileSync, writeFileSync, readdirSync, existsSync } from "node:fs";
import { gzipSync, gunzipSync } from "node:zlib";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
const __dirname = dirname(fileURLToPath(import.meta.url));
const RAW = join(__dirname, "..", "raw");
const FULLDIR = join(RAW, "shengkao-full");
const CURDIR = join(RAW, "shengkao");
const IDX = join(__dirname, "..", "data", "positions-index.json.gz");
// 全量表不可靠(多sheet残表/仅定向表)，仅用其审核退役清单补入
const EXCLUDE_FULL = new Set(["sichuan", "heilongjiang"]);
const GRASSROOTS_RE = /村官|三支一扶|特岗教师|特设岗位|西部计划|服务基层项目|基层服务项目|[一二三四五六七两](类服务人员|类基层项目人员|项目人员|类服务|类基层服务)|四项目人员/;
// 审核退役行统一打"退役士兵"，含基层项目词再加"基层项目"(合并定向)
function curatedTags(text) {
    return GRASSROOTS_RE.test(text) ? ["退役士兵", "基层项目"] : ["退役士兵"];
}
const norm = (s) => (s || "").replace(/\s|（[^）]*）|\([^)]*\)/g, "");
const SLUG2CN = {
    beijing: "北京", tianjin: "天津", hebei: "河北", shanxi: "山西", neimenggu: "内蒙古",
    liaoning: "辽宁", jilin: "吉林", heilongjiang: "黑龙江", shanghai: "上海", jiangsu: "江苏",
    zhejiang: "浙江", anhui: "安徽", fujian: "福建", jiangxi: "江西", shandong: "山东",
    henan: "河南", hubei: "湖北", hunan: "湖南", guangdong: "广东", guangxi: "广西",
    hainan: "海南", chongqing: "重庆", sichuan: "四川", guizhou: "贵州", yunnan: "云南",
    xizang: "西藏", shaanxi: "陕西", gansu: "甘肃", qinghai: "青海", ningxia: "宁夏", xinjiang: "新疆",
};
function cnProv(s) {
    const t = (s || "").trim();
    if (SLUG2CN[t.toLowerCase()])
        return SLUG2CN[t.toLowerCase()];
    return t.replace(/(省|市|自治区|维吾尔自治区|壮族自治区|回族自治区)$/, "");
}
function parseCSV(content) {
    const rows = [];
    let cur = [];
    let field = "";
    let q = false;
    for (let i = 0; i < content.length; i++) {
        const ch = content[i];
        if (q) {
            if (ch === '"' && content[i + 1] === '"') {
                field += '"';
                i++;
            }
            else if (ch === '"')
                q = false;
            else
                field += ch;
        }
        else {
            if (ch === '"')
                q = true;
            else if (ch === ",") {
                cur.push(field);
                field = "";
            }
            else if (ch === "\n" || ch === "\r") {
                if (ch === "\r" && content[i + 1] === "\n")
                    i++;
                cur.push(field);
                if (cur.some((x) => x !== ""))
                    rows.push(cur);
                cur = [];
                field = "";
            }
            else
                field += ch;
        }
    }
    if (field !== "" || cur.length) {
        cur.push(field);
        if (cur.some((x) => x !== ""))
            rows.push(cur);
    }
    return rows;
}
function readCSV(fp) {
    if (!existsSync(fp))
        return [];
    const rows = parseCSV(readFileSync(fp, "utf-8"));
    if (rows.length < 2)
        return [];
    const h = rows[0].map((x) => x.trim());
    return rows.slice(1).map((r) => { const o = {}; h.forEach((k, i) => (o[k] = (r[i] ?? "").trim())); return o; });
}
function toPos(row, exam) {
    const channel = row.channel || "省考";
    return {
        id: row.position_code || "", year: Number(row.year) || 2026, exam,
        dept_code: "", dept_name: row.dept_name || "", bureau: row.bureau || "", inst_type: "", inst_level: "",
        position_name: row.position_name || "", position_attr: "", position_dist: "", position_desc: row.position_desc || "",
        exam_category: channel, headcount: Number(row.headcount) || 1, education: row.education || "", degree: row.degree || "",
        major: row.major || "不限", political: row.political || "不限", grassroots_years: "无限制",
        // 部分省源表无独立"工作地点"列；为空时回退到中文省名(否则落库英文slug)，非空则保留原值
        work_location: row.work_location?.trim() ? row.work_location.trim() : cnProv(row.source_province),
        remarks: row.remarks || "", programs: [],
        province: cnProv(row.source_province), channel, service_year: row.service_year || "",
    };
}
const tagsOf = (r) => curatedTags(`${r.position_name} ${r.position_desc} ${r.remarks}`);
function main() {
    const idx = JSON.parse(gunzipSync(readFileSync(IDX)).toString());
    const out = idx.positions.filter((p) => p.exam === "guokao" || p.exam === "beijing");
    console.log(`保留 guokao+beijing: ${out.length}`);
    // 审核退役(省考)按 slug
    const curBySlug = {};
    for (const f of readdirSync(CURDIR)) {
        if (!f.endsWith(".csv") || f.endsWith("-shiye.csv"))
            continue;
        curBySlug[f.replace(".csv", "")] = readCSV(join(CURDIR, f));
    }
    const matched = new Set();
    let fullCount = 0;
    for (const f of readdirSync(FULLDIR)) {
        if (!f.endsWith(".csv"))
            continue;
        const slug = f.replace(".csv", "");
        if (EXCLUDE_FULL.has(slug))
            continue;
        const fullRows = readCSV(join(FULLDIR, f));
        const cur = curBySlug[slug] || [];
        const fullNameCount = new Map();
        for (const r of fullRows) {
            const k = norm(r.position_name) + "|" + norm(r.dept_name);
            fullNameCount.set(k, (fullNameCount.get(k) || 0) + 1);
        }
        const byCode = new Map();
        const byName = new Map();
        const nameDup = new Set();
        for (const r of cur) {
            if (r.position_code)
                byCode.set(r.position_code.trim(), r);
            const k = norm(r.position_name) + "|" + norm(r.dept_name);
            if (byName.has(k))
                nameDup.add(k);
            else
                byName.set(k, r);
        }
        for (const row of fullRows) {
            const p = toPos(row, "shengkao");
            const k = norm(p.position_name) + "|" + norm(p.dept_name);
            let cr;
            if (p.id && byCode.has(p.id))
                cr = byCode.get(p.id);
            else if (!p.id && byName.has(k) && !nameDup.has(k) && fullNameCount.get(k) === 1)
                cr = byName.get(k);
            if (cr) {
                p.programs = tagsOf(cr);
                if (cr.service_year)
                    p.service_year = cr.service_year;
                // I4: 若全量备注看不出退役依据，补上审核行的招录限制说明，使标签可被备注自证
                if (!/退役|服役|士兵|安置/.test(p.remarks) && /退役|服役|士兵|安置/.test(cr.remarks || "")) {
                    p.remarks = p.remarks ? `${cr.remarks} | ${p.remarks}` : (cr.remarks || "");
                }
                matched.add(cr);
            }
            out.push(p);
            fullCount++;
        }
    }
    console.log(`全量省考(29省): ${fullCount}`);
    // 未匹配的审核退役行作为"补充记录"补入：标 supplementary，仅在按退役专项检索时可见，
    // 避免与全量表中同一岗位(未打标孪生)在全量检索里重复计数。
    let addin = 0;
    for (const slug in curBySlug) {
        for (const r of curBySlug[slug]) {
            if (matched.has(r))
                continue;
            const p = toPos(r, "shengkao");
            p.programs = tagsOf(r);
            p.supplementary = true;
            out.push(p);
            addin++;
        }
    }
    console.log(`未匹配审核退役补入(supplementary): ${addin}`);
    let shiye = 0;
    for (const f of readdirSync(CURDIR)) {
        if (!f.endsWith("-shiye.csv"))
            continue;
        for (const r of readCSV(join(CURDIR, f))) {
            const p = toPos(r, "shiye");
            p.programs = tagsOf(r);
            out.push(p);
            shiye++;
        }
    }
    console.log(`事业单位: ${shiye}`);
    // 逐省覆盖标注(诚实披露)：full=全量职位表；vet_only=仅退役定向子集(四川/黑龙江全量表不可靠)
    const coverage = {};
    for (const p of out) {
        if (p.exam !== "shengkao" || !p.province)
            continue;
        const c = coverage[p.province] ?? (coverage[p.province] = { year: p.year, full_positions: 0, completeness: "full" });
        if (!p.supplementary)
            c.full_positions++;
        c.year = Math.max(c.year, p.year);
    }
    for (const prov of ["四川", "黑龙江"])
        if (coverage[prov])
            coverage[prov].completeness = "vet_only";
    idx.positions = out;
    idx.meta.version = "0.7.1";
    idx.meta.built_at = new Date().toISOString();
    idx.meta.years = [...new Set(out.map((p) => p.year))].sort();
    idx.meta.exams = [...new Set(out.map((p) => p.exam))].sort();
    idx.meta.total_positions = out.length;
    idx.meta.shengkao_coverage = coverage;
    const gz = gzipSync(JSON.stringify(idx));
    writeFileSync(IDX, gz);
    const vet = out.filter((p) => p.programs.includes("退役士兵")).length;
    console.log(`新索引总数 ${out.length}；退役士兵岗 ${vet}；gzip ${(gz.length / 1048576).toFixed(1)}MB；version ${idx.meta.version}`);
}
main();
