import { registerTokenizerFamily, unpackPackedAsset } from "@cyberlangke/tokkit-core";
export * from "@cyberlangke/tokkit-core";
const BUILTIN_FAMILIES = [
  {
    family: "cosmo-1b",
    aliases: ["cosmo1b", "cosmo_1b"],
    models: ["HuggingFaceTB/cosmo-1b"],
    modulePath: "./generated/cosmo_1b.js"
  },
  {
    family: "smollm",
    aliases: ["smollm2", "smollm_2"],
    models: [
      "HuggingFaceTB/SmolLM-135M",
      "HuggingFaceTB/SmolLM-135M-Instruct",
      "HuggingFaceTB/SmolLM-360M",
      "HuggingFaceTB/SmolLM-360M-Instruct",
      "HuggingFaceTB/SmolLM-1.7B-Instruct",
      "HuggingFaceTB/SmolLM2-135M",
      "HuggingFaceTB/SmolLM2-135M-Instruct",
      "HuggingFaceTB/SmolLM2-360M",
      "HuggingFaceTB/SmolLM2-360M-Instruct",
      "HuggingFaceTB/SmolLM2-1.7B",
      "HuggingFaceTB/SmolLM2-1.7B-Instruct"
    ],
    modulePath: "./generated/smollm.js"
  },
  {
    family: "smollm-1.7b",
    aliases: ["smollm_1_7b"],
    models: ["HuggingFaceTB/SmolLM-1.7B"],
    modulePath: "./generated/smollm_1_7b.js"
  },
  {
    family: "smollm2-16k",
    aliases: ["smollm2_16k"],
    models: ["HuggingFaceTB/SmolLM2-1.7B-Instruct-16k"],
    modulePath: "./generated/smollm2_16k.js"
  },
  {
    family: "smollm3",
    aliases: ["smollm3_3b"],
    models: ["HuggingFaceTB/SmolLM3-3B"],
    modulePath: "./generated/smollm3.js"
  },
  {
    family: "smollm3-base",
    aliases: ["smollm3_base", "smollm3_3b_base"],
    models: ["HuggingFaceTB/SmolLM3-3B-Base"],
    modulePath: "./generated/smollm3_base.js"
  }
];
function registerBuiltins() {
  for (const spec of BUILTIN_FAMILIES) {
    registerTokenizerFamily({
      family: spec.family,
      aliases: [...spec.aliases],
      models: [...spec.models],
      load: () => loadFamilyAsset(spec.modulePath)
    });
  }
}
async function loadFamilyAsset(modulePath) {
  const module = await import(modulePath);
  return unpackPackedAsset(module.default);
}
registerBuiltins();
export {
  registerBuiltins
};
