import { E as r } from "../index-DvYrXKkX.js";
const l = [
  "<eos>",
  "<bos>",
  "",
  "<pad>",
  "<|user_start|>",
  "<|user_end|>",
  "<|assistant_start|>",
  "<|assistant_end|>",
  "<|system_start|>",
  "<|system_end|>"
];
class T extends r {
  id = "untrained";
  datasetID;
  specialTokens = /* @__PURE__ */ new Map();
  specialTokenSet = /* @__PURE__ */ new Set();
  isSpecialToken(s) {
    return this.specialTokenSet.has(s);
  }
  addSpecialTokens() {
    l.forEach((s, t) => {
      this.addToken(s, t), this.specialTokens.set(s, t), this.specialTokenSet.add(t);
    });
  }
  addSpecialToken(s, t) {
    this.specialTokens.set(s, t), this.specialTokenSet.add(t);
  }
  generateID() {
    const s = this.getVocab();
    let t = 2166136261, e = 2654435769;
    if (s.length === 0) {
      this.id = "untrained";
      return;
    }
    for (let i = 0; i < s.length; i++) {
      const a = s[i];
      t ^= a.length, t = Math.imul(t, 16777619), e ^= i, e = Math.imul(e, 2246822507);
      for (let c = 0; c < a.length; c++) {
        const h = a.charCodeAt(c);
        t ^= h, t = Math.imul(t, 16777619), e ^= h, e = Math.imul(e, 3266489909);
      }
    }
    const o = (t >>> 0).toString(36), n = (e >>> 0).toString(36);
    this.id = "tokeniser_" + o + "_" + n;
  }
  encodeSequence(s) {
    const t = this.encode(s);
    return [this.bosToken, ...t, this.eosToken];
  }
  encodeAsSequence(s, t) {
    const e = s.flatMap((o) => this.encode(o.content));
    return t ? [this.bosToken, ...e, this.eosToken, this.bosToken] : [this.bosToken, ...e, this.eosToken];
  }
  encodeConversation(s, t) {
    const e = [[this.bosToken]], o = [
      this.getSpecialTokenIndex("<|user_start|>"),
      this.getSpecialTokenIndex("<|assistant_start|>"),
      this.getSpecialTokenIndex("<|system_start|>")
    ], n = [
      this.getSpecialTokenIndex("<|user_end|>"),
      this.getSpecialTokenIndex("<|assistant_end|>"),
      this.getSpecialTokenIndex("<|system_end|>")
    ];
    for (const a of s) {
      const c = this.encode(a.content);
      switch (a.role) {
        case "user":
          e.push([o[0]]);
          break;
        case "assistant":
          e.push([o[1]]);
          break;
        case "system":
          e.push([o[2]]);
          break;
      }
      switch (e.push(c), a.role) {
        case "user":
          e.push([n[0]]);
          break;
        case "assistant":
          e.push([n[1]]);
          break;
        case "system":
          e.push([n[2]]);
          break;
      }
    }
    const i = e.flat();
    return t ? i.push(o[1]) : i.push(this.eosToken), i;
  }
  decodeConversation(s) {
    const t = [];
    let e = 0;
    for (; e < s.length; ) {
      const o = s[e];
      let n = null;
      if (o === this.getSpecialTokenIndex("<|user_start|>") ? n = "user" : o === this.getSpecialTokenIndex("<|assistant_start|>") ? n = "assistant" : o === this.getSpecialTokenIndex("<|system_start|>") ? n = "system" : o === this.bosToken || (o === this.eosToken ? n = null : (n = "text", e--)), n) {
        e++;
        const i = [];
        for (; e < s.length && s[e] !== this.getSpecialTokenIndex(`<|${n}_end|>`) && s[e] !== this.eosToken; )
          i.push(s[e]), e++;
        const a = this.decode(i);
        t.push({ role: n, content: a });
      }
      e++;
    }
    return t;
  }
  getSpecialTokenIndex(s) {
    return this.specialTokens.get(s);
  }
}
export {
  l as SPECIALS,
  T as default
};
