import { t as f } from "../index-DSGwv2Yx.js";
import "../dataset-DlqAN81i.js";
import { g as a } from "../readers-17HLdxVM.js";
import "../index-Cp39cXWe.js";
const g = 8;
async function p(n, e) {
  const o = (await Promise.all(n.map((t) => e.encodeConversation(t)))).flat();
  return new Uint16Array(o);
}
function c(n) {
  for (let e = n.length - 1; e > 0; e--) {
    const i = Math.floor(Math.random() * (e + 1));
    [n[e], n[i]] = [n[i], n[e]];
  }
  return n;
}
class m {
  tokenizer;
  blockSize;
  constructor(e, i = 128) {
    this.tokenizer = e, this.blockSize = i;
  }
  // Create dataset from text files
  async createTextDataset(e, i = 32, o) {
    if (e.length < this.blockSize + 1)
      throw new Error(`Not enough tokens (${e.length}) for block size ${this.blockSize}`);
    const t = {
      shuffledIndexes: new Uint32Array(e.length),
      step: 0
    };
    if (o)
      t.shuffledIndexes = o;
    else {
      t.shuffledIndexes = new Uint32Array(e.length);
      for (let s = 0; s < e.length; s++)
        t.shuffledIndexes[s] = s;
      c(t.shuffledIndexes);
    }
    const h = (function* () {
      for (; ; ) {
        const s = t.shuffledIndexes[t.step++];
        if (t.step >= t.shuffledIndexes.length && (t.step = 0, c(t.shuffledIndexes)), s + this.blockSize + 1 > e.length)
          continue;
        const r = new Int32Array(e.subarray(s, s + this.blockSize)), l = new Int32Array(e.subarray(s + 1, s + this.blockSize + 1));
        yield { xs: r, ys: l };
      }
    }).bind(this);
    return {
      dataset: a(h).batch(i).map((s) => {
        const r = s;
        return f(() => ({
          xs: r.xs.cast("int32"),
          ys: r.ys.cast("int32")
          // this.tf.oneHot(batchData.ys.cast('int32'), this.tokenizer.vocabSize),
        }));
      }).prefetch(2),
      // Smaller prefetch to reduce memory pressure
      state: t
    };
  }
}
export {
  m as DatasetBuilder,
  g as PAGE_FACTOR,
  p as flattenTokens,
  c as shuffle
};
