/**
 * Unique placeholder used to mark greedy-matched regions so they are not
 * re-tokenized by later grammar rules.  The surrounding curly-brace pairs
 * make an accidental collision with real source code extremely unlikely.
 */
const GREEDY_PLACEHOLDER = '\x00GREEDY_PLACEHOLDER\x00';
/**
 * Core tokenizer that converts source code into syntax tokens based on a grammar definition.
 * This is a local replacement for Prism tokenize.
 *
 * The tokenizer works by:
 * 1. Pre-processing greedy patterns to claim their matched regions first
 * 2. Iterating through all grammar rules in order
 * 3. Finding regex matches in the source text
 * 4. Splitting the text into matched (typed) and unmatched (plain) segments
 * 5. Recursively tokenizing nested content when an `inside` grammar is specified
 *
 * @param {string} code - The source code string to tokenize.
 * @param {Grammar} grammar - The grammar rules to apply for tokenization.
 * @returns {(string | SyntaxToken)[]} An array of plain strings and syntax tokens.
 */
export function tokenize(code, grammar) {
  if (!code || !grammar) {
    return [code];
  }
  const tokens = [code];
  matchGrammar(tokens, grammar);
  return tokens;
}
/**
 * Apply grammar rules to tokenize string entries in the token list.
 * Greedy patterns are applied first: their matched regions are replaced with
 * opaque placeholder tokens so that subsequent (non-greedy) rules cannot
 * match inside those regions.  After all rules have run the placeholders are
 * restored to the original matched text.
 *
 * @param {(string | SyntaxToken)[]} tokens - The current token list to process in-place.
 * @param {Grammar} grammar - The grammar rules to apply.
 */
function matchGrammar(tokens, grammar) {
  const greedyEntries = [];
  const nonGreedyEntries = [];
  for (const tokenType of Object.keys(grammar)) {
    const grammarEntry = grammar[tokenType];
    const patterns = Array.isArray(grammarEntry) ? grammarEntry : [grammarEntry];
    for (const patternDef of patterns) {
      if (patternDef.greedy) {
        greedyEntries.push({ tokenType, patternDef });
      }
      else {
        nonGreedyEntries.push({ tokenType, patternDef });
      }
    }
  }
  // --- Phase 1: Apply greedy patterns first --------------------------------
  // Greedy patterns claim regions of the source text exclusively.
  // We replace matched regions with a placeholder string of equal length so
  // that character offsets remain stable.  Each greedy match is recorded so
  // we can later restore the original text inside the token.
  if (greedyEntries.length > 0) {
    applyGreedyPatterns(tokens, greedyEntries);
  }
  // --- Phase 2: Apply remaining non-greedy patterns ------------------------
  for (const { tokenType, patternDef } of nonGreedyEntries) {
    applyPattern(tokens, tokenType, patternDef);
  }
}
/**
 * Process all greedy patterns against the current token list.
 *
 * For every plain-string entry in the list the function builds a *mask* string
 * of the same length.  Greedy matches are recorded and their corresponding
 * character ranges in the mask are overwritten with a placeholder character so
 * that later greedy patterns (and all non-greedy patterns) will never match
 * inside an already-claimed region.
 *
 * After all greedy patterns have been collected the affected string entries are
 * rebuilt by splicing in SyntaxToken objects for each greedy match.
 *
 * @param {(string | SyntaxToken)[]} tokens - The token list to process in-place.
 * @param {{ tokenType: string; patternDef: GrammarPattern }[]} greedyEntries - Greedy patterns to apply.
 */
function applyGreedyPatterns(tokens, greedyEntries) {
  for (let i = 0; i < tokens.length; i++) {
    const token = tokens[i];
    /* istanbul ignore next -- defensive: greedy phase only sees strings from initial tokenize() call */
    if (typeof token !== 'string') {
      continue;
    }
    // Build the full text by concatenating all consecutive string segments
    // so greedy regex can match across them — but for simplicity we operate
    // per-segment, which matches prismjs behavior for the patterns we use.
    const originalText = token;
    // Mask tracks which characters have been claimed.  A GREEDY_PLACEHOLDER
    // character at position j means that character is already part of a greedy match.
    let mask = originalText;
    const matches = [];
    for (const { tokenType, patternDef } of greedyEntries) {
      const { pattern, lookbehind, inside } = patternDef;
      const regex = new RegExp(pattern.source, pattern.flags.replace(/g/g, '') + 'g');
      let execResult;
      while ((execResult = regex.exec(mask)) !== null) {
        let matchIndex = execResult.index;
        let match = execResult[0];
        /* istanbul ignore next -- defensive: prevents infinite loop on zero-length regex matches */
        if (match.length === 0) {
          regex.lastIndex++;
          continue;
        }
        // If matched text contains a placeholder char it overlaps a
        // previously claimed region — skip it.
        /* istanbul ignore next -- defensive: prevents re-tokenizing already-claimed greedy regions */
        if (match.indexOf(GREEDY_PLACEHOLDER) !== -1) {
          continue;
        }
        if (lookbehind && execResult[1]) {
          const lookbehindLength = execResult[1].length;
          matchIndex += lookbehindLength;
          match = match.slice(lookbehindLength);
        }
        // Record the match and blank out the region in the mask.
        const matchEnd = matchIndex + match.length;
        matches.push({
          tokenType,
          matchStart: matchIndex,
          matchEnd,
          matchText: originalText.slice(matchIndex, matchEnd),
          inside,
        });
        // Replace matched region in the mask with placeholder so later
        // patterns cannot match inside it.
        mask =
          mask.slice(0, matchIndex) +
            GREEDY_PLACEHOLDER.repeat(match.length).slice(0, match.length) +
            mask.slice(matchEnd);
      }
    }
    if (matches.length === 0) {
      continue;
    }
    // Sort matches by start position for deterministic splicing.
    matches.sort((a, b) => a.matchStart - b.matchStart);
    // Build replacement array.
    const replacement = [];
    let lastIndex = 0;
    for (const m of matches) {
      if (m.matchStart > lastIndex) {
        replacement.push(originalText.slice(lastIndex, m.matchStart));
      }
      let content = m.matchText;
      if (m.inside) {
        const nested = tokenize(m.matchText, m.inside);
        content = nested.length === 1 ? nested[0] : nested;
      }
      replacement.push({ type: m.tokenType, content });
      lastIndex = m.matchEnd;
    }
    if (lastIndex < originalText.length) {
      replacement.push(originalText.slice(lastIndex));
    }
    tokens.splice(i, 1, ...replacement);
    i += replacement.length - 1;
  }
}
/**
 * Apply a single pattern definition across all string entries in the token list.
 * For each string entry, finds ALL matches of the pattern and splits the string
 * into alternating unmatched/matched segments in one pass.
 *
 * @param {(string | SyntaxToken)[]} tokens - The token list to process.
 * @param {string} tokenType - The token type name for matched content.
 * @param {GrammarPattern} patternDef - The pattern definition to apply.
 */
function applyPattern(tokens, tokenType, patternDef) {
  for (let i = 0; i < tokens.length; i++) {
    const token = tokens[i];
    // Only process string tokens, skip already-tokenized entries
    if (typeof token !== 'string') {
      continue;
    }
    const replacement = tokenizeStringByPattern(token, tokenType, patternDef);
    // Only splice if matches were found (replacement differs from original)
    if (replacement.length === 1 && typeof replacement[0] === 'string') {
      continue;
    }
    tokens.splice(i, 1, ...replacement);
    // Skip past the entire replacement — all matches in this string are handled
    i += replacement.length - 1;
  }
}
/**
 * Find all matches of a pattern within a string and return the split result
 * as an array of alternating plain strings and syntax tokens.
 *
 * @param {string} text - The string to tokenize.
 * @param {string} tokenType - The token type name for matched content.
 * @param {GrammarPattern} patternDef - The pattern definition to apply.
 * @returns {(string | SyntaxToken)[]} The tokenized result.
 */
function tokenizeStringByPattern(text, tokenType, patternDef) {
  const { pattern, lookbehind, inside } = patternDef;
  const regex = new RegExp(pattern.source, pattern.flags.replace(/g/g, '') + 'g');
  const result = [];
  let lastIndex = 0;
  let execResult;
  while ((execResult = regex.exec(text)) !== null) {
    let matchIndex = execResult.index;
    let match = execResult[0];
    // Prevent zero-length match infinite loops
    if (match.length === 0) {
      regex.lastIndex++;
      continue;
    }
    // Handle lookbehind: adjust match position and content
    if (lookbehind && execResult[1]) {
      const lookbehindLength = execResult[1].length;
      matchIndex += lookbehindLength;
      match = match.slice(lookbehindLength);
    }
    // Add unmatched text before this match
    if (matchIndex > lastIndex) {
      result.push(text.slice(lastIndex, matchIndex));
    }
    // Process nested grammar if specified
    let content = match;
    if (inside) {
      const nestedTokens = tokenize(match, inside);
      content = nestedTokens.length === 1 ? nestedTokens[0] : nestedTokens;
    }
    result.push({ type: tokenType, content });
    lastIndex = matchIndex + match.length;
  }
  // Add remaining unmatched text
  if (lastIndex < text.length) {
    result.push(text.slice(lastIndex));
  }
  /* istanbul ignore next -- defensive: tokenizeStringByPattern always has remainder text */
  if (result.length === 0) {
    return [text];
  }
  return result;
}
