import { __exports as tokenizer } from '../../../../../../../../_virtual/tokenizer2.js';
import { __require as requireInputscanner } from '../core/inputscanner.js';
import { __require as requireTokenizer$1 } from '../core/tokenizer.js';
import { __require as requireDirectives } from '../core/directives.js';
import { __require as requireAcorn } from './acorn.js';
import { __require as requirePattern } from '../core/pattern.js';
import { __require as requireTemplatablepattern } from '../core/templatablepattern.js';

/*jshint node:true */

var hasRequiredTokenizer;

function requireTokenizer () {
	if (hasRequiredTokenizer) return tokenizer;
	hasRequiredTokenizer = 1;

	var InputScanner = requireInputscanner().InputScanner;
	var BaseTokenizer = requireTokenizer$1().Tokenizer;
	var BASETOKEN = requireTokenizer$1().TOKEN;
	var Directives = requireDirectives().Directives;
	var acorn = requireAcorn();
	var Pattern = requirePattern().Pattern;
	var TemplatablePattern = requireTemplatablepattern().TemplatablePattern;


	function in_array(what, arr) {
	  return arr.indexOf(what) !== -1;
	}


	var TOKEN = {
	  START_EXPR: 'TK_START_EXPR',
	  END_EXPR: 'TK_END_EXPR',
	  START_BLOCK: 'TK_START_BLOCK',
	  END_BLOCK: 'TK_END_BLOCK',
	  WORD: 'TK_WORD',
	  RESERVED: 'TK_RESERVED',
	  SEMICOLON: 'TK_SEMICOLON',
	  STRING: 'TK_STRING',
	  EQUALS: 'TK_EQUALS',
	  OPERATOR: 'TK_OPERATOR',
	  COMMA: 'TK_COMMA',
	  BLOCK_COMMENT: 'TK_BLOCK_COMMENT',
	  COMMENT: 'TK_COMMENT',
	  DOT: 'TK_DOT',
	  UNKNOWN: 'TK_UNKNOWN',
	  START: BASETOKEN.START,
	  RAW: BASETOKEN.RAW,
	  EOF: BASETOKEN.EOF
	};


	var directives_core = new Directives(/\/\*/, /\*\//);

	var number_pattern = /0[xX][0123456789abcdefABCDEF_]*n?|0[oO][01234567_]*n?|0[bB][01_]*n?|\d[\d_]*n|(?:\.\d[\d_]*|\d[\d_]*\.?[\d_]*)(?:[eE][+-]?[\d_]+)?/;

	var digit = /[0-9]/;

	// Dot "." must be distinguished from "..." and decimal
	var dot_pattern = /[^\d\.]/;

	var positionable_operators = (
	  ">>> === !== &&= ??= ||= " +
	  "<< && >= ** != == <= >> || ?? |> " +
	  "< / - + > : & % ? ^ | *").split(' ');

	// IMPORTANT: this must be sorted longest to shortest or tokenizing many not work.
	// Also, you must update possitionable operators separately from punct
	var punct =
	  ">>>= " +
	  "... >>= <<= === >>> !== **= &&= ??= ||= " +
	  "=> ^= :: /= << <= == && -= >= >> != -- += ** || ?? ++ %= &= *= |= |> " +
	  "= ! ? > < : / ^ - + * & % ~ |";

	punct = punct.replace(/[-[\]{}()*+?.,\\^$|#]/g, "\\$&");
	// ?. but not if followed by a number 
	punct = '\\?\\.(?!\\d) ' + punct;
	punct = punct.replace(/ /g, '|');

	var punct_pattern = new RegExp(punct);

	// words which should always start on new line.
	var line_starters = 'continue,try,throw,return,var,let,const,if,switch,case,default,for,while,break,function,import,export'.split(',');
	var reserved_words = line_starters.concat(['do', 'in', 'of', 'else', 'get', 'set', 'new', 'catch', 'finally', 'typeof', 'yield', 'async', 'await', 'from', 'as', 'class', 'extends']);
	var reserved_word_pattern = new RegExp('^(?:' + reserved_words.join('|') + ')$');

	// var template_pattern = /(?:(?:<\?php|<\?=)[\s\S]*?\?>)|(?:<%[\s\S]*?%>)/g;

	var in_html_comment;

	var Tokenizer = function(input_string, options) {
	  BaseTokenizer.call(this, input_string, options);

	  this._patterns.whitespace = this._patterns.whitespace.matching(
	    /\u00A0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff/.source,
	    /\u2028\u2029/.source);

	  var pattern_reader = new Pattern(this._input);
	  var templatable = new TemplatablePattern(this._input)
	    .read_options(this._options);

	  this.__patterns = {
	    template: templatable,
	    identifier: templatable.starting_with(acorn.identifier).matching(acorn.identifierMatch),
	    number: pattern_reader.matching(number_pattern),
	    punct: pattern_reader.matching(punct_pattern),
	    // comment ends just before nearest linefeed or end of file
	    comment: pattern_reader.starting_with(/\/\//).until(/[\n\r\u2028\u2029]/),
	    //  /* ... */ comment ends with nearest */ or end of file
	    block_comment: pattern_reader.starting_with(/\/\*/).until_after(/\*\//),
	    html_comment_start: pattern_reader.matching(/<!--/),
	    html_comment_end: pattern_reader.matching(/-->/),
	    include: pattern_reader.starting_with(/#include/).until_after(acorn.lineBreak),
	    shebang: pattern_reader.starting_with(/#!/).until_after(acorn.lineBreak),
	    xml: pattern_reader.matching(/[\s\S]*?<(\/?)([-a-zA-Z:0-9_.]+|{[^}]+?}|!\[CDATA\[[^\]]*?\]\]|)(\s*{[^}]+?}|\s+[-a-zA-Z:0-9_.]+|\s+[-a-zA-Z:0-9_.]+\s*=\s*('[^']*'|"[^"]*"|{([^{}]|{[^}]+?})+?}))*\s*(\/?)\s*>/),
	    single_quote: templatable.until(/['\\\n\r\u2028\u2029]/),
	    double_quote: templatable.until(/["\\\n\r\u2028\u2029]/),
	    template_text: templatable.until(/[`\\$]/),
	    template_expression: templatable.until(/[`}\\]/)
	  };

	};
	Tokenizer.prototype = new BaseTokenizer();

	Tokenizer.prototype._is_comment = function(current_token) {
	  return current_token.type === TOKEN.COMMENT || current_token.type === TOKEN.BLOCK_COMMENT || current_token.type === TOKEN.UNKNOWN;
	};

	Tokenizer.prototype._is_opening = function(current_token) {
	  return current_token.type === TOKEN.START_BLOCK || current_token.type === TOKEN.START_EXPR;
	};

	Tokenizer.prototype._is_closing = function(current_token, open_token) {
	  return (current_token.type === TOKEN.END_BLOCK || current_token.type === TOKEN.END_EXPR) &&
	    (open_token && (
	      (current_token.text === ']' && open_token.text === '[') ||
	      (current_token.text === ')' && open_token.text === '(') ||
	      (current_token.text === '}' && open_token.text === '{')));
	};

	Tokenizer.prototype._reset = function() {
	  in_html_comment = false;
	};

	Tokenizer.prototype._get_next_token = function(previous_token, open_token) { // jshint unused:false
	  var token = null;
	  this._readWhitespace();
	  var c = this._input.peek();

	  if (c === null) {
	    return this._create_token(TOKEN.EOF, '');
	  }

	  token = token || this._read_non_javascript(c);
	  token = token || this._read_string(c);
	  token = token || this._read_pair(c, this._input.peek(1)); // Issue #2062 hack for record type '#{'
	  token = token || this._read_word(previous_token);
	  token = token || this._read_singles(c);
	  token = token || this._read_comment(c);
	  token = token || this._read_regexp(c, previous_token);
	  token = token || this._read_xml(c, previous_token);
	  token = token || this._read_punctuation();
	  token = token || this._create_token(TOKEN.UNKNOWN, this._input.next());

	  return token;
	};

	Tokenizer.prototype._read_word = function(previous_token) {
	  var resulting_string;
	  resulting_string = this.__patterns.identifier.read();
	  if (resulting_string !== '') {
	    resulting_string = resulting_string.replace(acorn.allLineBreaks, '\n');
	    if (!(previous_token.type === TOKEN.DOT ||
	        (previous_token.type === TOKEN.RESERVED && (previous_token.text === 'set' || previous_token.text === 'get'))) &&
	      reserved_word_pattern.test(resulting_string)) {
	      if ((resulting_string === 'in' || resulting_string === 'of') &&
	        (previous_token.type === TOKEN.WORD || previous_token.type === TOKEN.STRING)) { // hack for 'in' and 'of' operators
	        return this._create_token(TOKEN.OPERATOR, resulting_string);
	      }
	      return this._create_token(TOKEN.RESERVED, resulting_string);
	    }
	    return this._create_token(TOKEN.WORD, resulting_string);
	  }

	  resulting_string = this.__patterns.number.read();
	  if (resulting_string !== '') {
	    return this._create_token(TOKEN.WORD, resulting_string);
	  }
	};

	Tokenizer.prototype._read_singles = function(c) {
	  var token = null;
	  if (c === '(' || c === '[') {
	    token = this._create_token(TOKEN.START_EXPR, c);
	  } else if (c === ')' || c === ']') {
	    token = this._create_token(TOKEN.END_EXPR, c);
	  } else if (c === '{') {
	    token = this._create_token(TOKEN.START_BLOCK, c);
	  } else if (c === '}') {
	    token = this._create_token(TOKEN.END_BLOCK, c);
	  } else if (c === ';') {
	    token = this._create_token(TOKEN.SEMICOLON, c);
	  } else if (c === '.' && dot_pattern.test(this._input.peek(1))) {
	    token = this._create_token(TOKEN.DOT, c);
	  } else if (c === ',') {
	    token = this._create_token(TOKEN.COMMA, c);
	  }

	  if (token) {
	    this._input.next();
	  }
	  return token;
	};

	Tokenizer.prototype._read_pair = function(c, d) {
	  var token = null;
	  if (c === '#' && d === '{') {
	    token = this._create_token(TOKEN.START_BLOCK, c + d);
	  }

	  if (token) {
	    this._input.next();
	    this._input.next();
	  }
	  return token;
	};

	Tokenizer.prototype._read_punctuation = function() {
	  var resulting_string = this.__patterns.punct.read();

	  if (resulting_string !== '') {
	    if (resulting_string === '=') {
	      return this._create_token(TOKEN.EQUALS, resulting_string);
	    } else if (resulting_string === '?.') {
	      return this._create_token(TOKEN.DOT, resulting_string);
	    } else {
	      return this._create_token(TOKEN.OPERATOR, resulting_string);
	    }
	  }
	};

	Tokenizer.prototype._read_non_javascript = function(c) {
	  var resulting_string = '';

	  if (c === '#') {
	    if (this._is_first_token()) {
	      resulting_string = this.__patterns.shebang.read();

	      if (resulting_string) {
	        return this._create_token(TOKEN.UNKNOWN, resulting_string.trim() + '\n');
	      }
	    }

	    // handles extendscript #includes
	    resulting_string = this.__patterns.include.read();

	    if (resulting_string) {
	      return this._create_token(TOKEN.UNKNOWN, resulting_string.trim() + '\n');
	    }

	    c = this._input.next();

	    // Spidermonkey-specific sharp variables for circular references. Considered obsolete.
	    var sharp = '#';
	    if (this._input.hasNext() && this._input.testChar(digit)) {
	      do {
	        c = this._input.next();
	        sharp += c;
	      } while (this._input.hasNext() && c !== '#' && c !== '=');
	      if (c === '#') ; else if (this._input.peek() === '[' && this._input.peek(1) === ']') {
	        sharp += '[]';
	        this._input.next();
	        this._input.next();
	      } else if (this._input.peek() === '{' && this._input.peek(1) === '}') {
	        sharp += '{}';
	        this._input.next();
	        this._input.next();
	      }
	      return this._create_token(TOKEN.WORD, sharp);
	    }

	    this._input.back();

	  } else if (c === '<' && this._is_first_token()) {
	    resulting_string = this.__patterns.html_comment_start.read();
	    if (resulting_string) {
	      while (this._input.hasNext() && !this._input.testChar(acorn.newline)) {
	        resulting_string += this._input.next();
	      }
	      in_html_comment = true;
	      return this._create_token(TOKEN.COMMENT, resulting_string);
	    }
	  } else if (in_html_comment && c === '-') {
	    resulting_string = this.__patterns.html_comment_end.read();
	    if (resulting_string) {
	      in_html_comment = false;
	      return this._create_token(TOKEN.COMMENT, resulting_string);
	    }
	  }

	  return null;
	};

	Tokenizer.prototype._read_comment = function(c) {
	  var token = null;
	  if (c === '/') {
	    var comment = '';
	    if (this._input.peek(1) === '*') {
	      // peek for comment /* ... */
	      comment = this.__patterns.block_comment.read();
	      var directives = directives_core.get_directives(comment);
	      if (directives && directives.ignore === 'start') {
	        comment += directives_core.readIgnored(this._input);
	      }
	      comment = comment.replace(acorn.allLineBreaks, '\n');
	      token = this._create_token(TOKEN.BLOCK_COMMENT, comment);
	      token.directives = directives;
	    } else if (this._input.peek(1) === '/') {
	      // peek for comment // ...
	      comment = this.__patterns.comment.read();
	      token = this._create_token(TOKEN.COMMENT, comment);
	    }
	  }
	  return token;
	};

	Tokenizer.prototype._read_string = function(c) {
	  if (c === '`' || c === "'" || c === '"') {
	    var resulting_string = this._input.next();
	    this.has_char_escapes = false;

	    if (c === '`') {
	      resulting_string += this._read_string_recursive('`', true, '${');
	    } else {
	      resulting_string += this._read_string_recursive(c);
	    }

	    if (this.has_char_escapes && this._options.unescape_strings) {
	      resulting_string = unescape_string(resulting_string);
	    }

	    if (this._input.peek() === c) {
	      resulting_string += this._input.next();
	    }

	    resulting_string = resulting_string.replace(acorn.allLineBreaks, '\n');

	    return this._create_token(TOKEN.STRING, resulting_string);
	  }

	  return null;
	};

	Tokenizer.prototype._allow_regexp_or_xml = function(previous_token) {
	  // regex and xml can only appear in specific locations during parsing
	  return (previous_token.type === TOKEN.RESERVED && in_array(previous_token.text, ['return', 'case', 'throw', 'else', 'do', 'typeof', 'yield'])) ||
	    (previous_token.type === TOKEN.END_EXPR && previous_token.text === ')' &&
	      previous_token.opened.previous.type === TOKEN.RESERVED && in_array(previous_token.opened.previous.text, ['if', 'while', 'for'])) ||
	    (in_array(previous_token.type, [TOKEN.COMMENT, TOKEN.START_EXPR, TOKEN.START_BLOCK, TOKEN.START,
	      TOKEN.END_BLOCK, TOKEN.OPERATOR, TOKEN.EQUALS, TOKEN.EOF, TOKEN.SEMICOLON, TOKEN.COMMA
	    ]));
	};

	Tokenizer.prototype._read_regexp = function(c, previous_token) {

	  if (c === '/' && this._allow_regexp_or_xml(previous_token)) {
	    // handle regexp
	    //
	    var resulting_string = this._input.next();
	    var esc = false;

	    var in_char_class = false;
	    while (this._input.hasNext() &&
	      ((esc || in_char_class || this._input.peek() !== c) &&
	        !this._input.testChar(acorn.newline))) {
	      resulting_string += this._input.peek();
	      if (!esc) {
	        esc = this._input.peek() === '\\';
	        if (this._input.peek() === '[') {
	          in_char_class = true;
	        } else if (this._input.peek() === ']') {
	          in_char_class = false;
	        }
	      } else {
	        esc = false;
	      }
	      this._input.next();
	    }

	    if (this._input.peek() === c) {
	      resulting_string += this._input.next();

	      // regexps may have modifiers /regexp/MOD , so fetch those, too
	      // Only [gim] are valid, but if the user puts in garbage, do what we can to take it.
	      resulting_string += this._input.read(acorn.identifier);
	    }
	    return this._create_token(TOKEN.STRING, resulting_string);
	  }
	  return null;
	};

	Tokenizer.prototype._read_xml = function(c, previous_token) {

	  if (this._options.e4x && c === "<" && this._allow_regexp_or_xml(previous_token)) {
	    var xmlStr = '';
	    var match = this.__patterns.xml.read_match();
	    // handle e4x xml literals
	    //
	    if (match) {
	      // Trim root tag to attempt to
	      var rootTag = match[2].replace(/^{\s+/, '{').replace(/\s+}$/, '}');
	      var isCurlyRoot = rootTag.indexOf('{') === 0;
	      var depth = 0;
	      while (match) {
	        var isEndTag = !!match[1];
	        var tagName = match[2];
	        var isSingletonTag = (!!match[match.length - 1]) || (tagName.slice(0, 8) === "![CDATA[");
	        if (!isSingletonTag &&
	          (tagName === rootTag || (isCurlyRoot && tagName.replace(/^{\s+/, '{').replace(/\s+}$/, '}')))) {
	          if (isEndTag) {
	            --depth;
	          } else {
	            ++depth;
	          }
	        }
	        xmlStr += match[0];
	        if (depth <= 0) {
	          break;
	        }
	        match = this.__patterns.xml.read_match();
	      }
	      // if we didn't close correctly, keep unformatted.
	      if (!match) {
	        xmlStr += this._input.match(/[\s\S]*/g)[0];
	      }
	      xmlStr = xmlStr.replace(acorn.allLineBreaks, '\n');
	      return this._create_token(TOKEN.STRING, xmlStr);
	    }
	  }

	  return null;
	};

	function unescape_string(s) {
	  // You think that a regex would work for this
	  // return s.replace(/\\x([0-9a-f]{2})/gi, function(match, val) {
	  //         return String.fromCharCode(parseInt(val, 16));
	  //     })
	  // However, dealing with '\xff', '\\xff', '\\\xff' makes this more fun.
	  var out = '',
	    escaped = 0;

	  var input_scan = new InputScanner(s);
	  var matched = null;

	  while (input_scan.hasNext()) {
	    // Keep any whitespace, non-slash characters
	    // also keep slash pairs.
	    matched = input_scan.match(/([\s]|[^\\]|\\\\)+/g);

	    if (matched) {
	      out += matched[0];
	    }

	    if (input_scan.peek() === '\\') {
	      input_scan.next();
	      if (input_scan.peek() === 'x') {
	        matched = input_scan.match(/x([0-9A-Fa-f]{2})/g);
	      } else if (input_scan.peek() === 'u') {
	        matched = input_scan.match(/u([0-9A-Fa-f]{4})/g);
	        if (!matched) {
	          matched = input_scan.match(/u\{([0-9A-Fa-f]+)\}/g);
	        }
	      } else {
	        out += '\\';
	        if (input_scan.hasNext()) {
	          out += input_scan.next();
	        }
	        continue;
	      }

	      // If there's some error decoding, return the original string
	      if (!matched) {
	        return s;
	      }

	      escaped = parseInt(matched[1], 16);

	      if (escaped > 0x7e && escaped <= 0xff && matched[0].indexOf('x') === 0) {
	        // we bail out on \x7f..\xff,
	        // leaving whole string escaped,
	        // as it's probably completely binary
	        return s;
	      } else if (escaped >= 0x00 && escaped < 0x20) {
	        // leave 0x00...0x1f escaped
	        out += '\\' + matched[0];
	      } else if (escaped > 0x10FFFF) {
	        // If the escape sequence is out of bounds, keep the original sequence and continue conversion
	        out += '\\' + matched[0];
	      } else if (escaped === 0x22 || escaped === 0x27 || escaped === 0x5c) {
	        // single-quote, apostrophe, backslash - escape these
	        out += '\\' + String.fromCharCode(escaped);
	      } else {
	        out += String.fromCharCode(escaped);
	      }
	    }
	  }

	  return out;
	}

	// handle string
	//
	Tokenizer.prototype._read_string_recursive = function(delimiter, allow_unescaped_newlines, start_sub) {
	  var current_char;
	  var pattern;
	  if (delimiter === '\'') {
	    pattern = this.__patterns.single_quote;
	  } else if (delimiter === '"') {
	    pattern = this.__patterns.double_quote;
	  } else if (delimiter === '`') {
	    pattern = this.__patterns.template_text;
	  } else if (delimiter === '}') {
	    pattern = this.__patterns.template_expression;
	  }

	  var resulting_string = pattern.read();
	  var next = '';
	  while (this._input.hasNext()) {
	    next = this._input.next();
	    if (next === delimiter ||
	      (!allow_unescaped_newlines && acorn.newline.test(next))) {
	      this._input.back();
	      break;
	    } else if (next === '\\' && this._input.hasNext()) {
	      current_char = this._input.peek();

	      if (current_char === 'x' || current_char === 'u') {
	        this.has_char_escapes = true;
	      } else if (current_char === '\r' && this._input.peek(1) === '\n') {
	        this._input.next();
	      }
	      next += this._input.next();
	    } else if (start_sub) {
	      if (start_sub === '${' && next === '$' && this._input.peek() === '{') {
	        next += this._input.next();
	      }

	      if (start_sub === next) {
	        if (delimiter === '`') {
	          next += this._read_string_recursive('}', allow_unescaped_newlines, '`');
	        } else {
	          next += this._read_string_recursive('`', allow_unescaped_newlines, '${');
	        }
	        if (this._input.hasNext()) {
	          next += this._input.next();
	        }
	      }
	    }
	    next += pattern.read();
	    resulting_string += next;
	  }

	  return resulting_string;
	};

	tokenizer.Tokenizer = Tokenizer;
	tokenizer.TOKEN = TOKEN;
	tokenizer.positionable_operators = positionable_operators.slice();
	tokenizer.line_starters = line_starters.slice();
	return tokenizer;
}

export { requireTokenizer as __require };
//# sourceMappingURL=tokenizer.js.map
