# All files / src/tokenizer index.js

100% Statements 49/49
100% Branches 16/16
100% Functions 10/10
100% Lines 45/45

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    27x
    27x
    27x
    27x
    27x
    27x
    27x
    27x

    134x
    134x
    134x
    134x

    9667x
    9667x
    9667x
    9667x

    9667x
    9667x

    9667x
    32924x
    32924x
    32924x
    32924x
    32924x
    32924x

    9667x
    46x

    9621x
    9621x
    9621x

    9621x
    9502x

    9621x

    211956x

    9621x
    3948x
    1963x
    3844x

    9621x

    9621x

    9667x
    9824x

    133x
    9666x

    133x

     // @flow
    import Stream from "../utils/stream";
    import punctuator from "./punctuator";
    import constant from "./constant";
    import identifier from "./identifier";
    import keyword from "./keyword";
    import string from "./string";
    import comments from "./comments";
    import type from "./type";

    type Parsers = ((string) => any)[];

    class Tokenizer {
      stream: Stream;
      tokens: Array<any>;
      pos: number;
      parsers: Parsers;

      constructor(
        stream: Stream,
        parsers: Parsers = [
          punctuator,
          constant,
          identifier,
          keyword,
          string,
          type,
          comments,
        ]
      ) {
        this.stream = stream;
        this.tokens = [];
        this.pos = 0;
        this.parsers = parsers;
      }

      /**
       * Get next token
       */
      next() {
        let value = "";
        this.seekNonWhitespace();
        let char = "";
        let matchers = this.parsers;
        let next;
        let nextMatchers = this.match(char, matchers);
        let start = {
          sourceLine: this.stream.lines[this.stream.line - 1],
          line: this.stream.line,
          col: this.stream.col,
        };

        do {
          char = this.stream.peek();
          matchers = this.match(char, matchers);
          value += char;
          this.stream.next();
          next = this.stream.peek();
          nextMatchers = this.match(next, matchers);
        } while (!this.stream.eof(next) && nextMatchers.length > 0);

        // If we fell off the end then bail out
        if (this.stream.eof(value)) {
          return null;
        }

        const token = this.token(value, matchers);
        token.start = start;
        token.end = {
          sourceLine: this.stream.lines[this.stream.line - 1],
          line: this.stream.line,
          col: this.stream.col,
        };
        // Comments are ignored for now
        if (token.type !== comments.type) {
          this.tokens.push(token);
        }

        return this.tokens[this.pos++];
      }

      match(char: string, parsers: Parsers) {
        return parsers.map(parse => parse(char)).filter(p => p);
      }

      /**
       * Match a particular non-whitespace value to a token
       */
      token(
        value: string,
        parsers: Parsers,
        token: { type: string, value: string, start: {}, end: {} } = {
          type: "unknown",
          value,
          start: {},
          end: {},
        }
      ) {
        // Strict parsers must end on a leaf node
        if (parsers.length > 1) {
          parsers = parsers.filter(parser => (parser.strict ? parser.leaf : true));
          if (parsers.length > 1) {
            parsers = parsers.filter(parser => parser.strict);
          }
        }

        token.type = parsers.pop().type;

        return token;
      }

      /**
       * Seek Stream until next non-whitespace character. Can end in eof/eol
       */
      seekNonWhitespace() {
        while (this.stream.peek() && Stream.whitespace(this.stream.peek())) {
          this.stream.next();
        }
      }

      parse() {
        while (!this.stream.eof(this.stream.peek())) {
          this.next();
        }
        return this.tokens;
      }
    }

    export default Tokenizer;