# All files / src/tokenizer index.js

100% Statements 50/50
94.44% Branches 17/18
100% Functions 10/10
100% Lines 46/46

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132
    133
    28x
    28x
    28x
    28x
    28x
    28x
    28x
    28x

    115x
    115x
    115x
    115x

    5153x
    5153x
    5153x
    5153x

    5153x
    5153x

    5153x
    14158x
    14158x
    14158x
    14158x
    14158x
    14158x

    5153x
    39x

    5114x
    5114x
    5114x

    5114x
    5080x

    5114x

    107251x

    5114x
    2367x
    1173x
    2266x

    5114x
    5114x

    5114x

    5153x
    5893x

    114x
    5152x

    114x

     // @flow
    import Stream from "../utils/stream";
    import punctuator from "./punctuator";
    import constant from "./constant";
    import identifier from "./identifier";
    import keyword from "./keyword";
    import string from "./string";
    import comments from "./comments";
    import type from "./type";

    type Parsers = ((string) => any)[];

    class Tokenizer {
      stream: Stream;
      tokens: Array<any>;
      pos: number;
      parsers: Parsers;

      constructor(
        stream: Stream,
        parsers: Parsers = [
          punctuator,
          constant,
          identifier,
          keyword,
          string,
          type,
          comments,
        ]
      ) {
        this.stream = stream;
        this.tokens = [];
        this.pos = 0;
        this.parsers = parsers;
      }

      /**
       * Get next token
       */
      next() {
        let value = "";
        this.seekNonWhitespace();
        let char = "";
        let matchers = this.parsers;
        let next;
        let nextMatchers = this.match(char, matchers);
        let start = {
          sourceLine: this.stream.lines[this.stream.line - 1],
          line: this.stream.line,
          col: this.stream.col,
        };

        do {
          char = this.stream.peek();
          matchers = this.match(char, matchers);
          value += char;
          this.stream.next();
          next = this.stream.peek();
          nextMatchers = this.match(next, matchers);
        } while (!Stream.eof(next) && nextMatchers.length > 0);

        // If we fell off the end then bail out
        if (Stream.eof(value)) {
          return null;
        }

        const token = this.token(value, matchers);
        token.start = start;
        token.end = {
          sourceLine: this.stream.lines[this.stream.line - 1],
          line: this.stream.line,
          col: this.stream.col,
        };
        // Comments are ignored for now
        if (token.type !== comments.type) {
          this.tokens.push(token);
        }

        return this.tokens[this.pos++];
      }

      match(char: string, parsers: Parsers) {
        return parsers.map(parse => parse(char)).filter(p => p);
      }

      /**
       * Match a particular non-whitespace value to a token
       */
      token(
        value: string,
        parsers: Parsers,
        token: { type: string, value: string, start: {}, end: {} } = {
          type: "unknown",
          value,
          start: {},
          end: {},
        }
      ) {
        // Strict parsers must end on a leaf node
        if (parsers.length > 1) {
          parsers = parsers.filter(parser => (parser.strict ? parser.leaf : true));
          if (parsers.length > 1) {
            parsers = parsers.filter(parser => parser.strict);
          }
        }

        Eif (parsers.length === 1) {
          token.type = parsers[0].type;
        }

        return token;
      }

      /**
       * Seek Stream until next non-whitespace character. Can end in eof/eol
       */
      seekNonWhitespace() {
        while (this.stream.peek() && Stream.whitespace(this.stream.peek())) {
          this.stream.next();
        }
      }

      parse() {
        while (!Stream.eof(this.stream.peek())) {
          this.next();
        }

        return this.tokens;
      }
    }

    export default Tokenizer;