# All files / src/tokenizer index.js

100% Statements 49/49
100% Branches 16/16
100% Functions 10/10
100% Lines 45/45

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    26x
    26x
    26x
    26x
    26x
    26x
    26x
    26x

    117x
    117x
    117x
    117x

    6897x
    6897x
    6897x
    6897x

    6897x
    6897x

    6897x
    21452x
    21452x
    21452x
    21452x
    21452x
    21452x

    6897x
    35x

    6862x
    6862x
    6862x

    6862x
    6817x

    6862x

    146958x

    6862x
    2810x
    1395x
    2714x

    6862x

    6862x

    6897x
    7008x

    116x
    6896x

    116x

     // @flow
    import Stream from "../utils/stream";
    import punctuator from "./punctuator";
    import constant from "./constant";
    import identifier from "./identifier";
    import keyword from "./keyword";
    import string from "./string";
    import comments from "./comments";
    import type from "./type";

    type Parsers = ((string) => any)[];

    class Tokenizer {
      stream: Stream;
      tokens: Array<any>;
      pos: number;
      parsers: Parsers;

      constructor(
        stream: Stream,
        parsers: Parsers = [
          punctuator,
          constant,
          identifier,
          keyword,
          string,
          type,
          comments,
        ]
      ) {
        this.stream = stream;
        this.tokens = [];
        this.pos = 0;
        this.parsers = parsers;
      }

      /**
       * Get next token
       */
      next() {
        let value = "";
        this.seekNonWhitespace();
        let char = "";
        let matchers = this.parsers;
        let next;
        let nextMatchers = this.match(char, matchers);
        let start = {
          sourceLine: this.stream.lines[this.stream.line - 1],
          line: this.stream.line,
          col: this.stream.col,
        };

        do {
          char = this.stream.peek();
          matchers = this.match(char, matchers);
          value += char;
          this.stream.next();
          next = this.stream.peek();
          nextMatchers = this.match(next, matchers);
        } while (!Stream.eof(next) && nextMatchers.length > 0);

        // If we fell off the end then bail out
        if (Stream.eof(value)) {
          return null;
        }

        const token = this.token(value, matchers);
        token.start = start;
        token.end = {
          sourceLine: this.stream.lines[this.stream.line - 1],
          line: this.stream.line,
          col: this.stream.col,
        };
        // Comments are ignored for now
        if (token.type !== comments.type) {
          this.tokens.push(token);
        }

        return this.tokens[this.pos++];
      }

      match(char: string, parsers: Parsers) {
        return parsers.map(parse => parse(char)).filter(p => p);
      }

      /**
       * Match a particular non-whitespace value to a token
       */
      token(
        value: string,
        parsers: Parsers,
        token: { type: string, value: string, start: {}, end: {} } = {
          type: "unknown",
          value,
          start: {},
          end: {},
        }
      ) {
        // Strict parsers must end on a leaf node
        if (parsers.length > 1) {
          parsers = parsers.filter(parser => (parser.strict ? parser.leaf : true));
          if (parsers.length > 1) {
            parsers = parsers.filter(parser => parser.strict);
          }
        }

        token.type = parsers.pop().type;

        return token;
      }

      /**
       * Seek Stream until next non-whitespace character. Can end in eof/eol
       */
      seekNonWhitespace() {
        while (this.stream.peek() && Stream.whitespace(this.stream.peek())) {
          this.stream.next();
        }
      }

      parse() {
        while (!Stream.eof(this.stream.peek())) {
          this.next();
        }

        return this.tokens;
      }
    }

    export default Tokenizer;