scanner.h
Go to the documentation of this file.
1 #pragma once
7 typedef enum {
8  TOKEN_LEFT_PAREN, TOKEN_RIGHT_PAREN,
9  TOKEN_LEFT_BRACE, TOKEN_RIGHT_BRACE,
10  TOKEN_LEFT_SQUARE, TOKEN_RIGHT_SQUARE,
11  TOKEN_COLON,
12  TOKEN_COMMA,
13  TOKEN_DOT,
14  TOKEN_MINUS,
15  TOKEN_PLUS,
16  TOKEN_SEMICOLON,
17  TOKEN_SOLIDUS,
18  TOKEN_DOUBLE_SOLIDUS,
19  TOKEN_ASTERISK,
20  TOKEN_POW,
21  TOKEN_MODULO,
22  TOKEN_AT,
23  TOKEN_CARET, /* ^ (xor) */
24  TOKEN_AMPERSAND, /* & (and) */
25  TOKEN_PIPE, /* | (or) */
26  TOKEN_TILDE, /* ~ (negate) */
27  TOKEN_LEFT_SHIFT, /* << */
28  TOKEN_RIGHT_SHIFT,/* >> */
29  TOKEN_BANG,
30  TOKEN_GREATER,
31  TOKEN_LESS,
32  TOKEN_ARROW, /* -> */
33  TOKEN_WALRUS, /* := */
34 
35  /* Comparisons */
36  TOKEN_GREATER_EQUAL,
37  TOKEN_LESS_EQUAL,
38  TOKEN_BANG_EQUAL,
39  TOKEN_EQUAL_EQUAL,
40 
41  /* Assignments */
42  TOKEN_EQUAL,
43  TOKEN_LSHIFT_EQUAL, /* <<= */
44  TOKEN_RSHIFT_EQUAL, /* >>= */
45  TOKEN_PLUS_EQUAL, /* += */
46  TOKEN_MINUS_EQUAL, /* -= */
47  TOKEN_PLUS_PLUS, /* ++ */
48  TOKEN_MINUS_MINUS, /* -- */
49  TOKEN_CARET_EQUAL,
50  TOKEN_PIPE_EQUAL,
51  TOKEN_AMP_EQUAL,
52  TOKEN_SOLIDUS_EQUAL,
53  TOKEN_ASTERISK_EQUAL,
54  TOKEN_POW_EQUAL,
55  TOKEN_DSOLIDUS_EQUAL,
56  TOKEN_AT_EQUAL,
57  TOKEN_MODULO_EQUAL,
58 
59  TOKEN_STRING,
60  TOKEN_BIG_STRING,
61  TOKEN_NUMBER,
62 
63  /*
64  * Everything after this, up to indentation,
65  * consists of alphanumerics.
66  */
67  TOKEN_IDENTIFIER,
68  TOKEN_AND,
69  TOKEN_CLASS,
70  TOKEN_DEF,
71  TOKEN_DEL,
72  TOKEN_ELSE,
73  TOKEN_FALSE,
74  TOKEN_FINALLY,
75  TOKEN_FOR,
76  TOKEN_IF,
77  TOKEN_IMPORT,
78  TOKEN_IN,
79  TOKEN_IS,
80  TOKEN_LET,
81  TOKEN_NONE,
82  TOKEN_NOT,
83  TOKEN_OR,
84  TOKEN_ELIF,
85  TOKEN_PASS,
86  TOKEN_RETURN,
87  TOKEN_SUPER,
88  TOKEN_TRUE,
89  TOKEN_WHILE,
90  TOKEN_TRY,
91  TOKEN_EXCEPT,
92  TOKEN_RAISE,
93  TOKEN_BREAK,
94  TOKEN_CONTINUE,
95  TOKEN_AS,
96  TOKEN_FROM,
97  TOKEN_LAMBDA,
98  TOKEN_ASSERT,
99  TOKEN_YIELD,
100  TOKEN_ASYNC,
101  TOKEN_AWAIT,
102  TOKEN_WITH,
103 
104  TOKEN_PREFIX_B,
105  TOKEN_PREFIX_F,
106  TOKEN_PREFIX_R,
107 
108  TOKEN_INDENTATION,
109 
110  TOKEN_EOL,
111  TOKEN_RETRY,
112  TOKEN_ERROR,
113  TOKEN_EOF,
114 
115  TOKEN_ELLIPSIS, /* ... */
116 } KrkTokenType;
117 
124 typedef struct {
125  KrkTokenType type;
126  const char * start;
127  size_t length;
128  size_t line;
129  const char * linePtr;
130  size_t col;
131  size_t literalWidth;
132 } KrkToken;
133 
140 typedef struct {
141  const char * start;
142  const char * cur;
143  const char * linePtr;
144  size_t line;
145  int startOfLine;
146  int hasUnget;
147  KrkToken unget;
148 } KrkScanner;
149 
153 extern KrkScanner krk_initScanner(const char * src);
154 
158 extern KrkToken krk_scanToken(KrkScanner*);
159 
167 extern void krk_ungetToken(KrkScanner*, KrkToken token);
168 
176 extern void krk_rewindScanner(KrkScanner*, KrkScanner to);
177 
184 extern KrkScanner krk_tellScanner(KrkScanner*);
krk_ungetToken
void krk_ungetToken(KrkScanner *, KrkToken token)
Push a token back to the scanner to be reprocessed.
Definition: scanner.c:285
krk_scanToken
KrkToken krk_scanToken(KrkScanner *)
Read the next token from the scanner.
Definition: scanner.c:301
krk_initScanner
KrkScanner krk_initScanner(const char *src)
Initialize the compiler to scan tokens from 'src'.
Definition: scanner.c:11
krk_tellScanner
KrkScanner krk_tellScanner(KrkScanner *)
Retreive a copy of the current scanner state.
Definition: scanner.c:293
krk_rewindScanner
void krk_rewindScanner(KrkScanner *, KrkScanner to)
Rewind the scanner to a previous state.
Definition: scanner.c:297
KrkScanner
Token scanner state.
Definition: scanner.h:140
KrkToken
A token from the scanner.
Definition: scanner.h:124