compiler.c
Go to the documentation of this file.
1 
30 #include <assert.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <sys/types.h>
35 
36 #include <kuroko/kuroko.h>
37 #include <kuroko/compiler.h>
38 #include <kuroko/memory.h>
39 #include <kuroko/scanner.h>
40 #include <kuroko/object.h>
41 #include <kuroko/debug.h>
42 #include <kuroko/vm.h>
43 #include <kuroko/util.h>
44 
45 #include "private.h"
46 #include "opcode_enum.h"
47 
56 typedef struct {
59  char hadError;
60  unsigned int eatingWhitespace;
61 } Parser;
62 
69 typedef enum {
70  PREC_NONE,
90 } Precedence;
91 
97 typedef enum {
103  EXPR_CLASS_PARAMETERS,
105 
106 struct RewindState;
107 struct GlobalState;
108 
116 typedef void (*ParseFn)(struct GlobalState *, int, struct RewindState *);
117 
124 typedef struct {
128 } ParseRule;
129 
137 typedef struct {
139  ssize_t depth;
140  char isCaptured;
141 } Local;
142 
148 typedef struct {
149  size_t index;
150  char isLocal;
152 } Upvalue;
153 
160 typedef enum {
171 } FunctionType;
172 
181  size_t ind;
182  struct IndexWithNext * next;
183 };
184 
188 struct LoopExit {
189  int offset;
191 };
192 
199 typedef struct Compiler {
200  struct Compiler * enclosing;
203  size_t scopeDepth;
204  size_t localCount;
205  size_t localsSpace;
207  size_t upvaluesSpace;
210  size_t loopLocalCount;
211  size_t breakCount;
212  size_t breakSpace;
213  struct LoopExit * breaks;
214  size_t continueCount;
215  size_t continueSpace;
216  struct LoopExit * continues;
221  struct Compiler * enclosed;
226  size_t optionsFlags;
229 
230 #define OPTIONS_FLAG_COMPILE_TIME_BUILTINS (1 << 0)
231 #define OPTIONS_FLAG_NO_IMPLICIT_SELF (1 << 1)
232 
240 typedef struct ClassCompiler {
245 
258 typedef struct ChunkRecorder {
259  size_t count;
260  size_t lines;
261  size_t constants;
262  size_t expressions;
264 
270 typedef struct RewindState {
275 
276 typedef struct GlobalState {
282 } GlobalState;
283 
284 static void _GlobalState_gcscan(KrkInstance * _self) {
285  struct GlobalState * self = (void*)_self;
286  Compiler * compiler = self->current;
287  while (compiler != NULL) {
288  if (compiler->enclosed && compiler->enclosed->codeobject) krk_markObject((KrkObj*)compiler->enclosed->codeobject);
289  krk_markObject((KrkObj*)compiler->codeobject);
290  compiler = compiler->enclosing;
291  }
292 }
293 
294 static void _GlobalState_gcsweep(KrkInstance * _self) {
295  /* nothing to do? */
296 }
297 
298 #define currentChunk() (&state->current->codeobject->chunk)
299 
300 #define EMIT_OPERAND_OP(opc, arg) do { if (arg < 256) { emitBytes(opc, arg); } \
301  else { emitBytes(opc ## _LONG, arg >> 16); emitBytes(arg >> 8, arg); } } while (0)
302 
303 static int isMethod(int type) {
304  return type == TYPE_METHOD || type == TYPE_INIT || type == TYPE_COROUTINE_METHOD;
305 }
306 
307 static int isCoroutine(int type) {
308  return type == TYPE_COROUTINE || type == TYPE_COROUTINE_METHOD;
309 }
310 
311 static char * calculateQualName(struct GlobalState * state) {
312  static char space[1024]; /* We'll just truncate if we need to */
313  space[1023] = '\0';
314  char * writer = &space[1023];
315 
316 #define WRITE(s) do { \
317  size_t len = strlen(s); \
318  if (writer - len < space) goto _exit; \
319  writer -= len; \
320  memcpy(writer, s, len); \
321 } while (0)
322 
323  WRITE(state->current->codeobject->name->chars);
324  /* Go up by _compiler_, ignore class compilers as we don't need them. */
325  Compiler * ptr = state->current->enclosing;
326  while (ptr->enclosing) { /* Ignores the top level module */
327  if (ptr->type != TYPE_CLASS) {
328  /* We must be the locals of a function. */
329  WRITE("<locals>.");
330  }
331  WRITE(".");
332  WRITE(ptr->codeobject->name->chars);
333  ptr = ptr->enclosing;
334  }
335 
336 _exit:
337  return writer;
338 }
339 
340 #define codeobject_from_chunk_pointer(ptr) ((KrkCodeObject*)((char *)(ptr) - offsetof(KrkCodeObject, chunk)))
341 static ChunkRecorder recordChunk(KrkChunk * in) {
342  return (ChunkRecorder){in->count, in->linesCount, in->constants.count, codeobject_from_chunk_pointer(in)->expressionsCount};
343 }
344 
345 static void rewindChunk(KrkChunk * out, ChunkRecorder from) {
346  out->count = from.count;
347  out->linesCount = from.lines;
348  out->constants.count = from.constants;
349  codeobject_from_chunk_pointer(out)->expressionsCount = from.expressions;
350 }
351 #undef codeobject_from_chunk_pointer
352 
353 static size_t renameLocal(struct GlobalState * state, size_t ind, KrkToken name);
354 
355 static void initCompiler(struct GlobalState * state, Compiler * compiler, FunctionType type) {
356  compiler->enclosing = state->current;
357  state->current = compiler;
358  compiler->codeobject = NULL;
359  compiler->type = type;
360  compiler->scopeDepth = 0;
361  compiler->enclosed = NULL;
362  compiler->codeobject = krk_newCodeObject();
363  compiler->localCount = 0;
364  compiler->localsSpace = 8;
365  compiler->locals = KRK_GROW_ARRAY(Local,NULL,0,8);
366  compiler->upvaluesSpace = 0;
367  compiler->upvalues = NULL;
368  compiler->breakCount = 0;
369  compiler->breakSpace = 0;
370  compiler->breaks = NULL;
371  compiler->continueCount = 0;
372  compiler->continueSpace = 0;
373  compiler->continues = NULL;
374  compiler->loopLocalCount = 0;
375  compiler->localNameCapacity = 0;
376  compiler->properties = NULL;
377  compiler->annotationCount = 0;
378  compiler->delSatisfied = 0;
379  compiler->unnamedArgs = 0;
380  compiler->optionsFlags = compiler->enclosing ? compiler->enclosing->optionsFlags : 0;
381 
382  if (type != TYPE_MODULE) {
383  state->current->codeobject->name = krk_copyString(state->parser.previous.start, state->parser.previous.length);
384  char * qualname = calculateQualName(state);
385  state->current->codeobject->qualname = krk_copyString(qualname, strlen(qualname));
386  }
387 
388  if (isMethod(type) && !(compiler->optionsFlags & OPTIONS_FLAG_NO_IMPLICIT_SELF)) {
389  Local * local = &state->current->locals[state->current->localCount++];
390  local->depth = 0;
391  local->isCaptured = 0;
392  local->name.start = "self";
393  local->name.length = 4;
394  renameLocal(state, 0, local->name);
395  state->current->codeobject->requiredArgs = 1;
397  }
398 
399  if (type == TYPE_CLASS) {
400  Local * local = &state->current->locals[state->current->localCount++];
401  local->depth = 0;
402  local->isCaptured = 0;
403  local->name.start = "";
404  local->name.length = 0;
405  renameLocal(state, 0, local->name);
406  state->current->codeobject->requiredArgs = 1;
408  }
409 
410  if (isCoroutine(type)) state->current->codeobject->obj.flags |= KRK_OBJ_FLAGS_CODEOBJECT_IS_COROUTINE;
411 }
412 
413 static void rememberClassProperty(struct GlobalState * state, size_t ind) {
414  struct IndexWithNext * me = malloc(sizeof(struct IndexWithNext));
415  me->ind = ind;
416  me->next = state->current->properties;
417  state->current->properties = me;
418 }
419 
420 static void parsePrecedence(struct GlobalState * state, Precedence precedence);
421 static ParseRule * getRule(KrkTokenType type);
422 
423 /* These need to be forward declared or the ordering just gets really confusing... */
424 static void defDeclaration(struct GlobalState * state);
425 static void asyncDeclaration(struct GlobalState * state, int);
426 static void statement(struct GlobalState * state);
427 static void declaration(struct GlobalState * state);
428 static KrkToken classDeclaration(struct GlobalState * state);
429 static void declareVariable(struct GlobalState * state);
430 static void string(struct GlobalState * state, int exprType, RewindState *rewind);
431 static KrkToken decorator(struct GlobalState * state, size_t level, FunctionType type);
432 static void complexAssignment(struct GlobalState * state, ChunkRecorder before, KrkScanner oldScanner, Parser oldParser, size_t targetCount, int parenthesized, size_t argBefore, size_t argAfter);
433 static void complexAssignmentTargets(struct GlobalState * state, KrkScanner oldScanner, Parser oldParser, size_t targetCount, int parenthesized, size_t argBefore, size_t argAfter);
434 static int invalidTarget(struct GlobalState * state, int exprType, const char * description);
435 static void call(struct GlobalState * state, int exprType, RewindState *rewind);
436 
437 static void finishError(struct GlobalState * state, KrkToken * token) {
438  if (!token->linePtr) token->linePtr = token->start;
439  size_t i = 0;
440  while (token->linePtr[i] && token->linePtr[i] != '\n') i++;
441 
442  krk_attachNamedObject(&AS_INSTANCE(krk_currentThread.currentException)->fields, "line", (KrkObj*)krk_copyString(token->linePtr, i));
443  krk_attachNamedObject(&AS_INSTANCE(krk_currentThread.currentException)->fields, "file", (KrkObj*)currentChunk()->filename);
444  krk_attachNamedValue (&AS_INSTANCE(krk_currentThread.currentException)->fields, "lineno", INTEGER_VAL(token->line));
445  krk_attachNamedValue (&AS_INSTANCE(krk_currentThread.currentException)->fields, "colno", INTEGER_VAL(token->col));
446  krk_attachNamedValue (&AS_INSTANCE(krk_currentThread.currentException)->fields, "width", INTEGER_VAL(token->literalWidth));
447 
448  if (state->current->codeobject->name) {
449  krk_attachNamedObject(&AS_INSTANCE(krk_currentThread.currentException)->fields, "func", (KrkObj*)state->current->codeobject->name);
450  } else {
451  KrkValue name = NONE_VAL();
452  krk_tableGet(&krk_currentThread.module->fields, vm.specialMethodNames[METHOD_NAME], &name);
453  krk_attachNamedValue(&AS_INSTANCE(krk_currentThread.currentException)->fields, "func", name);
454  }
455 
456  state->parser.hadError = 1;
457 }
458 
459 #ifdef KRK_NO_DOCUMENTATION
460 # define raiseSyntaxError(token, ...) do { if (state->parser.hadError) break; krk_runtimeError(vm.exceptions->syntaxError, "syntax error"); finishError(state,token); } while (0)
461 #else
462 # define raiseSyntaxError(token, ...) do { if (state->parser.hadError) break; krk_runtimeError(vm.exceptions->syntaxError, __VA_ARGS__); finishError(state,token); } while (0)
463 #endif
464 
465 #define error(...) raiseSyntaxError(&state->parser.previous, __VA_ARGS__)
466 #define errorAtCurrent(...) raiseSyntaxError(&state->parser.current, __VA_ARGS__)
467 
468 static void _advance(struct GlobalState * state) {
469  state->parser.previous = state->parser.current;
470 
471  for (;;) {
472  state->parser.current = krk_scanToken(&state->scanner);
473 
474  if (state->parser.eatingWhitespace &&
475  (state->parser.current.type == TOKEN_INDENTATION || state->parser.current.type == TOKEN_EOL)) continue;
476 
477  if (state->parser.current.type == TOKEN_RETRY) continue;
478  if (state->parser.current.type != TOKEN_ERROR) break;
479 
480  errorAtCurrent("%s", state->parser.current.start);
481  break;
482  }
483 }
484 
485 #define advance() _advance(state)
486 
487 static void _skipToEnd(struct GlobalState * state) {
488  while (state->parser.current.type != TOKEN_EOF) advance();
489 }
490 
491 #define skipToEnd() _skipToEnd(state)
492 
493 static void _startEatingWhitespace(struct GlobalState * state) {
494  state->parser.eatingWhitespace++;
495  if (state->parser.current.type == TOKEN_INDENTATION || state->parser.current.type == TOKEN_EOL) advance();
496 }
497 
498 #define startEatingWhitespace() _startEatingWhitespace(state)
499 
500 static void _stopEatingWhitespace(struct GlobalState * state) {
501  if (state->parser.eatingWhitespace == 0) {
502  error("Internal scanner error: Invalid nesting of `startEatingWhitespace`/`stopEatingWhitespace` calls.");
503  }
504  state->parser.eatingWhitespace--;
505 }
506 
507 #define stopEatingWhitespace() _stopEatingWhitespace(state)
508 
509 static void _consume(struct GlobalState * state, KrkTokenType type, const char * message) {
510  if (state->parser.current.type == type) {
511  advance();
512  return;
513  }
514 
515  if (state->parser.current.type == TOKEN_EOL || state->parser.current.type == TOKEN_EOF) {
516  state->parser.current = state->parser.previous;
517  }
518  errorAtCurrent("%s", message);
519 }
520 
521 #define consume(...) _consume(state,__VA_ARGS__)
522 
523 static int _check(struct GlobalState * state, KrkTokenType type) {
524  return state->parser.current.type == type;
525 }
526 
527 #define check(t) _check(state,t)
528 
529 static int _match(struct GlobalState * state, KrkTokenType type) {
530  if (!check(type)) return 0;
531  advance();
532  return 1;
533 }
534 
535 #define match(t) _match(state,t)
536 
537 static int identifiersEqual(KrkToken * a, KrkToken * b) {
538  return (a->length == b->length && memcmp(a->start, b->start, a->length) == 0);
539 }
540 
541 static KrkToken _syntheticToken(struct GlobalState * state, const char * text) {
542  KrkToken token;
543  token.start = text;
544  token.length = (int)strlen(text);
545  token.line = state->parser.previous.line;
546  return token;
547 }
548 
549 #define syntheticToken(t) _syntheticToken(state,t)
550 
551 static void _emitByte(struct GlobalState * state, uint8_t byte) {
552  krk_writeChunk(currentChunk(), byte, state->parser.previous.line);
553 }
554 
555 #define emitByte(b) _emitByte(state,b)
556 
557 static void _emitBytes(struct GlobalState * state, uint8_t byte1, uint8_t byte2) {
558  emitByte(byte1);
559  emitByte(byte2);
560 }
561 
562 #define emitBytes(a,b) _emitBytes(state,a,b)
563 
564 static void emitReturn(struct GlobalState * state) {
565  if (state->current->type != TYPE_LAMBDA && state->current->type != TYPE_CLASS) {
566  emitByte(OP_NONE);
567  }
568  emitByte(OP_RETURN);
569 }
570 
571 static KrkCodeObject * endCompiler(struct GlobalState * state) {
572  KrkCodeObject * function = state->current->codeobject;
573 
574  for (size_t i = 0; i < function->localNameCount; i++) {
575  if (function->localNames[i].deathday == 0) {
576  function->localNames[i].deathday = currentChunk()->count;
577  }
578  }
579  function->localNames = KRK_GROW_ARRAY(KrkLocalEntry, function->localNames,
580  state->current->localNameCapacity, function->localNameCount); /* Shorten this down for runtime */
581 
582  if (state->current->continueCount) { state->parser.previous = state->current->continues[0].token; error("continue without loop"); }
583  if (state->current->breakCount) { state->parser.previous = state->current->breaks[0].token; error("break without loop"); }
584  emitReturn(state);
585 
586  /* Reduce the size of dynamic arrays to their fixed sizes. */
587  function->chunk.lines = KRK_GROW_ARRAY(KrkLineMap, function->chunk.lines,
588  function->chunk.linesCapacity, function->chunk.linesCount);
589  function->chunk.linesCapacity = function->chunk.linesCount;
590  function->chunk.code = KRK_GROW_ARRAY(uint8_t, function->chunk.code,
591  function->chunk.capacity, function->chunk.count);
592  function->chunk.capacity = function->chunk.count;
593 
594  function->expressions = KRK_GROW_ARRAY(KrkExpressionsMap, function->expressions,
595  function->expressionsCapacity, function->expressionsCount);
596  function->expressionsCapacity = function->expressionsCount;
597 
598  /* Attach contants for arguments */
599  for (int i = 0; i < function->potentialPositionals; ++i) {
600  if (i < state->current->unnamedArgs) {
601  krk_writeValueArray(&function->positionalArgNames, NONE_VAL());
602  continue;
603  }
604  KrkValue value = OBJECT_VAL(krk_copyString(state->current->locals[i].name.start, state->current->locals[i].name.length));
605  krk_push(value);
606  krk_writeValueArray(&function->positionalArgNames, value);
607  krk_pop();
608  }
609 
610  size_t args = function->potentialPositionals;
611  if (function->obj.flags & KRK_OBJ_FLAGS_CODEOBJECT_COLLECTS_ARGS) {
612  KrkValue value = OBJECT_VAL(krk_copyString(state->current->locals[args].name.start,
613  state->current->locals[args].name.length));
614  krk_push(value);
615  krk_writeValueArray(&function->positionalArgNames, value);
616  krk_pop();
617  args++;
618  }
619 
620  for (int i = 0; i < function->keywordArgs; ++i) {
621  KrkValue value = OBJECT_VAL(krk_copyString(state->current->locals[i+args].name.start,
622  state->current->locals[i+args].name.length));
623  krk_push(value);
624  krk_writeValueArray(&function->keywordArgNames, value);
625  krk_pop();
626  }
627  args += function->keywordArgs;
628 
629  if (function->obj.flags & KRK_OBJ_FLAGS_CODEOBJECT_COLLECTS_KWS) {
630  KrkValue value = OBJECT_VAL(krk_copyString(state->current->locals[args].name.start,
631  state->current->locals[args].name.length));
632  krk_push(value);
633  krk_writeValueArray(&function->keywordArgNames, value);
634  krk_pop();
635  args++;
636  }
637 
638  function->totalArguments = function->potentialPositionals + function->keywordArgs + !!(function->obj.flags & KRK_OBJ_FLAGS_CODEOBJECT_COLLECTS_ARGS) + !!(function->obj.flags & KRK_OBJ_FLAGS_CODEOBJECT_COLLECTS_KWS);
639 
640 #if !defined(KRK_NO_DISASSEMBLY) && !defined(KRK_DISABLE_DEBUG)
641  if ((krk_currentThread.flags & KRK_THREAD_ENABLE_DISASSEMBLY) && !state->parser.hadError) {
642  krk_disassembleCodeObject(stderr, function, function->name ? function->name->chars : "(module)");
643  }
644 #endif
645 
646  state->current = state->current->enclosing;
647  return function;
648 }
649 
650 static void freeCompiler(Compiler * compiler) {
651  KRK_FREE_ARRAY(Local,compiler->locals, compiler->localsSpace);
652  KRK_FREE_ARRAY(Upvalue,compiler->upvalues, compiler->upvaluesSpace);
653  KRK_FREE_ARRAY(struct LoopExit,compiler->breaks, compiler->breakSpace);
654  KRK_FREE_ARRAY(struct LoopExit,compiler->continues, compiler->continueSpace);
655 
656  while (compiler->properties) {
657  void * tmp = compiler->properties;
658  compiler->properties = compiler->properties->next;
659  free(tmp);
660  }
661 }
662 
663 static size_t _emitConstant(struct GlobalState * state, KrkValue value) {
664  return krk_writeConstant(currentChunk(), value, state->parser.previous.line);
665 }
666 
667 #define emitConstant(v) _emitConstant(state,v)
668 
669 static int isMangleable(const char * name, size_t len) {
670  return (len > 2 && name[0] == '_' && name[1] == '_' && name[len-1] != '_' && (len < 4 || name[len-2] != '_'));
671 }
672 
673 static ssize_t identifierConstant(struct GlobalState * state, KrkToken * name) {
674  if (state->currentClass && isMangleable(name->start, name->length)) {
675  /* Mangle it */
676  const char * className = state->currentClass->name.start;
677  size_t classLength = state->currentClass->name.length;
678  while (classLength && *className == '_') {
679  classLength--;
680  className++;
681  }
682 
683  struct StringBuilder sb = {0};
684  krk_pushStringBuilderFormat(&sb,"_%.*s%.*s",
685  (int)classLength, className,
686  (int)name->length, name->start);
687 
688  return krk_addConstant(currentChunk(), krk_finishStringBuilder(&sb));
689  }
690 
691  return krk_addConstant(currentChunk(), OBJECT_VAL(krk_copyString(name->start, name->length)));
692 }
693 
694 static ssize_t nonidentifierTokenConstant(struct GlobalState * state, KrkToken * name) {
695  return krk_addConstant(currentChunk(), OBJECT_VAL(krk_copyString(name->start, name->length)));
696 }
697 
698 static ssize_t resolveLocal(struct GlobalState * state, Compiler * compiler, KrkToken * name) {
699  for (ssize_t i = compiler->localCount - 1; i >= 0; i--) {
700  Local * local = &compiler->locals[i];
701  if (identifiersEqual(name, &local->name)) {
702  if (local->depth == -1) {
703  error("Invalid recursive reference in declaration initializer");
704  }
705  if (local->depth == -2) {
706  continue;
707  }
708  return i;
709  }
710  }
711  return -1;
712 }
713 
714 static size_t renameLocal(struct GlobalState * state, size_t ind, KrkToken name) {
715  if (state->current->codeobject->localNameCount + 1 > state->current->localNameCapacity) {
716  size_t old = state->current->localNameCapacity;
717  state->current->localNameCapacity = KRK_GROW_CAPACITY(old);
718  state->current->codeobject->localNames = KRK_GROW_ARRAY(KrkLocalEntry, state->current->codeobject->localNames, old, state->current->localNameCapacity);
719  }
721  state->current->codeobject->localNames[state->current->codeobject->localNameCount].birthday = currentChunk()->count;
723  state->current->codeobject->localNames[state->current->codeobject->localNameCount].name = krk_copyString(name.start, name.length);
724  return state->current->codeobject->localNameCount++;
725 }
726 
727 static size_t addLocal(struct GlobalState * state, KrkToken name) {
728  if (state->current->localCount + 1 > state->current->localsSpace) {
729  size_t old = state->current->localsSpace;
730  state->current->localsSpace = KRK_GROW_CAPACITY(old);
731  state->current->locals = KRK_GROW_ARRAY(Local,state->current->locals,old,state->current->localsSpace);
732  }
733  size_t out = state->current->localCount;
734  Local * local = &state->current->locals[state->current->localCount++];
735  local->name = name;
736  local->depth = -1;
737  local->isCaptured = 0;
738 
739  if (name.length) {
740  renameLocal(state, out, name);
741  }
742 
743  return out;
744 }
745 
746 static void declareVariable(struct GlobalState * state) {
747  if (state->current->scopeDepth == 0) return;
748  KrkToken * name = &state->parser.previous;
749  /* Detect duplicate definition */
750  for (ssize_t i = state->current->localCount - 1; i >= 0; i--) {
751  Local * local = &state->current->locals[i];
752  if (local->depth != -1 && local->depth < (ssize_t)state->current->scopeDepth) break;
753  if (identifiersEqual(name, &local->name)) {
754  error("Duplicate definition for local '%.*s' in this scope.", (int)name->literalWidth, name->start);
755  }
756  }
757  addLocal(state, *name);
758 }
759 
760 static ssize_t parseVariable(struct GlobalState * state, const char * errorMessage) {
761  consume(TOKEN_IDENTIFIER, errorMessage);
762 
763  declareVariable(state);
764  if (state->current->scopeDepth > 0) return 0;
765 
766  if ((state->current->optionsFlags & OPTIONS_FLAG_COMPILE_TIME_BUILTINS) && *state->parser.previous.start != '_') {
767  KrkValue value;
768  if (krk_tableGet_fast(&vm.builtins->fields, krk_copyString(state->parser.previous.start, state->parser.previous.length), &value)) {
769  error("Conflicting declaration of global '%.*s' is invalid when 'compile_time_builtins' is enabled.",
770  (int)state->parser.previous.length, state->parser.previous.start);
771  return 0;
772  }
773  }
774 
775  return identifierConstant(state, &state->parser.previous);
776 }
777 
778 static void markInitialized(struct GlobalState * state) {
779  if (state->current->scopeDepth == 0) return;
780  state->current->locals[state->current->localCount - 1].depth = state->current->scopeDepth;
781 }
782 
783 static size_t anonymousLocal(struct GlobalState * state) {
784  size_t val = addLocal(state, syntheticToken(""));
785  markInitialized(state);
786  return val;
787 }
788 
789 static void defineVariable(struct GlobalState * state, size_t global) {
790  if (state->current->scopeDepth > 0) {
791  markInitialized(state);
792  return;
793  }
794 
795  EMIT_OPERAND_OP(OP_DEFINE_GLOBAL, global);
796 }
797 
798 static void number(struct GlobalState * state, int exprType, RewindState *rewind) {
799  const char * start = state->parser.previous.start;
800  invalidTarget(state, exprType, "literal");
801 
802  for (size_t j = 0; j < state->parser.previous.length; ++j) {
803  if (start[j] == 'x' || start[j] == 'X') break;
804  if (start[j] == '.' || start[j] == 'e' || start[j] == 'E') {
805 #ifndef KRK_NO_FLOAT
806  emitConstant(krk_parse_float(start, state->parser.previous.length));
807 #else
808  error("no float support");
809 #endif
810  return;
811  }
812  }
813 
814  /* If we got here, it's an integer of some sort. */
815  KrkValue result = krk_parse_int(start, state->parser.previous.literalWidth, 0);
816  if (IS_NONE(result)) {
817  error("invalid numeric literal");
818  return;
819  }
820  emitConstant(result);
821 }
822 
823 static int _emitJump(struct GlobalState * state, uint8_t opcode) {
824  emitByte(opcode);
825  emitBytes(0xFF, 0xFF);
826  return currentChunk()->count - 2;
827 }
828 #define emitJump(o) _emitJump(state,o)
829 
841 static void _emitOverlongJump(struct GlobalState * state, int offset, int jump) {
842  KrkCodeObject * co = state->current->codeobject;
843  size_t i = 0;
844  while (i < co->overlongJumpsCount && co->overlongJumps[i].instructionOffset != (uint32_t)offset) i++;
845  if (i == co->overlongJumpsCount) {
846  /* Not an existing overlong jump, need to make a new one. */
847  if (co->overlongJumpsCount + 1 > co->overlongJumpsCapacity) {
848  size_t old = co->overlongJumpsCapacity;
849  co->overlongJumpsCapacity = KRK_GROW_CAPACITY(old);
850  co->overlongJumps = KRK_GROW_ARRAY(KrkOverlongJump,co->overlongJumps,old,co->overlongJumpsCapacity);
851  }
852  co->overlongJumps[i].instructionOffset = offset;
853  co->overlongJumps[i].originalOpcode = currentChunk()->code[offset-1];
854  co->overlongJumpsCount++;
855  currentChunk()->code[offset-1] = OP_OVERLONG_JUMP;
856  }
857  /* Update jump target */
858  co->overlongJumps[i].intendedTarget = jump >> 16;
859 }
860 
861 static void _patchJump(struct GlobalState * state, int offset) {
862  int jump = currentChunk()->count - offset - 2;
863 
864  if (jump > 0xFFFF) {
865  _emitOverlongJump(state, offset, jump);
866  }
867 
868  currentChunk()->code[offset] = (jump >> 8) & 0xFF;
869  currentChunk()->code[offset + 1] = (jump) & 0xFF;
870 }
871 
872 #define patchJump(o) _patchJump(state,o)
873 
891 static void writeExpressionLocation(KrkToken * before, KrkToken * after, KrkToken * current, struct GlobalState * state) {
892 #ifndef KRK_DISABLE_DEBUG
893  /* We can only support the display of underlines when the whole expression is on one line. */
894  if (after->linePtr != before->linePtr) return;
895  if (after->line != state->parser.previous.line) return;
896 
897  /* For the sake of runtime storage, we only store this debug information
898  * when it fits within the first 256 characters of a line, so we can
899  * use uint8_ts to store these. */
900  if (before->col > 255 || current->col > 255 ||
901  (current->col + current->literalWidth) > 255 ||
902  (after->col + after->literalWidth) > 255) return;
903 
905  before->col, current->col,
906  current->col + current->literalWidth,
907  after->col + after->literalWidth);
908 #endif
909 }
910 
911 static void compareChained(struct GlobalState * state, int inner, KrkToken * preceding) {
912  KrkTokenType operatorType = state->parser.previous.type;
913  if (operatorType == TOKEN_NOT) consume(TOKEN_IN, "'in' must follow infix 'not'");
914  int invert = (operatorType == TOKEN_IS && match(TOKEN_NOT));
915  KrkToken this = state->parser.previous;
916  KrkToken next = state->parser.current;
917 
918  ParseRule * rule = getRule(operatorType);
919  parsePrecedence(state, (Precedence)(rule->precedence + 1));
920 
921  if (getRule(state->parser.current.type)->precedence == PREC_COMPARISON) {
922  emitByte(OP_SWAP);
923  emitBytes(OP_DUP, 1);
924  }
925 
926  switch (operatorType) {
927  case TOKEN_BANG_EQUAL: emitByte(OP_EQUAL); invert = 1; break;
928  case TOKEN_EQUAL_EQUAL: emitByte(OP_EQUAL); break;
929  case TOKEN_GREATER: emitByte(OP_GREATER); break;
930  case TOKEN_GREATER_EQUAL: emitByte(OP_GREATER_EQUAL); break;
931  case TOKEN_LESS: emitByte(OP_LESS); break;
932  case TOKEN_LESS_EQUAL: emitByte(OP_LESS_EQUAL); break;
933 
934  case TOKEN_IS: emitByte(OP_IS); break;
935 
936  case TOKEN_IN: emitByte(OP_INVOKE_CONTAINS); break;
937  case TOKEN_NOT: emitByte(OP_INVOKE_CONTAINS); invert = 1; break;
938 
939  default: error("Invalid binary comparison operator?"); break;
940  }
941 
942  writeExpressionLocation(preceding, &state->parser.previous, &this, state);
943  if (invert) emitByte(OP_NOT);
944 
945  if (getRule(state->parser.current.type)->precedence == PREC_COMPARISON) {
946  size_t exitJump = emitJump(OP_JUMP_IF_FALSE_OR_POP);
947  advance();
948  compareChained(state, 1, &next);
949  patchJump(exitJump);
950  if (getRule(state->parser.current.type)->precedence != PREC_COMPARISON) {
951  if (!inner) {
952  emitBytes(OP_SWAP,OP_POP);
953  }
954  }
955  } else if (inner) {
956  emitByte(OP_JUMP);
957  emitBytes(0,2);
958  }
959 }
960 
961 static void compare(struct GlobalState * state, int exprType, RewindState *rewind) {
962  compareChained(state, 0, &rewind->oldParser.current);
963  invalidTarget(state, exprType, "operator");
964 }
965 
966 static void binary(struct GlobalState * state, int exprType, RewindState *rewind) {
967  KrkTokenType operatorType = state->parser.previous.type;
968  ParseRule * rule = getRule(operatorType);
969  KrkToken this = state->parser.previous;
970  parsePrecedence(state, (Precedence)(rule->precedence + (rule->precedence != PREC_EXPONENT)));
971  invalidTarget(state, exprType, "operator");
972 
973  switch (operatorType) {
974  case TOKEN_PIPE: emitByte(OP_BITOR); break;
975  case TOKEN_CARET: emitByte(OP_BITXOR); break;
976  case TOKEN_AMPERSAND: emitByte(OP_BITAND); break;
977  case TOKEN_LEFT_SHIFT: emitByte(OP_SHIFTLEFT); break;
978  case TOKEN_RIGHT_SHIFT: emitByte(OP_SHIFTRIGHT); break;
979 
980  case TOKEN_PLUS: emitByte(OP_ADD); break;
981  case TOKEN_MINUS: emitByte(OP_SUBTRACT); break;
982  case TOKEN_ASTERISK: emitByte(OP_MULTIPLY); break;
983  case TOKEN_POW: emitByte(OP_POW); break;
984  case TOKEN_SOLIDUS: emitByte(OP_DIVIDE); break;
985  case TOKEN_DOUBLE_SOLIDUS: emitByte(OP_FLOORDIV); break;
986  case TOKEN_MODULO: emitByte(OP_MODULO); break;
987  case TOKEN_IN: emitByte(OP_EQUAL); break;
988  case TOKEN_AT: emitByte(OP_MATMUL); break;
989  default: return;
990  }
991  writeExpressionLocation(&rewind->oldParser.current, &state->parser.previous, &this, state);
992 }
993 
994 static int matchAssignment(struct GlobalState * state) {
995  return (state->parser.current.type >= TOKEN_EQUAL && state->parser.current.type <= TOKEN_MODULO_EQUAL) ? (advance(), 1) : 0;
996 }
997 
998 static int checkEndOfDel(struct GlobalState * state) {
999  if (check(TOKEN_COMMA) || check(TOKEN_EOL) || check(TOKEN_EOF) || check(TOKEN_SEMICOLON)) {
1000  state->current->delSatisfied = 1;
1001  return 1;
1002  }
1003  return 0;
1004 }
1005 
1006 static int matchComplexEnd(struct GlobalState * state) {
1007  return match(TOKEN_COMMA) ||
1008  match(TOKEN_EQUAL) ||
1009  match(TOKEN_RIGHT_PAREN);
1010 }
1011 
1012 static int invalidTarget(struct GlobalState * state, int exprType, const char * description) {
1013  if (exprType == EXPR_CAN_ASSIGN && matchAssignment(state)) {
1014  error("Can not assign to %s", description);
1015  return 0;
1016  }
1017 
1018  if (exprType == EXPR_DEL_TARGET && checkEndOfDel(state)) {
1019  error("Can not delete %s", description);
1020  return 0;
1021  }
1022 
1023  return 1;
1024 }
1025 
1026 static void assignmentValue(struct GlobalState * state) {
1027  KrkTokenType type = state->parser.previous.type;
1028  KrkToken left = state->parser.previous;
1029  if (type == TOKEN_PLUS_PLUS || type == TOKEN_MINUS_MINUS) {
1030  emitConstant(INTEGER_VAL(1));
1031  } else {
1032  parsePrecedence(state, PREC_COMMA); /* But adding a tuple is maybe not defined */
1033  }
1034 
1035  switch (type) {
1036  case TOKEN_PIPE_EQUAL: emitByte(OP_INPLACE_BITOR); break;
1037  case TOKEN_CARET_EQUAL: emitByte(OP_INPLACE_BITXOR); break;
1038  case TOKEN_AMP_EQUAL: emitByte(OP_INPLACE_BITAND); break;
1039  case TOKEN_LSHIFT_EQUAL: emitByte(OP_INPLACE_SHIFTLEFT); break;
1040  case TOKEN_RSHIFT_EQUAL: emitByte(OP_INPLACE_SHIFTRIGHT); break;
1041 
1042  case TOKEN_PLUS_EQUAL: emitByte(OP_INPLACE_ADD); break;
1043  case TOKEN_PLUS_PLUS: emitByte(OP_INPLACE_ADD); break;
1044  case TOKEN_MINUS_EQUAL: emitByte(OP_INPLACE_SUBTRACT); break;
1045  case TOKEN_MINUS_MINUS: emitByte(OP_INPLACE_SUBTRACT); break;
1046  case TOKEN_ASTERISK_EQUAL: emitByte(OP_INPLACE_MULTIPLY); break;
1047  case TOKEN_POW_EQUAL: emitByte(OP_INPLACE_POW); break;
1048  case TOKEN_SOLIDUS_EQUAL: emitByte(OP_INPLACE_DIVIDE); break;
1049  case TOKEN_DSOLIDUS_EQUAL: emitByte(OP_INPLACE_FLOORDIV); break;
1050  case TOKEN_MODULO_EQUAL: emitByte(OP_INPLACE_MODULO); break;
1051  case TOKEN_AT_EQUAL: emitByte(OP_INPLACE_MATMUL); break;
1052 
1053  default:
1054  error("Unexpected operand in assignment");
1055  break;
1056  }
1057  writeExpressionLocation(&left,&state->parser.previous,&left,state);
1058 }
1059 
1060 static void expression(struct GlobalState * state) {
1061  parsePrecedence(state, PREC_CAN_ASSIGN);
1062 }
1063 
1064 static void sliceExpression(struct GlobalState * state) {
1065  int isSlice = 0;
1066  if (match(TOKEN_COLON)) {
1067  emitByte(OP_NONE);
1068  isSlice = 1;
1069  } else {
1070  parsePrecedence(state, PREC_CAN_ASSIGN);
1071  }
1072  if (isSlice || match(TOKEN_COLON)) {
1073  /* We have the start value, which is either something or None */
1074  if (check(TOKEN_RIGHT_SQUARE) || check(TOKEN_COMMA)) {
1075  /* foo[x:] */
1076  emitByte(OP_NONE);
1077  EMIT_OPERAND_OP(OP_SLICE, 2);
1078  } else {
1079  if (check(TOKEN_COLON)) {
1080  /* foo[x::... */
1081  emitByte(OP_NONE);
1082  } else {
1083  /* foo[x:e... */
1084  parsePrecedence(state, PREC_CAN_ASSIGN);
1085  }
1086  if (match(TOKEN_COLON) && !check(TOKEN_RIGHT_SQUARE) && !check(TOKEN_COMMA)) {
1087  /* foo[x:e:s] */
1088  parsePrecedence(state, PREC_CAN_ASSIGN);
1089  EMIT_OPERAND_OP(OP_SLICE, 3);
1090  } else {
1091  /* foo[x:e] */
1092  EMIT_OPERAND_OP(OP_SLICE, 2);
1093  }
1094  }
1095  }
1096 }
1097 
1098 static void getitem(struct GlobalState * state, int exprType, RewindState *rewind) {
1099 
1100  KrkToken *left = &rewind->oldParser.current;
1101  KrkToken this = state->parser.previous;
1102 
1103  sliceExpression(state);
1104 
1105  if (match(TOKEN_COMMA)) {
1106  size_t argCount = 1;
1107  if (!check(TOKEN_RIGHT_SQUARE)) {
1108  do {
1109  sliceExpression(state);
1110  argCount++;
1111  } while (match(TOKEN_COMMA) && !check(TOKEN_RIGHT_SQUARE));
1112  }
1113  EMIT_OPERAND_OP(OP_TUPLE, argCount);
1114  }
1115 
1116  consume(TOKEN_RIGHT_SQUARE, "Expected ']' after index.");
1117  if (exprType == EXPR_ASSIGN_TARGET) {
1118  if (matchComplexEnd(state)) {
1119  EMIT_OPERAND_OP(OP_DUP, 2);
1120  emitByte(OP_INVOKE_SETTER);
1121  writeExpressionLocation(left,&state->parser.current,&this,state);
1122  emitByte(OP_POP);
1123  return;
1124  }
1125  exprType = EXPR_NORMAL;
1126  }
1127  if (exprType == EXPR_CAN_ASSIGN && match(TOKEN_EQUAL)) {
1128  parsePrecedence(state, PREC_ASSIGNMENT);
1129  emitByte(OP_INVOKE_SETTER);
1130  writeExpressionLocation(left,&state->parser.previous,&this,state);
1131  } else if (exprType == EXPR_CAN_ASSIGN && matchAssignment(state)) {
1132  emitBytes(OP_DUP, 1); /* o e o */
1133  emitBytes(OP_DUP, 1); /* o e o e */
1134  emitByte(OP_INVOKE_GETTER); /* o e v */
1135  writeExpressionLocation(left,&state->parser.previous,&this,state);
1136  assignmentValue(state); /* o e v a */
1137  emitByte(OP_INVOKE_SETTER); /* r */
1138  writeExpressionLocation(left,&state->parser.previous,&this,state);
1139  } else if (exprType == EXPR_DEL_TARGET && checkEndOfDel(state)) {
1140  emitByte(OP_INVOKE_DELETE);
1141  writeExpressionLocation(left,&state->parser.previous,&this,state);
1142  } else {
1143  emitByte(OP_INVOKE_GETTER);
1144  writeExpressionLocation(left,&state->parser.previous,&this,state);
1145  }
1146 }
1147 
1148 static void attributeUnpack(struct GlobalState * state, int exprType) {
1149  startEatingWhitespace();
1150  size_t argCount = 0;
1151  size_t argSpace = 1;
1152  ssize_t * args = KRK_GROW_ARRAY(ssize_t,NULL,0,1);
1153 
1154  do {
1155  if (argSpace < argCount + 1) {
1156  size_t old = argSpace;
1157  argSpace = KRK_GROW_CAPACITY(old);
1158  args = KRK_GROW_ARRAY(ssize_t,args,old,argSpace);
1159  }
1160  consume(TOKEN_IDENTIFIER, "Expected attribute name");
1161  size_t ind = identifierConstant(state, &state->parser.previous);
1162  args[argCount++] = ind;
1163  } while (match(TOKEN_COMMA));
1164 
1165  stopEatingWhitespace();
1166  consume(TOKEN_RIGHT_PAREN, "Expected ')' after attribute list");
1167 
1168  if (exprType == EXPR_ASSIGN_TARGET) {
1169  error("Can not assign to '.(' in multiple target list");
1170  goto _dotDone;
1171  }
1172 
1173  if (exprType == EXPR_CAN_ASSIGN && match(TOKEN_EQUAL)) {
1174  size_t expressionCount = 0;
1175  do {
1176  expressionCount++;
1177  expression(state);
1178  } while (match(TOKEN_COMMA));
1179 
1180  if (expressionCount == 1 && argCount > 1) {
1181  EMIT_OPERAND_OP(OP_UNPACK, argCount);
1182  } else if (expressionCount > 1 && argCount == 1) {
1183  EMIT_OPERAND_OP(OP_TUPLE, expressionCount);
1184  } else if (expressionCount != argCount) {
1185  error("Invalid assignment to attribute pack");
1186  goto _dotDone;
1187  }
1188 
1189  for (size_t i = argCount; i > 0; i--) {
1190  if (i != 1) {
1191  emitBytes(OP_DUP, i);
1192  emitByte(OP_SWAP);
1193  }
1194  EMIT_OPERAND_OP(OP_SET_PROPERTY, args[i-1]);
1195  if (i != 1) {
1196  emitByte(OP_POP);
1197  }
1198  }
1199  } else {
1200  for (size_t i = 0; i < argCount; i++) {
1201  emitBytes(OP_DUP,0);
1202  EMIT_OPERAND_OP(OP_GET_PROPERTY,args[i]);
1203  emitByte(OP_SWAP);
1204  }
1205  emitByte(OP_POP);
1206  emitBytes(OP_TUPLE,argCount);
1207  }
1208 
1209 _dotDone:
1210  KRK_FREE_ARRAY(ssize_t,args,argSpace);
1211  return;
1212 }
1213 
1214 static void dot(struct GlobalState * state, int exprType, RewindState *rewind) {
1215  if (match(TOKEN_LEFT_PAREN)) {
1216  attributeUnpack(state, exprType);
1217  return;
1218  }
1219  KrkToken name = state->parser.current;
1220  KrkToken this = state->parser.previous;
1221  consume(TOKEN_IDENTIFIER, "Expected property name");
1222  size_t ind = identifierConstant(state, &state->parser.previous);
1223  if (exprType == EXPR_ASSIGN_TARGET) {
1224  if (matchComplexEnd(state)) {
1225  EMIT_OPERAND_OP(OP_DUP, 1);
1226  EMIT_OPERAND_OP(OP_SET_PROPERTY, ind);
1227  writeExpressionLocation(&rewind->oldParser.current, &state->parser.previous, &name, state);
1228  emitByte(OP_POP);
1229  return;
1230  }
1231  exprType = EXPR_NORMAL;
1232  }
1233  if (exprType == EXPR_CAN_ASSIGN && match(TOKEN_EQUAL)) {
1234  parsePrecedence(state, PREC_ASSIGNMENT);
1235  EMIT_OPERAND_OP(OP_SET_PROPERTY, ind);
1236  writeExpressionLocation(&rewind->oldParser.current, &state->parser.previous, &name, state);
1237  } else if (exprType == EXPR_CAN_ASSIGN && matchAssignment(state)) {
1238  emitBytes(OP_DUP, 0); /* Duplicate the object */
1239  EMIT_OPERAND_OP(OP_GET_PROPERTY, ind);
1240  writeExpressionLocation(&rewind->oldParser.current, &name, &this, state);
1241  assignmentValue(state);
1242  EMIT_OPERAND_OP(OP_SET_PROPERTY, ind);
1243  writeExpressionLocation(&rewind->oldParser.current, &state->parser.previous, &name, state);
1244  } else if (exprType == EXPR_DEL_TARGET && checkEndOfDel(state)) {
1245  EMIT_OPERAND_OP(OP_DEL_PROPERTY, ind);
1246  writeExpressionLocation(&rewind->oldParser.current, &name, &this, state);
1247  } else if (match(TOKEN_LEFT_PAREN)) {
1248  EMIT_OPERAND_OP(OP_GET_METHOD, ind);
1249  writeExpressionLocation(&rewind->oldParser.current, &name, &this, state);
1250  call(state, EXPR_METHOD_CALL, rewind);
1251  } else {
1252  EMIT_OPERAND_OP(OP_GET_PROPERTY, ind);
1253  writeExpressionLocation(&rewind->oldParser.current, &name, &this, state);
1254  }
1255 }
1256 
1257 static void literal(struct GlobalState * state, int exprType, RewindState *rewind) {
1258  invalidTarget(state, exprType, "literal");
1259  switch (state->parser.previous.type) {
1260  case TOKEN_FALSE: emitByte(OP_FALSE); break;
1261  case TOKEN_NONE: emitByte(OP_NONE); break;
1262  case TOKEN_TRUE: emitByte(OP_TRUE); break;
1263  default: return;
1264  }
1265 }
1266 
1267 static void ellipsis(struct GlobalState * state, int exprType, RewindState *rewind) {
1268  invalidTarget(state, exprType, "literal");
1269  KrkValue value;
1270  if (!krk_tableGet_fast(&vm.builtins->fields, S("Ellipsis"), &value)) {
1271  error("internal compiler error");
1272  return;
1273  }
1274  emitConstant(value);
1275 }
1276 
1277 static void typeHintLocal(struct GlobalState * state) {
1278  state->current->enclosing->enclosed = state->current;
1279  state->current = state->current->enclosing;
1280  state->current->enclosed->annotationCount++;
1281  emitConstant(INTEGER_VAL(state->current->enclosed->codeobject->localNameCount-1));
1282  parsePrecedence(state, PREC_TERNARY);
1283  state->current = state->current->enclosed;
1284  state->current->enclosing->enclosed = NULL;
1285 }
1286 
1287 static void letDeclaration(struct GlobalState * state) {
1288  size_t argCount = 0;
1289  size_t argSpace = 1;
1290  ssize_t * args = KRK_GROW_ARRAY(ssize_t,NULL,0,1);
1291 
1292  do {
1293  if (argSpace < argCount + 1) {
1294  size_t old = argSpace;
1295  argSpace = KRK_GROW_CAPACITY(old);
1296  args = KRK_GROW_ARRAY(ssize_t,args,old,argSpace);
1297  }
1298  ssize_t ind = parseVariable(state, "Expected variable name.");
1299  if (state->parser.hadError) goto _letDone;
1300  if (state->current->scopeDepth > 0) {
1301  /* Need locals space */
1302  args[argCount++] = state->current->localCount - 1;
1303  } else {
1304  args[argCount++] = ind;
1305  }
1306  if (check(TOKEN_COLON)) {
1307  KrkToken name = state->parser.previous;
1308  match(TOKEN_COLON);
1309  if (state->current->enclosing) {
1310  typeHintLocal(state);
1311  } else {
1312  KrkToken annotations = syntheticToken("__annotations__");
1313  size_t ind = identifierConstant(state, &annotations);
1314  EMIT_OPERAND_OP(OP_GET_GLOBAL, ind);
1315  emitConstant(OBJECT_VAL(krk_copyString(name.start, name.length)));
1316  parsePrecedence(state, PREC_TERNARY);
1317  emitBytes(OP_INVOKE_SETTER, OP_POP);
1318  }
1319  }
1320  } while (match(TOKEN_COMMA));
1321 
1322  if (match(TOKEN_EQUAL)) {
1323  size_t expressionCount = 0;
1324  do {
1325  expressionCount++;
1326  expression(state);
1327  } while (match(TOKEN_COMMA));
1328  if (expressionCount == 1 && argCount > 1) {
1329  EMIT_OPERAND_OP(OP_UNPACK, argCount);
1330  } else if (expressionCount == argCount) {
1331  /* Do nothing */
1332  } else if (expressionCount > 1 && argCount == 1) {
1333  EMIT_OPERAND_OP(OP_TUPLE, expressionCount);
1334  } else {
1335  error("Invalid sequence unpack in 'let' statement");
1336  goto _letDone;
1337  }
1338  } else {
1339  /* Need to nil it */
1340  for (size_t i = 0; i < argCount; ++i) {
1341  emitByte(OP_NONE);
1342  }
1343  }
1344 
1345  if (state->current->scopeDepth == 0) {
1346  for (size_t i = argCount; i > 0; i--) {
1347  defineVariable(state, args[i-1]);
1348  }
1349  } else {
1350  for (size_t i = 0; i < argCount; i++) {
1351  state->current->locals[state->current->localCount - 1 - i].depth = state->current->scopeDepth;
1352  }
1353  }
1354 
1355 _letDone:
1356  KRK_FREE_ARRAY(ssize_t,args,argSpace);
1357  return;
1358 }
1359 
1360 static void declaration(struct GlobalState * state) {
1361  if (check(TOKEN_DEF)) {
1362  defDeclaration(state);
1363  } else if (check(TOKEN_CLASS)) {
1364  KrkToken className = classDeclaration(state);
1365  size_t classConst = identifierConstant(state, &className);
1366  state->parser.previous = className;
1367  declareVariable(state);
1368  defineVariable(state, classConst);
1369  } else if (check(TOKEN_AT)) {
1370  decorator(state, 0, TYPE_FUNCTION);
1371  } else if (check(TOKEN_ASYNC)) {
1372  asyncDeclaration(state, 1);
1373  } else if (match(TOKEN_EOL) || match(TOKEN_EOF)) {
1374  return;
1375  } else if (check(TOKEN_INDENTATION)) {
1376  return;
1377  } else {
1378  statement(state);
1379  }
1380 
1381  if (state->parser.hadError) skipToEnd();
1382 }
1383 
1384 static void expressionStatement(struct GlobalState * state) {
1385  parsePrecedence(state, PREC_ASSIGNMENT);
1386  emitByte(OP_POP);
1387 }
1388 
1389 static void beginScope(struct GlobalState * state) {
1390  state->current->scopeDepth++;
1391 }
1392 
1393 static void endScope(struct GlobalState * state) {
1394  state->current->scopeDepth--;
1395 
1396  int closeCount = 0;
1397  int popCount = 0;
1398 
1399  while (state->current->localCount > 0 &&
1400  state->current->locals[state->current->localCount - 1].depth > (ssize_t)state->current->scopeDepth) {
1401  if (state->current->locals[state->current->localCount - 1].isCaptured) {
1402  if (popCount) {
1403  if (popCount == 1) emitByte(OP_POP);
1404  else { EMIT_OPERAND_OP(OP_POP_MANY, popCount); }
1405  popCount = 0;
1406  }
1407  closeCount++;
1408  } else {
1409  if (closeCount) {
1410  if (closeCount == 1) emitByte(OP_CLOSE_UPVALUE);
1411  else { EMIT_OPERAND_OP(OP_CLOSE_MANY, closeCount); }
1412  closeCount = 0;
1413  }
1414  popCount++;
1415  }
1416 
1417  for (size_t i = 0; i < state->current->codeobject->localNameCount; i++) {
1418  if (state->current->codeobject->localNames[i].id == state->current->localCount - 1 &&
1419  state->current->codeobject->localNames[i].deathday == 0) {
1420  state->current->codeobject->localNames[i].deathday = (size_t)currentChunk()->count;
1421  }
1422  }
1423  state->current->localCount--;
1424  }
1425 
1426  if (popCount) {
1427  if (popCount == 1) emitByte(OP_POP);
1428  else { EMIT_OPERAND_OP(OP_POP_MANY, popCount); }
1429  }
1430  if (closeCount) {
1431  if (closeCount == 1) emitByte(OP_CLOSE_UPVALUE);
1432  else { EMIT_OPERAND_OP(OP_CLOSE_MANY, closeCount); }
1433  }
1434 }
1435 
1436 static void block(struct GlobalState * state, size_t indentation, const char * blockName) {
1437  if (match(TOKEN_EOL)) {
1438  if (check(TOKEN_INDENTATION)) {
1439  size_t currentIndentation = state->parser.current.length;
1440  if (currentIndentation <= indentation) return;
1441  advance();
1442  if (!strcmp(blockName,"def") && (match(TOKEN_STRING) || match(TOKEN_BIG_STRING))) {
1443  size_t before = currentChunk()->count;
1444  string(state, EXPR_NORMAL, NULL);
1445  /* That wrote to the chunk, rewind it; this should only ever go back two bytes
1446  * because this should only happen as the first thing in a function definition,
1447  * and thus this _should_ be the first constant and thus opcode + one-byte operand
1448  * to OP_CONSTANT, but just to be safe we'll actually use the previous offset... */
1449  currentChunk()->count = before;
1450  /* Retreive the docstring from the constant table */
1451  state->current->codeobject->docstring = AS_STRING(currentChunk()->constants.values[currentChunk()->constants.count-1]);
1452  consume(TOKEN_EOL,"Garbage after docstring defintion");
1453  if (!check(TOKEN_INDENTATION) || state->parser.current.length != currentIndentation) {
1454  error("Expected at least one statement in function with docstring.");
1455  }
1456  advance();
1457  }
1458  declaration(state);
1459  while (check(TOKEN_INDENTATION)) {
1460  if (state->parser.current.length < currentIndentation) break;
1461  advance();
1462  declaration(state);
1463  if (check(TOKEN_EOL)) {
1464  advance();
1465  }
1466  if (state->parser.hadError) skipToEnd();
1467  };
1468  }
1469  } else {
1470  statement(state);
1471  }
1472 }
1473 
1474 static void doUpvalues(struct GlobalState * state, Compiler * compiler, KrkCodeObject * function) {
1475  assert(!!function->upvalueCount == !!compiler->upvalues);
1476  for (size_t i = 0; i < function->upvalueCount; ++i) {
1477  size_t index = compiler->upvalues[i].index;
1478  emitByte((compiler->upvalues[i].isLocal) | ((index > 255) ? 2 : 0));
1479  if (index > 255) {
1480  emitByte((index >> 16) & 0xFF);
1481  emitByte((index >> 8) & 0xFF);
1482  }
1483  emitByte(index & 0xFF);
1484  }
1485 }
1486 
1487 static void typeHint(struct GlobalState * state, KrkToken name) {
1488  state->current->enclosing->enclosed = state->current;
1489  state->current = state->current->enclosing;
1490 
1491  state->current->enclosed->annotationCount++;
1492 
1493  /* Emit name */
1494  emitConstant(OBJECT_VAL(krk_copyString(name.start, name.length)));
1495  parsePrecedence(state, PREC_TERNARY);
1496 
1497  state->current = state->current->enclosed;
1498  state->current->enclosing->enclosed = NULL;
1499 }
1500 
1501 static void hideLocal(struct GlobalState * state) {
1502  state->current->locals[state->current->localCount - 1].depth = -2;
1503 }
1504 
1505 static void argumentDefinition(struct GlobalState * state, int hasCollectors) {
1506  if (match(TOKEN_EQUAL)) {
1507  /*
1508  * We inline default arguments by checking if they are equal
1509  * to a sentinel value and replacing them with the requested
1510  * argument. This allows us to send None (useful) to override
1511  * defaults that are something else. This essentially ends
1512  * up as the following at the top of the function:
1513  * if param == KWARGS_SENTINEL:
1514  * param = EXPRESSION
1515  */
1516  size_t myLocal = state->current->localCount - 1;
1517  EMIT_OPERAND_OP(OP_GET_LOCAL, myLocal);
1518  int jumpIndex = emitJump(OP_TEST_ARG);
1519  beginScope(state);
1520  expression(state); /* Read expression */
1521  EMIT_OPERAND_OP(OP_SET_LOCAL_POP, myLocal);
1522  endScope(state);
1523  patchJump(jumpIndex);
1524  if (hasCollectors) {
1525  state->current->codeobject->keywordArgs++;
1526  } else {
1528  }
1529  } else {
1530  if (hasCollectors) {
1531  size_t myLocal = state->current->localCount - 1;
1532  EMIT_OPERAND_OP(OP_GET_LOCAL, myLocal);
1533  int jumpIndex = emitJump(OP_TEST_ARG);
1534  EMIT_OPERAND_OP(OP_MISSING_KW, state->current->codeobject->keywordArgs);
1535  patchJump(jumpIndex);
1536  state->current->codeobject->keywordArgs++;
1537  } else {
1539  error("non-default argument follows default argument");
1540  return;
1541  }
1542  state->current->codeobject->requiredArgs++;
1544  }
1545  }
1546 }
1547 
1548 static void functionPrologue(struct GlobalState * state, Compiler * compiler) {
1549  KrkCodeObject * func = endCompiler(state);
1550  if (compiler->annotationCount) {
1551  EMIT_OPERAND_OP(OP_MAKE_DICT, compiler->annotationCount * 2);
1552  }
1553  size_t ind = krk_addConstant(currentChunk(), OBJECT_VAL(func));
1554  EMIT_OPERAND_OP(OP_CLOSURE, ind);
1555  doUpvalues(state, compiler, func);
1556  if (compiler->annotationCount) {
1557  emitByte(OP_ANNOTATE);
1558  }
1559  freeCompiler(compiler);
1560 }
1561 
1562 static int argumentList(struct GlobalState * state, FunctionType type) {
1563  int hasCollectors = 0;
1564  KrkToken self = syntheticToken("self");
1565 
1566  do {
1567  if (!(state->current->optionsFlags & OPTIONS_FLAG_NO_IMPLICIT_SELF) &&
1568  isMethod(type) && check(TOKEN_IDENTIFIER) &&
1569  identifiersEqual(&state->parser.current, &self)) {
1570  if (hasCollectors || state->current->codeobject->requiredArgs != 1) {
1571  errorAtCurrent("Argument name 'self' in a method signature is reserved for the implicit first argument.");
1572  return 1;
1573  }
1574  advance();
1575  if (type != TYPE_LAMBDA && check(TOKEN_COLON)) {
1576  KrkToken name = state->parser.previous;
1577  match(TOKEN_COLON);
1578  typeHint(state, name);
1579  }
1580  if (check(TOKEN_EQUAL)) {
1581  errorAtCurrent("'self' can not be a default argument.");
1582  return 1;
1583  }
1584  continue;
1585  }
1586  if (match(TOKEN_SOLIDUS)) {
1587  if (hasCollectors || state->current->unnamedArgs || !state->current->codeobject->potentialPositionals) {
1588  error("Syntax error.");
1589  return 1;
1590  }
1592  continue;
1593  }
1594  if (match(TOKEN_ASTERISK) || check(TOKEN_POW)) {
1595  if (match(TOKEN_POW)) {
1596  if (hasCollectors == 2) {
1597  error("Duplicate ** in parameter list.");
1598  return 1;
1599  }
1600  hasCollectors = 2;
1601  state->current->codeobject->obj.flags |= KRK_OBJ_FLAGS_CODEOBJECT_COLLECTS_KWS;
1602  } else {
1603  if (hasCollectors) {
1604  error("Syntax error.");
1605  return 1;
1606  }
1607  hasCollectors = 1;
1608  if (check(TOKEN_COMMA)) {
1609  continue;
1610  }
1611  state->current->codeobject->obj.flags |= KRK_OBJ_FLAGS_CODEOBJECT_COLLECTS_ARGS;
1612  }
1613  /* Collect a name, specifically "args" or "kwargs" are commont */
1614  ssize_t paramConstant = parseVariable(state,
1615  (hasCollectors == 1) ? "Expected parameter name after '*'." : "Expected parameter name after '**'.");
1616  if (state->parser.hadError) return 1;
1617  defineVariable(state, paramConstant);
1618  KrkToken name = state->parser.previous;
1619  if (!(state->current->optionsFlags & OPTIONS_FLAG_NO_IMPLICIT_SELF) && isMethod(type) && identifiersEqual(&name,&self)) {
1620  errorAtCurrent("Argument name 'self' in a method signature is reserved for the implicit first argument.");
1621  return 1;
1622  }
1623  if (type != TYPE_LAMBDA && check(TOKEN_COLON)) {
1624  match(TOKEN_COLON);
1625  typeHint(state, name);
1626  }
1627  /* Make that a valid local for this function */
1628  size_t myLocal = state->current->localCount - 1;
1629  EMIT_OPERAND_OP(OP_GET_LOCAL, myLocal);
1630  /* Check if it's equal to the unset-kwarg-sentinel value */
1631  int jumpIndex = emitJump(OP_TEST_ARG);
1632  /* And if it is, set it to the appropriate type */
1633  beginScope(state);
1634  if (hasCollectors == 1) EMIT_OPERAND_OP(OP_MAKE_LIST,0);
1635  else EMIT_OPERAND_OP(OP_MAKE_DICT,0);
1636  EMIT_OPERAND_OP(OP_SET_LOCAL_POP, myLocal);
1637  endScope(state);
1638  /* Otherwise pop the comparison. */
1639  patchJump(jumpIndex);
1640  continue;
1641  }
1642  if (hasCollectors == 2) {
1643  error("arguments follow catch-all keyword collector");
1644  break;
1645  }
1646  ssize_t paramConstant = parseVariable(state, "Expected parameter name.");
1647  if (state->parser.hadError) return 1;
1648  hideLocal(state);
1649  if (type != TYPE_LAMBDA && check(TOKEN_COLON)) {
1650  KrkToken name = state->parser.previous;
1651  match(TOKEN_COLON);
1652  typeHint(state, name);
1653  }
1654  argumentDefinition(state, hasCollectors);
1655  defineVariable(state, paramConstant);
1656  } while (match(TOKEN_COMMA));
1657 
1658  return 0;
1659 }
1660 
1661 static void function(struct GlobalState * state, FunctionType type, size_t blockWidth) {
1662  Compiler compiler;
1663  initCompiler(state, &compiler, type);
1664  compiler.codeobject->chunk.filename = compiler.enclosing->codeobject->chunk.filename;
1665 
1666  beginScope(state);
1667 
1668  consume(TOKEN_LEFT_PAREN, "Expected start of parameter list after function name.");
1669  startEatingWhitespace();
1670  if (!check(TOKEN_RIGHT_PAREN)) {
1671  if (argumentList(state, type)) goto _bail;
1672  }
1673  stopEatingWhitespace();
1674  consume(TOKEN_RIGHT_PAREN, "Expected end of parameter list.");
1675 
1676  if (match(TOKEN_ARROW)) {
1677  typeHint(state, syntheticToken("return"));
1678  }
1679 
1680  consume(TOKEN_COLON, "Expected colon after function signature.");
1681  block(state, blockWidth,"def");
1682 _bail: (void)0;
1683  functionPrologue(state, &compiler);
1684 }
1685 
1686 static void classBody(struct GlobalState * state, size_t blockWidth) {
1687  if (match(TOKEN_EOL)) {
1688  return;
1689  }
1690 
1691  if (check(TOKEN_AT)) {
1692  /* '@decorator' which should be attached to a method. */
1693  decorator(state, 0, TYPE_METHOD);
1694  } else if (match(TOKEN_IDENTIFIER)) {
1695  /* Class field */
1696  size_t ind = identifierConstant(state, &state->parser.previous);
1697 
1698  if (check(TOKEN_COLON)) {
1699  /* Type annotation for field */
1700  KrkToken name = state->parser.previous;
1701  match(TOKEN_COLON);
1702  /* Get __annotations__ from class */
1703  KrkToken annotations = syntheticToken("__annotations__");
1704  size_t ind = identifierConstant(state, &annotations);
1705  if (!state->currentClass->hasAnnotations) {
1706  EMIT_OPERAND_OP(OP_MAKE_DICT, 0);
1707  EMIT_OPERAND_OP(OP_SET_NAME, ind);
1708  state->currentClass->hasAnnotations = 1;
1709  } else {
1710  EMIT_OPERAND_OP(OP_GET_NAME, ind);
1711  }
1712  emitConstant(OBJECT_VAL(krk_copyString(name.start, name.length)));
1713  parsePrecedence(state, PREC_TERNARY);
1714  emitBytes(OP_INVOKE_SETTER, OP_POP);
1715 
1716  /* A class field with a type hint can be valueless */
1717  if (match(TOKEN_EOL) || match(TOKEN_EOF)) return;
1718  }
1719 
1720  consume(TOKEN_EQUAL, "Class field must have value.");
1721 
1722  /* Value */
1723  parsePrecedence(state, PREC_COMMA);
1724 
1725  rememberClassProperty(state, ind);
1726  EMIT_OPERAND_OP(OP_SET_NAME, ind);
1727  emitByte(OP_POP);
1728 
1729  if (!match(TOKEN_EOL) && !match(TOKEN_EOF)) {
1730  errorAtCurrent("Expected end of line after class attribute declaration");
1731  }
1732  } else if (match(TOKEN_PASS)) {
1733  /* `pass` is just a general empty statement */
1734  consume(TOKEN_EOL, "Expected end of line after 'pass' in class body.");
1735  } else {
1736  /* Must be a function of some sort */
1737  FunctionType type = TYPE_METHOD;
1738  if (match(TOKEN_ASYNC)) {
1739  type = TYPE_COROUTINE_METHOD;
1740  consume(TOKEN_DEF, "Expected 'def' after 'async'");
1741  } else if (!match(TOKEN_DEF)) {
1742  error("Expected method, decorator, or class variable.");
1743  }
1744  consume(TOKEN_IDENTIFIER, "Expected method name after 'def'");
1745  size_t ind = identifierConstant(state, &state->parser.previous);
1746 
1747  static struct CompilerSpecialMethod { const char * name; int type; } compilerSpecialMethods[] = {
1748  {"__init__", TYPE_INIT},
1749  {"__class_getitem__", TYPE_CLASSMETHOD},
1750  {"__init_subclass__", TYPE_CLASSMETHOD},
1751  {"__new__", TYPE_STATIC},
1752  {NULL,0}
1753  };
1754 
1755  for (struct CompilerSpecialMethod * method = compilerSpecialMethods; method->name; method++) {
1756  if (state->parser.previous.length == strlen(method->name) && memcmp(state->parser.previous.start, method->name, strlen(method->name)) == 0) {
1757  if (type == TYPE_COROUTINE_METHOD) {
1758  error("'%s' can not be a coroutine",method->name);
1759  return;
1760  }
1761  type = method->type;
1762  }
1763  }
1764 
1765  function(state, type, blockWidth);
1766  rememberClassProperty(state, ind);
1767  EMIT_OPERAND_OP(OP_SET_NAME, ind);
1768  emitByte(OP_POP);
1769  }
1770 }
1771 
1772 #define ATTACH_PROPERTY(propName,how,propValue) do { \
1773  KrkToken val_tok = syntheticToken(propValue); \
1774  size_t val_ind = nonidentifierTokenConstant(state, &val_tok); \
1775  EMIT_OPERAND_OP(how, val_ind); \
1776  KrkToken name_tok = syntheticToken(propName); \
1777  size_t name_ind = identifierConstant(state, &name_tok); \
1778  EMIT_OPERAND_OP(OP_SET_NAME, name_ind); \
1779  emitByte(OP_POP); \
1780 } while (0)
1781 
1782 static size_t addUpvalue(struct GlobalState * state, Compiler * compiler, ssize_t index, int isLocal, KrkToken name);
1783 static KrkToken classDeclaration(struct GlobalState * state) {
1784  size_t blockWidth = (state->parser.previous.type == TOKEN_INDENTATION) ? state->parser.previous.length : 0;
1785  advance(); /* Collect the `class` */
1786 
1787  consume(TOKEN_IDENTIFIER, "Expected class name after 'class'.");
1788 
1789  emitByte(OP_PUSH_BUILD_CLASS);
1790 
1791  Compiler subcompiler;
1792  initCompiler(state, &subcompiler, TYPE_CLASS);
1793  subcompiler.codeobject->chunk.filename = subcompiler.enclosing->codeobject->chunk.filename;
1794 
1795  beginScope(state);
1796 
1797  KrkToken classNameToken = state->parser.previous;
1798  assert(addUpvalue(state, state->current, 0, 4, classNameToken) == 0);
1799 
1800  ClassCompiler classCompiler;
1801  classCompiler.name = state->parser.previous;
1802  classCompiler.enclosing = state->currentClass;
1803  state->currentClass = &classCompiler;
1804  classCompiler.hasAnnotations = 0;
1805 
1806  RewindState parameters = {recordChunk(currentChunk()), krk_tellScanner(&state->scanner), state->parser};
1807 
1808  /* Class parameters */
1809  if (match(TOKEN_LEFT_PAREN)) {
1810  int parenDepth = 0;
1811  while (!check(TOKEN_EOF)) {
1812  if (check(TOKEN_RIGHT_PAREN) && parenDepth == 0) {
1813  advance();
1814  break;
1815  } else if (match(TOKEN_LEFT_BRACE)) {
1816  parenDepth++;
1817  } else if (match(TOKEN_RIGHT_BRACE)) {
1818  parenDepth--;
1819  } else {
1820  advance();
1821  }
1822  }
1823  }
1824 
1825  beginScope(state);
1826 
1827  consume(TOKEN_COLON, "Expected ':' after class.");
1828 
1829  /* Set Class.__module__ to the value of __name__, which is the string
1830  * name of the current module. */
1831  ATTACH_PROPERTY("__module__", OP_GET_GLOBAL, "__name__");
1832  ATTACH_PROPERTY("__qualname__", OP_CONSTANT, calculateQualName(state));
1833 
1834  if (match(TOKEN_EOL)) {
1835  if (check(TOKEN_INDENTATION)) {
1836  size_t currentIndentation = state->parser.current.length;
1837  if (currentIndentation <= blockWidth) {
1838  errorAtCurrent("Unexpected indentation level for class");
1839  }
1840  advance();
1841  if (match(TOKEN_STRING) || match(TOKEN_BIG_STRING)) {
1842  string(state, EXPR_NORMAL, NULL);
1843  KrkToken doc = syntheticToken("__doc__");
1844  size_t ind = identifierConstant(state, &doc);
1845  EMIT_OPERAND_OP(OP_SET_NAME, ind);
1846  emitByte(OP_POP);
1847  consume(TOKEN_EOL,"Garbage after docstring defintion");
1848  if (!check(TOKEN_INDENTATION) || state->parser.current.length != currentIndentation) {
1849  goto _pop_class;
1850  }
1851  advance();
1852  }
1853  classBody(state, currentIndentation);
1854  while (check(TOKEN_INDENTATION)) {
1855  if (state->parser.current.length < currentIndentation) break;
1856  advance(); /* Pass the indentation */
1857  classBody(state, currentIndentation);
1858  }
1859  /* Exit from block */
1860  }
1861  } /* else empty class (and at end of file?) we'll allow it for now... */
1862 _pop_class:
1863  state->currentClass = state->currentClass->enclosing;
1864  KrkCodeObject * makeclass = endCompiler(state);
1865  size_t indFunc = krk_addConstant(currentChunk(), OBJECT_VAL(makeclass));
1866 
1867  RewindState afterFunction = {recordChunk(currentChunk()), krk_tellScanner(&state->scanner), state->parser};
1868 
1869  size_t nameInd = nonidentifierTokenConstant(state, &classNameToken);
1870 
1871  krk_rewindScanner(&state->scanner, parameters.oldScanner);
1872  state->parser = parameters.oldParser;
1873 
1874  EMIT_OPERAND_OP(OP_CLOSURE, indFunc);
1875  doUpvalues(state, &subcompiler, makeclass);
1876  freeCompiler(&subcompiler);
1877  EMIT_OPERAND_OP(OP_CONSTANT, nameInd);
1878 
1879  if (match(TOKEN_LEFT_PAREN)) {
1880  call(state, EXPR_CLASS_PARAMETERS, NULL);
1881  } else {
1882  emitBytes(OP_CALL, 2);
1883  }
1884 
1885  krk_rewindScanner(&state->scanner, afterFunction.oldScanner);
1886  state->parser = afterFunction.oldParser;
1887 
1888  return classCompiler.name;
1889 }
1890 
1891 static void lambda(struct GlobalState * state, int exprType, RewindState *rewind) {
1892  Compiler lambdaCompiler;
1893  state->parser.previous = syntheticToken("<lambda>");
1894  initCompiler(state, &lambdaCompiler, TYPE_LAMBDA);
1895  lambdaCompiler.codeobject->chunk.filename = lambdaCompiler.enclosing->codeobject->chunk.filename;
1896  beginScope(state);
1897 
1898  if (!check(TOKEN_COLON)) {
1899  if (argumentList(state, TYPE_LAMBDA)) goto _bail;
1900  }
1901 
1902  consume(TOKEN_COLON, "Expected ':' after lambda arguments");
1903  expression(state);
1904 
1905 _bail:
1906  functionPrologue(state, &lambdaCompiler);
1907 
1908  invalidTarget(state, exprType, "lambda");
1909 }
1910 
1911 static void defDeclaration(struct GlobalState * state) {
1912  size_t blockWidth = (state->parser.previous.type == TOKEN_INDENTATION) ? state->parser.previous.length : 0;
1913  advance(); /* Collect the `def` */
1914 
1915  ssize_t global = parseVariable(state, "Expected function name after 'def'.");
1916  if (state->parser.hadError) return;
1917  markInitialized(state);
1918  function(state, TYPE_FUNCTION, blockWidth);
1919  if (state->parser.hadError) return;
1920  defineVariable(state, global);
1921 }
1922 
1923 static void asyncDeclaration(struct GlobalState * state, int declarationLevel) {
1924  size_t blockWidth = (state->parser.previous.type == TOKEN_INDENTATION) ? state->parser.previous.length : 0;
1925  advance(); /* 'async' */
1926 
1927  if (match(TOKEN_DEF)) {
1928  if (!declarationLevel) {
1929  error("'async def' not valid here");
1930  return;
1931  }
1932  ssize_t global = parseVariable(state, "Expected coroutine name after 'async def'");
1933  if (state->parser.hadError) return;
1934  markInitialized(state);
1935  function(state, TYPE_COROUTINE, blockWidth);
1936  if (state->parser.hadError) return;
1937  defineVariable(state, global);
1938  } else if (match(TOKEN_FOR)) {
1939  if (!isCoroutine(state->current->type)) {
1940  error("'async for' outside of async function");
1941  return;
1942  }
1943  error("'async for' unsupported (GH-12)");
1944  return;
1945  } else if (match(TOKEN_WITH)) {
1946  if (!isCoroutine(state->current->type)) {
1947  error("'async with' outside of async function");
1948  return;
1949  }
1950  error("'async with' unsupported (GH-12)");
1951  return;
1952  } else {
1953  errorAtCurrent("Expected 'def' after 'async'.");
1954  return;
1955  }
1956 }
1957 
1958 static KrkToken decorator(struct GlobalState * state, size_t level, FunctionType type) {
1959  int inType = type;
1960  size_t blockWidth = (state->parser.previous.type == TOKEN_INDENTATION) ? state->parser.previous.length : 0;
1961  advance(); /* Collect the `@` */
1962 
1963  KrkToken funcName = {0};
1964 
1965  KrkToken at_staticmethod = syntheticToken("staticmethod");
1966  KrkToken at_classmethod = syntheticToken("classmethod");
1967 
1968  if (type == TYPE_METHOD) {
1969  if (identifiersEqual(&at_staticmethod, &state->parser.current)) type = TYPE_STATIC;
1970  if (identifiersEqual(&at_classmethod, &state->parser.current)) type = TYPE_CLASSMETHOD;
1971  }
1972 
1973  if (level == 0 && inType == TYPE_FUNCTION && state->current->scopeDepth != 0) {
1974  emitByte(OP_NONE);
1975  }
1976 
1977  expression(state);
1978 
1979  consume(TOKEN_EOL, "Expected end of line after decorator.");
1980  if (blockWidth) {
1981  consume(TOKEN_INDENTATION, "Expected next line after decorator to have same indentation.");
1982  if (state->parser.previous.length != blockWidth) error("Expected next line after decorator to have same indentation.");
1983  }
1984 
1985  if (check(TOKEN_DEF)) {
1986  /* We already checked for block level */
1987  advance();
1988  consume(TOKEN_IDENTIFIER, "Expected function name after 'def'");
1989  funcName = state->parser.previous;
1990  if (type == TYPE_METHOD && funcName.length == 8 && !memcmp(funcName.start,"__init__",8)) {
1991  type = TYPE_INIT;
1992  }
1993  if (type == TYPE_FUNCTION && state->current->scopeDepth > 0) {
1994  declareVariable(state);
1995  markInitialized(state);
1996  }
1997  function(state, type, blockWidth);
1998  } else if (match(TOKEN_ASYNC)) {
1999  if (!match(TOKEN_DEF)) {
2000  errorAtCurrent("Expected 'def' after 'async' with decorator, not '%*.s'",
2001  (int)state->parser.current.length, state->parser.current.start);
2002  }
2003  consume(TOKEN_IDENTIFIER, "Expected coroutine name after 'def'.");
2004  funcName = state->parser.previous;
2005  if (type == TYPE_FUNCTION && state->current->scopeDepth > 0) {
2006  declareVariable(state);
2007  markInitialized(state);
2008  }
2009  function(state, type == TYPE_METHOD ? TYPE_COROUTINE_METHOD : TYPE_COROUTINE, blockWidth);
2010  } else if (check(TOKEN_AT)) {
2011  funcName = decorator(state, level+1, type);
2012  } else if (check(TOKEN_CLASS)) {
2013  if (type != TYPE_FUNCTION) {
2014  error("Invalid decorator applied to class");
2015  return funcName;
2016  }
2017  funcName = classDeclaration(state);
2018  } else {
2019  error("Expected a function declaration or another decorator.");
2020  return funcName;
2021  }
2022 
2023  emitBytes(OP_CALL, 1);
2024 
2025  if (level == 0) {
2026  if (inType == TYPE_FUNCTION) {
2027  if (state->current->scopeDepth == 0) {
2028  size_t ind = identifierConstant(state, &funcName);
2029  defineVariable(state, ind);
2030  } else {
2031  emitByte(OP_SWAP_POP);
2032  }
2033  } else {
2034  size_t ind = identifierConstant(state, &funcName);
2035  rememberClassProperty(state, ind);
2036  EMIT_OPERAND_OP(OP_SET_NAME, ind);
2037  emitByte(OP_POP);
2038  }
2039  }
2040 
2041  return funcName;
2042 }
2043 
2044 static void emitLoop(struct GlobalState * state, int loopStart, uint8_t loopType) {
2045 
2046  /* Patch continue statements to point to here, before the loop operation (yes that's silly) */
2047  while (state->current->continueCount > 0 && state->current->continues[state->current->continueCount-1].offset > loopStart) {
2048  patchJump(state->current->continues[state->current->continueCount-1].offset);
2049  state->current->continueCount--;
2050  }
2051 
2052  emitByte(loopType);
2053 
2054  int offset = currentChunk()->count - loopStart + ((loopType == OP_LOOP_ITER) ? -1 : 2);
2055  if (offset > 0xFFFF) {
2056  _emitOverlongJump(state, currentChunk()->count, offset);
2057  }
2058  emitBytes(offset >> 8, offset);
2059 
2060  /* Patch break statements */
2061 }
2062 
2063 static void withStatement(struct GlobalState * state) {
2064  /* We only need this for block() */
2065  size_t blockWidth = (state->parser.previous.type == TOKEN_INDENTATION) ? state->parser.previous.length : 0;
2066  KrkToken myPrevious = state->parser.previous;
2067 
2068  /* Collect the with token that started this statement */
2069  advance();
2070 
2071  beginScope(state);
2072  expression(state);
2073 
2074  if (match(TOKEN_AS)) {
2075  consume(TOKEN_IDENTIFIER, "Expected variable name after 'as'");
2076  size_t ind = identifierConstant(state, &state->parser.previous);
2077  declareVariable(state);
2078  defineVariable(state, ind);
2079  } else {
2080  /* Otherwise we want an unnamed local */
2081  anonymousLocal(state);
2082  }
2083 
2084  /* Storage for return / exception */
2085  anonymousLocal(state);
2086 
2087  /* Handler object */
2088  anonymousLocal(state);
2089  int withJump = emitJump(OP_PUSH_WITH);
2090 
2091  if (check(TOKEN_COMMA)) {
2092  state->parser.previous = myPrevious;
2093  withStatement(state); /* Keep nesting */
2094  } else {
2095  consume(TOKEN_COLON, "Expected ',' or ':' after 'with' statement");
2096 
2097  beginScope(state);
2098  block(state,blockWidth,"with");
2099  endScope(state);
2100  }
2101 
2102  patchJump(withJump);
2103  emitByte(OP_CLEANUP_WITH);
2104 
2105  /* Scope exit pops context manager */
2106  endScope(state);
2107 }
2108 
2109 static void ifStatement(struct GlobalState * state) {
2110  /* Figure out what block level contains us so we can match our partner else */
2111  size_t blockWidth = (state->parser.previous.type == TOKEN_INDENTATION) ? state->parser.previous.length : 0;
2112  KrkToken myPrevious = state->parser.previous;
2113 
2114  /* Collect the if token that started this statement */
2115  advance();
2116 
2117  /* Collect condition expression */
2118  expression(state);
2119 
2120  /* if EXPR: */
2121  consume(TOKEN_COLON, "Expected ':' after 'if' condition.");
2122 
2123  if (state->parser.hadError) return;
2124 
2125  int thenJump = emitJump(OP_POP_JUMP_IF_FALSE);
2126 
2127  /* Start a new scope and enter a block */
2128  beginScope(state);
2129  block(state,blockWidth,"if");
2130  endScope(state);
2131 
2132  if (state->parser.hadError) return;
2133 
2134  /* See if we have a matching else block */
2135  if (blockWidth == 0 || (check(TOKEN_INDENTATION) && (state->parser.current.length == blockWidth))) {
2136  /* This is complicated */
2137  KrkToken previous;
2138  if (blockWidth) {
2139  previous = state->parser.previous;
2140  advance();
2141  }
2142  if (match(TOKEN_ELSE) || check(TOKEN_ELIF)) {
2143  int elseJump = emitJump(OP_JUMP);
2144  patchJump(thenJump);
2145  if (state->parser.current.type == TOKEN_ELIF || check(TOKEN_IF)) {
2146  state->parser.previous = myPrevious;
2147  ifStatement(state); /* Keep nesting */
2148  } else {
2149  consume(TOKEN_COLON, "Expected ':' after 'else'.");
2150  beginScope(state);
2151  block(state,blockWidth,"else");
2152  endScope(state);
2153  }
2154  patchJump(elseJump);
2155  return;
2156  } else if (!check(TOKEN_EOF) && !check(TOKEN_EOL)) {
2157  if (blockWidth) {
2158  krk_ungetToken(&state->scanner, state->parser.current);
2159  state->parser.current = state->parser.previous;
2160  state->parser.previous = previous;
2161  }
2162  } else {
2163  advance(); /* Ignore this blank indentation line */
2164  }
2165  }
2166 
2167  patchJump(thenJump);
2168 }
2169 
2170 static void patchBreaks(struct GlobalState * state, int loopStart) {
2171  /* Patch break statements to go here, after the loop operation and operand. */
2172  while (state->current->breakCount > 0 && state->current->breaks[state->current->breakCount-1].offset > loopStart) {
2173  patchJump(state->current->breaks[state->current->breakCount-1].offset);
2174  state->current->breakCount--;
2175  }
2176 }
2177 
2178 static void breakStatement(struct GlobalState * state) {
2179  if (state->current->breakSpace < state->current->breakCount + 1) {
2180  size_t old = state->current->breakSpace;
2181  state->current->breakSpace = KRK_GROW_CAPACITY(old);
2182  state->current->breaks = KRK_GROW_ARRAY(struct LoopExit,state->current->breaks,old,state->current->breakSpace);
2183  }
2184 
2185  if (state->current->loopLocalCount != state->current->localCount) {
2186  EMIT_OPERAND_OP(OP_EXIT_LOOP, state->current->loopLocalCount);
2187  }
2188 
2189  state->current->breaks[state->current->breakCount++] = (struct LoopExit){emitJump(OP_JUMP),state->parser.previous};
2190 }
2191 
2192 static void continueStatement(struct GlobalState * state) {
2193  if (state->current->continueSpace < state->current->continueCount + 1) {
2194  size_t old = state->current->continueSpace;
2195  state->current->continueSpace = KRK_GROW_CAPACITY(old);
2196  state->current->continues = KRK_GROW_ARRAY(struct LoopExit,state->current->continues,old,state->current->continueSpace);
2197  }
2198 
2199  if (state->current->loopLocalCount != state->current->localCount) {
2200  EMIT_OPERAND_OP(OP_EXIT_LOOP, state->current->loopLocalCount);
2201  }
2202 
2203  state->current->continues[state->current->continueCount++] = (struct LoopExit){emitJump(OP_JUMP),state->parser.previous};
2204 }
2205 
2206 static void optionalElse(struct GlobalState * state, size_t blockWidth) {
2207  KrkScanner scannerBefore = krk_tellScanner(&state->scanner);
2208  Parser parserBefore = state->parser;
2209  if (blockWidth == 0 || (check(TOKEN_INDENTATION) && (state->parser.current.length == blockWidth))) {
2210  if (blockWidth) advance();
2211  if (match(TOKEN_ELSE)) {
2212  consume(TOKEN_COLON, "Expected ':' after 'else'.");
2213  beginScope(state);
2214  block(state,blockWidth,"else");
2215  endScope(state);
2216  } else {
2217  krk_rewindScanner(&state->scanner, scannerBefore);
2218  state->parser = parserBefore;
2219  }
2220  }
2221 }
2222 
2223 static void whileStatement(struct GlobalState * state) {
2224  size_t blockWidth = (state->parser.previous.type == TOKEN_INDENTATION) ? state->parser.previous.length : 0;
2225  advance();
2226 
2227  int loopStart = currentChunk()->count;
2228  int exitJump = 0;
2229 
2230  /* Identify two common infinite loops and optimize them (True and 1) */
2231  RewindState rewind = {recordChunk(currentChunk()), krk_tellScanner(&state->scanner), state->parser};
2232  if (!(match(TOKEN_TRUE) && match(TOKEN_COLON)) &&
2233  !(match(TOKEN_NUMBER) && (state->parser.previous.length == 1 && *state->parser.previous.start == '1') && match(TOKEN_COLON))) {
2234  /* We did not match a common infinite loop, roll back... */
2235  krk_rewindScanner(&state->scanner, rewind.oldScanner);
2236  state->parser = rewind.oldParser;
2237 
2238  /* Otherwise, compile a real loop condition. */
2239  expression(state);
2240  consume(TOKEN_COLON, "Expected ':' after 'while' condition.");
2241 
2242  exitJump = emitJump(OP_JUMP_IF_FALSE_OR_POP);
2243  }
2244 
2245  int oldLocalCount = state->current->loopLocalCount;
2246  state->current->loopLocalCount = state->current->localCount;
2247  beginScope(state);
2248  block(state,blockWidth,"while");
2249  endScope(state);
2250 
2251  state->current->loopLocalCount = oldLocalCount;
2252  emitLoop(state, loopStart, OP_LOOP);
2253 
2254  if (exitJump) {
2255  patchJump(exitJump);
2256  emitByte(OP_POP);
2257  }
2258 
2259  /* else: block must still be compiled even if we optimized
2260  * out the loop condition check... */
2261  optionalElse(state, blockWidth);
2262 
2263  patchBreaks(state, loopStart);
2264 }
2265 
2266 static void forStatement(struct GlobalState * state) {
2267  /* I'm not sure if I want this to be more like Python or C/Lox/etc. */
2268  size_t blockWidth = (state->parser.previous.type == TOKEN_INDENTATION) ? state->parser.previous.length : 0;
2269  advance();
2270 
2271  /* For now this is going to be kinda broken */
2272  beginScope(state);
2273 
2274  ssize_t loopInd = state->current->localCount;
2275  int sawComma = 0;
2276  ssize_t varCount = 0;
2277  int matchedEquals = 0;
2278 
2279  if (!check(TOKEN_IDENTIFIER)) {
2280  errorAtCurrent("Empty variable list in 'for'");
2281  return;
2282  }
2283 
2284  do {
2285  if (!check(TOKEN_IDENTIFIER)) break;
2286  ssize_t ind = parseVariable(state, "Expected name for loop iterator.");
2287  if (state->parser.hadError) return;
2288  if (match(TOKEN_EQUAL)) {
2289  matchedEquals = 1;
2290  expression(state);
2291  } else {
2292  emitByte(OP_NONE);
2293  }
2294  defineVariable(state, ind);
2295  varCount++;
2296  if (check(TOKEN_COMMA)) sawComma = 1;
2297  } while (match(TOKEN_COMMA));
2298 
2299  int loopStart;
2300  int exitJump;
2301  int isIter = 0;
2302 
2303  if (!matchedEquals && match(TOKEN_IN)) {
2304 
2305  beginScope(state);
2306  expression(state);
2307  endScope(state);
2308 
2309  anonymousLocal(state);
2310  emitByte(OP_INVOKE_ITER);
2311  loopStart = currentChunk()->count;
2312  exitJump = emitJump(OP_CALL_ITER);
2313 
2314  if (varCount > 1 || sawComma) {
2315  EMIT_OPERAND_OP(OP_UNPACK, varCount);
2316  for (ssize_t i = loopInd + varCount - 1; i >= loopInd; i--) {
2317  EMIT_OPERAND_OP(OP_SET_LOCAL_POP, i);
2318  }
2319  } else {
2320  EMIT_OPERAND_OP(OP_SET_LOCAL_POP, loopInd);
2321  }
2322 
2323  isIter = 1;
2324 
2325  } else {
2326  consume(TOKEN_SEMICOLON,"Expected ';' after C-style loop initializer.");
2327  loopStart = currentChunk()->count;
2328 
2329  beginScope(state);
2330  expression(state); /* condition */
2331  endScope(state);
2332  exitJump = emitJump(OP_JUMP_IF_FALSE_OR_POP);
2333 
2334  if (check(TOKEN_SEMICOLON)) {
2335  advance();
2336  int bodyJump = emitJump(OP_JUMP);
2337  int incrementStart = currentChunk()->count;
2338  beginScope(state);
2339  do {
2340  expressionStatement(state);
2341  } while (match(TOKEN_COMMA));
2342  endScope(state);
2343 
2344  emitLoop(state, loopStart, OP_LOOP);
2345  loopStart = incrementStart;
2346  patchJump(bodyJump);
2347  }
2348  }
2349 
2350  consume(TOKEN_COLON,"Expected ':' after loop conditions.");
2351 
2352  int oldLocalCount = state->current->loopLocalCount;
2353  state->current->loopLocalCount = state->current->localCount;
2354  beginScope(state);
2355  block(state,blockWidth,"for");
2356  endScope(state);
2357 
2358  state->current->loopLocalCount = oldLocalCount;
2359  emitLoop(state, loopStart, isIter ? OP_LOOP_ITER : OP_LOOP);
2360  patchJump(exitJump);
2361  emitByte(OP_POP);
2362  optionalElse(state, blockWidth);
2363  patchBreaks(state, loopStart);
2364  endScope(state);
2365 }
2366 
2367 static void returnStatement(struct GlobalState * state) {
2368  if (check(TOKEN_EOL) || check(TOKEN_EOF)) {
2369  emitReturn(state);
2370  } else {
2371  if (state->current->type == TYPE_INIT) {
2372  error("__init__ may not return a value.");
2373  }
2374  parsePrecedence(state, PREC_ASSIGNMENT);
2375  emitByte(OP_RETURN);
2376  }
2377 }
2378 
2379 static void tryStatement(struct GlobalState * state) {
2380  size_t blockWidth = (state->parser.previous.type == TOKEN_INDENTATION) ? state->parser.previous.length : 0;
2381  advance();
2382  consume(TOKEN_COLON, "Expected ':' after 'try'.");
2383 
2384  /* Make sure we are in a local scope so this ends up on the stack */
2385  beginScope(state);
2386  int tryJump = emitJump(OP_PUSH_TRY);
2387 
2388  size_t exceptionObject = anonymousLocal(state);
2389  anonymousLocal(state); /* Try */
2390 
2391  beginScope(state);
2392  block(state,blockWidth,"try");
2393  endScope(state);
2394 
2395  if (state->parser.hadError) return;
2396 
2397 #define EXIT_JUMP_MAX 64
2398  int exitJumps = 2;
2399  int exitJumpOffsets[EXIT_JUMP_MAX] = {0};
2400 
2401  /* Jump possibly to `else` */
2402  exitJumpOffsets[0] = emitJump(OP_JUMP);
2403 
2404  /* Except entry point; ENTER_EXCEPT jumps to `finally` or continues to
2405  * first `except` expression test; may end up redundant if there is only an 'else'. */
2406  patchJump(tryJump);
2407  exitJumpOffsets[1] = emitJump(OP_ENTER_EXCEPT);
2408 
2409  int firstJump = 0;
2410  int nextJump = -1;
2411 
2412 _anotherExcept:
2413  if (state->parser.hadError) return;
2414  if (blockWidth == 0 || (check(TOKEN_INDENTATION) && (state->parser.current.length == blockWidth))) {
2415  KrkToken previous;
2416  if (blockWidth) {
2417  previous = state->parser.previous;
2418  advance();
2419  }
2420  if (exitJumps && !firstJump && match(TOKEN_EXCEPT)) {
2421  if (nextJump != -1) {
2422  patchJump(nextJump);
2423  }
2424  /* Match filter expression (should be class or tuple) */
2425  if (!check(TOKEN_COLON) && !check(TOKEN_AS)) {
2426  expression(state);
2427  } else {
2428  emitByte(OP_NONE);
2429  }
2430  nextJump = emitJump(OP_FILTER_EXCEPT);
2431 
2432  /* Match 'as' to rename exception */
2433  size_t nameInd = 0;
2434  if (match(TOKEN_AS)) {
2435  consume(TOKEN_IDENTIFIER, "Expected identifier after 'as'.");
2436  state->current->locals[exceptionObject].name = state->parser.previous;
2437  /* `renameLocal` only introduces names for scoped debugging */
2438  nameInd = renameLocal(state, exceptionObject, state->parser.previous);
2439  } else {
2440  state->current->locals[exceptionObject].name = syntheticToken("");
2441  }
2442 
2443  consume(TOKEN_COLON, "Expected ':' after 'except'.");
2444  beginScope(state);
2445  block(state,blockWidth,"except");
2446  endScope(state);
2447 
2448  /* Remove scoped name */
2449  if (nameInd) state->current->codeobject->localNames[nameInd].deathday = (size_t)currentChunk()->count;
2450 
2451  if (exitJumps < EXIT_JUMP_MAX) {
2452  exitJumpOffsets[exitJumps++] = emitJump(OP_JUMP);
2453  } else {
2454  error("Too many 'except' clauses.");
2455  return;
2456  }
2457 
2458  goto _anotherExcept;
2459  } else if (firstJump != 1 && match(TOKEN_ELSE)) {
2460  consume(TOKEN_COLON, "Expected ':' after 'else'.");
2461  patchJump(exitJumpOffsets[0]);
2462  firstJump = 1;
2463  emitByte(OP_TRY_ELSE);
2464  state->current->locals[exceptionObject].name = syntheticToken("");
2465  beginScope(state);
2466  block(state, blockWidth, "else");
2467  endScope(state);
2468  if (nextJump == -1) {
2469  /* If there were no except: blocks, we need to make sure that the
2470  * 'try' handler goes directly to the finally, so that 'break'/'continue'
2471  * within the 'try' does not run this 'else' step. */
2472  patchJump(tryJump);
2473  }
2474  goto _anotherExcept;
2475  } else if (match(TOKEN_FINALLY)) {
2476  consume(TOKEN_COLON, "Expected ':' after 'finally'.");
2477  for (int i = firstJump; i < exitJumps; ++i) {
2478  patchJump(exitJumpOffsets[i]);
2479  }
2480  if (nextJump != -1) {
2481  patchJump(nextJump);
2482  }
2483  emitByte(OP_BEGIN_FINALLY);
2484  exitJumps = 0;
2485  state->current->locals[exceptionObject].name = syntheticToken("");
2486  beginScope(state);
2487  block(state,blockWidth,"finally");
2488  endScope(state);
2489  nextJump = -2;
2490  emitByte(OP_END_FINALLY);
2491  } else if (!check(TOKEN_EOL) && !check(TOKEN_EOF)) {
2492  krk_ungetToken(&state->scanner, state->parser.current);
2493  state->parser.current = state->parser.previous;
2494  if (blockWidth) {
2495  state->parser.previous = previous;
2496  }
2497  } else {
2498  advance(); /* Ignore this blank indentation line */
2499  }
2500  }
2501 
2502  for (int i = firstJump; i < exitJumps; ++i) {
2503  patchJump(exitJumpOffsets[i]);
2504  }
2505 
2506  if (nextJump >= 0) {
2507  patchJump(nextJump);
2508  emitByte(OP_BEGIN_FINALLY);
2509  emitByte(OP_END_FINALLY);
2510  }
2511 
2512  endScope(state); /* will pop the exception handler */
2513 }
2514 
2515 static void raiseStatement(struct GlobalState * state) {
2516  parsePrecedence(state, PREC_ASSIGNMENT);
2517 
2518  if (match(TOKEN_FROM)) {
2519  parsePrecedence(state, PREC_ASSIGNMENT);
2520  emitByte(OP_RAISE_FROM);
2521  } else {
2522  emitByte(OP_RAISE);
2523  }
2524 }
2525 
2526 static size_t importModule(struct GlobalState * state, KrkToken * startOfName, int leadingDots) {
2527  size_t ind = 0;
2528  struct StringBuilder sb = {0};
2529 
2530  for (int i = 0; i < leadingDots; ++i) {
2531  pushStringBuilder(&sb, '.');
2532  }
2533 
2534  if (!(leadingDots && check(TOKEN_IMPORT))) {
2535  consume(TOKEN_IDENTIFIER, "Expected module name after 'import'.");
2536  if (state->parser.hadError) goto _freeImportName;
2537  pushStringBuilderStr(&sb, state->parser.previous.start, state->parser.previous.length);
2538 
2539  while (match(TOKEN_DOT)) {
2540  pushStringBuilderStr(&sb, state->parser.previous.start, state->parser.previous.length);
2541  consume(TOKEN_IDENTIFIER, "Expected module path element after '.'");
2542  if (state->parser.hadError) goto _freeImportName;
2543  pushStringBuilderStr(&sb, state->parser.previous.start, state->parser.previous.length);
2544  }
2545  }
2546 
2547  startOfName->start = sb.bytes;
2548  startOfName->length = sb.length;
2549 
2550  ind = identifierConstant(state, startOfName);
2551  EMIT_OPERAND_OP(OP_IMPORT, ind);
2552 
2553 _freeImportName:
2554  discardStringBuilder(&sb);
2555  return ind;
2556 }
2557 
2558 static void importStatement(struct GlobalState * state) {
2559  do {
2560  KrkToken firstName = state->parser.current;
2561  KrkToken startOfName = {0};
2562  size_t ind = importModule(state, &startOfName, 0);
2563  if (match(TOKEN_AS)) {
2564  consume(TOKEN_IDENTIFIER, "Expected identifier after 'as'.");
2565  ind = identifierConstant(state, &state->parser.previous);
2566  } else if (startOfName.length != firstName.length) {
2574  emitByte(OP_POP);
2575  state->parser.previous = firstName;
2576  ind = identifierConstant(state, &firstName);
2577  EMIT_OPERAND_OP(OP_IMPORT, ind);
2578  }
2579  declareVariable(state);
2580  defineVariable(state, ind);
2581  } while (match(TOKEN_COMMA));
2582 }
2583 
2584 static void optionsImport(struct GlobalState * state) {
2585  int expectCloseParen = 0;
2586 
2587  KrkToken compile_time_builtins = syntheticToken("compile_time_builtins");
2588  KrkToken no_implicit_self = syntheticToken("no_implicit_self");
2589 
2590  advance();
2591  consume(TOKEN_IMPORT, "__options__ is not a package\n");
2592 
2593  if (match(TOKEN_LEFT_PAREN)) {
2594  expectCloseParen = 1;
2595  startEatingWhitespace();
2596  }
2597 
2598  do {
2599  consume(TOKEN_IDENTIFIER, "Expected member name");
2600 
2601  /* Okay, what is it? */
2602  if (identifiersEqual(&state->parser.previous, &compile_time_builtins)) {
2603  state->current->optionsFlags |= OPTIONS_FLAG_COMPILE_TIME_BUILTINS;
2604  } else if (identifiersEqual(&state->parser.previous, &no_implicit_self)) {
2605  state->current->optionsFlags |= OPTIONS_FLAG_NO_IMPLICIT_SELF;
2606  } else {
2607  error("'%.*s' is not a recognized __options__ import",
2608  (int)state->parser.previous.length, state->parser.previous.start);
2609  break;
2610  }
2611 
2612  if (check(TOKEN_AS)) {
2613  errorAtCurrent("__options__ imports can not be given names");
2614  break;
2615  }
2616 
2617  } while (match(TOKEN_COMMA) && !check(TOKEN_RIGHT_PAREN));
2618 
2619  if (expectCloseParen) {
2620  stopEatingWhitespace();
2621  consume(TOKEN_RIGHT_PAREN, "Expected ')' after import list started with '('");
2622  }
2623 }
2624 
2625 static void fromImportStatement(struct GlobalState * state) {
2626  int expectCloseParen = 0;
2627  KrkToken startOfName = {0};
2628  int leadingDots = 0;
2629 
2630  KrkToken options = syntheticToken("__options__");
2631  if (check(TOKEN_IDENTIFIER) && identifiersEqual(&state->parser.current, &options)) {
2632  /* from __options__ import ... */
2633  optionsImport(state);
2634  return;
2635  }
2636 
2637  while (match(TOKEN_DOT) || match(TOKEN_ELLIPSIS)) {
2638  leadingDots += state->parser.previous.length;
2639  }
2640 
2641  importModule(state, &startOfName, leadingDots);
2642  consume(TOKEN_IMPORT, "Expected 'import' after module name");
2643  if (match(TOKEN_LEFT_PAREN)) {
2644  expectCloseParen = 1;
2645  startEatingWhitespace();
2646  }
2647  do {
2648  consume(TOKEN_IDENTIFIER, "Expected member name");
2649  size_t member = identifierConstant(state, &state->parser.previous);
2650  emitBytes(OP_DUP, 0); /* Duplicate the package object so we can GET_PROPERTY on it? */
2651  EMIT_OPERAND_OP(OP_IMPORT_FROM, member);
2652  if (match(TOKEN_AS)) {
2653  consume(TOKEN_IDENTIFIER, "Expected identifier after 'as'");
2654  member = identifierConstant(state, &state->parser.previous);
2655  }
2656  if (state->current->scopeDepth) {
2657  /* Swaps the original module and the new possible local so it can be in the right place */
2658  emitByte(OP_SWAP);
2659  }
2660  declareVariable(state);
2661  defineVariable(state, member);
2662  } while (match(TOKEN_COMMA) && !check(TOKEN_RIGHT_PAREN));
2663  if (expectCloseParen) {
2664  stopEatingWhitespace();
2665  consume(TOKEN_RIGHT_PAREN, "Expected ')' after import list started with '('");
2666  }
2667  emitByte(OP_POP); /* Pop the remaining copy of the module. */
2668 }
2669 
2670 static void delStatement(struct GlobalState * state) {
2671  do {
2672  state->current->delSatisfied = 0;
2673  parsePrecedence(state, PREC_DEL_TARGET);
2674  if (!state->current->delSatisfied) {
2675  errorAtCurrent("Invalid del target");
2676  }
2677  } while (match(TOKEN_COMMA));
2678 }
2679 
2680 static void assertStatement(struct GlobalState * state) {
2681  expression(state);
2682  int elseJump = emitJump(OP_JUMP_IF_TRUE_OR_POP);
2683 
2684  KrkToken assertionError = syntheticToken("AssertionError");
2685  size_t ind = identifierConstant(state, &assertionError);
2686  EMIT_OPERAND_OP(OP_GET_GLOBAL, ind);
2687  int args = 0;
2688 
2689  if (match(TOKEN_COMMA)) {
2690  expression(state);
2691  args = 1;
2692  }
2693 
2694  EMIT_OPERAND_OP(OP_CALL, args);
2695  emitByte(OP_RAISE);
2696 
2697  patchJump(elseJump);
2698  emitByte(OP_POP);
2699 }
2700 
2701 static void errorAfterStatement(struct GlobalState * state) {
2702  switch (state->parser.current.type) {
2703  case TOKEN_RIGHT_BRACE:
2704  case TOKEN_RIGHT_PAREN:
2705  case TOKEN_RIGHT_SQUARE:
2706  errorAtCurrent("Unmatched '%.*s'",
2707  (int)state->parser.current.length, state->parser.current.start);
2708  break;
2709  case TOKEN_IDENTIFIER:
2710  errorAtCurrent("Unexpected %.*s after statement.",10,"identifier");
2711  break;
2712  case TOKEN_STRING:
2713  case TOKEN_BIG_STRING:
2714  errorAtCurrent("Unexpected %.*s after statement.",6,"string");
2715  break;
2716  default:
2717  errorAtCurrent("Unexpected %.*s after statement.",
2718  (int)state->parser.current.length, state->parser.current.start);
2719  }
2720 }
2721 
2722 static void simpleStatement(struct GlobalState * state) {
2723 _anotherSimpleStatement:
2724  if (match(TOKEN_RAISE)) {
2725  raiseStatement(state);
2726  } else if (match(TOKEN_RETURN)) {
2727  returnStatement(state);
2728  } else if (match(TOKEN_IMPORT)) {
2729  importStatement(state);
2730  } else if (match(TOKEN_FROM)) {
2731  fromImportStatement(state);
2732  } else if (match(TOKEN_BREAK)) {
2733  breakStatement(state);
2734  } else if (match(TOKEN_CONTINUE)) {
2735  continueStatement(state);
2736  } else if (match(TOKEN_DEL)) {
2737  delStatement(state);
2738  } else if (match(TOKEN_ASSERT)) {
2739  assertStatement(state);
2740  } else if (match(TOKEN_PASS)) {
2741  /* Do nothing. */
2742  } else if (match(TOKEN_LET)) {
2743  letDeclaration(state);
2744  } else {
2745  expressionStatement(state);
2746  }
2747  if (match(TOKEN_SEMICOLON)) goto _anotherSimpleStatement;
2748  if (!match(TOKEN_EOL) && !match(TOKEN_EOF)) {
2749  errorAfterStatement(state);
2750  }
2751 }
2752 
2753 static void statement(struct GlobalState * state) {
2754  if (match(TOKEN_EOL) || match(TOKEN_EOF)) {
2755  return; /* Meaningless blank line */
2756  }
2757 
2758  if (check(TOKEN_IF)) {
2759  ifStatement(state);
2760  } else if (check(TOKEN_WHILE)) {
2761  whileStatement(state);
2762  } else if (check(TOKEN_FOR)) {
2763  forStatement(state);
2764  } else if (check(TOKEN_ASYNC)) {
2765  asyncDeclaration(state, 0);
2766  } else if (check(TOKEN_TRY)) {
2767  tryStatement(state);
2768  } else if (check(TOKEN_WITH)) {
2769  withStatement(state);
2770  } else {
2771  /* These statements don't eat line feeds, so we need expect to see another one. */
2772  simpleStatement(state);
2773  }
2774 }
2775 
2776 static void yield(struct GlobalState * state, int exprType, RewindState *rewind) {
2777  if (state->current->type == TYPE_MODULE ||
2778  state->current->type == TYPE_INIT ||
2779  state->current->type == TYPE_CLASS) {
2780  error("'yield' outside function");
2781  return;
2782  }
2783  state->current->codeobject->obj.flags |= KRK_OBJ_FLAGS_CODEOBJECT_IS_GENERATOR;
2784  if (match(TOKEN_FROM)) {
2785  parsePrecedence(state, PREC_ASSIGNMENT);
2786  emitByte(OP_INVOKE_ITER);
2787  emitByte(OP_NONE);
2788  size_t loopContinue = currentChunk()->count;
2789  size_t exitJump = emitJump(OP_YIELD_FROM);
2790  emitByte(OP_YIELD);
2791  emitLoop(state, loopContinue, OP_LOOP);
2792  patchJump(exitJump);
2793  } else if (check(TOKEN_EOL) || check(TOKEN_EOF) || check(TOKEN_RIGHT_PAREN) || check(TOKEN_RIGHT_BRACE)) {
2794  emitByte(OP_NONE);
2795  emitByte(OP_YIELD);
2796  } else {
2797  parsePrecedence(state, PREC_ASSIGNMENT);
2798  emitByte(OP_YIELD);
2799  }
2800  invalidTarget(state, exprType, "yield");
2801 }
2802 
2803 static void await(struct GlobalState * state, int exprType, RewindState *rewind) {
2804  if (!isCoroutine(state->current->type)) {
2805  error("'await' outside async function");
2806  return;
2807  }
2808 
2809  parsePrecedence(state, PREC_ASSIGNMENT);
2810  emitByte(OP_INVOKE_AWAIT);
2811  emitByte(OP_NONE);
2812  size_t loopContinue = currentChunk()->count;
2813  size_t exitJump = emitJump(OP_YIELD_FROM);
2814  emitByte(OP_YIELD);
2815  emitLoop(state, loopContinue, OP_LOOP);
2816  patchJump(exitJump);
2817  invalidTarget(state, exprType, "await");
2818 }
2819 
2820 static void unot_(struct GlobalState * state, int exprType, RewindState *rewind) {
2821  parsePrecedence(state, PREC_NOT);
2822  emitByte(OP_NOT);
2823  invalidTarget(state, exprType, "operator");
2824 }
2825 
2826 static void unary(struct GlobalState * state, int exprType, RewindState *rewind) {
2827  KrkTokenType operatorType = state->parser.previous.type;
2828  parsePrecedence(state, PREC_FACTOR);
2829  invalidTarget(state, exprType, "operator");
2830  switch (operatorType) {
2831  case TOKEN_PLUS: emitByte(OP_POS); break;
2832  case TOKEN_MINUS: emitByte(OP_NEGATE); break;
2833  case TOKEN_TILDE: emitByte(OP_BITNEGATE); break;
2834  case TOKEN_BANG: emitByte(OP_NOT); break;
2835  default: return;
2836  }
2837 }
2838 
2839 static int isHex(int c) {
2840  return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
2841 }
2842 
2843 static int _pushHex(struct GlobalState * state, int isBytes, struct StringBuilder * sb, const char *c, const char *end, size_t n, char type) {
2844  char tmpbuf[10] = {0};
2845  for (size_t i = 0; i < n; ++i) {
2846  if (c + i + 2 == end || !isHex(c[i+2])) {
2847  error("truncated \\%c escape", type);
2848  return 1;
2849  }
2850  tmpbuf[i] = c[i+2];
2851  }
2852  unsigned long value = strtoul(tmpbuf, NULL, 16);
2853  if (value >= 0x110000) {
2854  error("invalid codepoint in \\%c escape", type);
2855  return 1;
2856  }
2857  if (isBytes) {
2858  krk_pushStringBuilder(sb, value);
2859  } else {
2860  unsigned char bytes[5] = {0};
2861  size_t len = krk_codepointToBytes(value, bytes);
2862  krk_pushStringBuilderStr(sb, (char*)bytes, len);
2863  }
2864  return 0;
2865 }
2866 
2867 static void string(struct GlobalState * state, int exprType, RewindState *rewind) {
2868  struct StringBuilder sb = {0};
2869 #define PUSH_CHAR(c) krk_pushStringBuilder(&sb, c)
2870 #define PUSH_HEX(n, type) _pushHex(state, isBytes, &sb, c, end, n, type)
2871 
2872  int isBytes = (state->parser.previous.type == TOKEN_PREFIX_B);
2873  int isFormat = (state->parser.previous.type == TOKEN_PREFIX_F);
2874  int isRaw = (state->parser.previous.type == TOKEN_PREFIX_R);
2875 
2876  const char * lineBefore = krk_tellScanner(&state->scanner).linePtr;
2877  size_t lineNo = krk_tellScanner(&state->scanner).line;
2878 
2879  if ((isBytes || isFormat || isRaw) && !(match(TOKEN_STRING) || match(TOKEN_BIG_STRING))) {
2880  error("Expected string after prefix? (Internal error - scanner should not have produced this.)");
2881  return;
2882  }
2883 
2884  int formatElements = 0;
2885 
2886  /* This should capture everything but the quotes. */
2887  do {
2888  if (isRaw) {
2889  for (size_t i = 0; i < state->parser.previous.length - (state->parser.previous.type == TOKEN_BIG_STRING ? 6 : 2); ++i) {
2890  PUSH_CHAR(state->parser.previous.start[(state->parser.previous.type == TOKEN_BIG_STRING ? 3 : 1) + i]);
2891  }
2892  goto _nextStr;
2893  }
2894  int type = state->parser.previous.type == TOKEN_BIG_STRING ? 3 : 1;
2895  const char * c = state->parser.previous.start + type;
2896  const char * end = state->parser.previous.start + state->parser.previous.length - type;
2897  while (c < end) {
2898  if (*c == '\\') {
2899  switch (c[1]) {
2900  case '\\': PUSH_CHAR('\\'); break;
2901  case '\'': PUSH_CHAR('\''); break;
2902  case '\"': PUSH_CHAR('\"'); break;
2903  case 'a': PUSH_CHAR('\a'); break;
2904  case 'b': PUSH_CHAR('\b'); break;
2905  case 'f': PUSH_CHAR('\f'); break;
2906  case 'n': PUSH_CHAR('\n'); break;
2907  case 'r': PUSH_CHAR('\r'); break;
2908  case 't': PUSH_CHAR('\t'); break;
2909  case 'v': PUSH_CHAR('\v'); break;
2910  case '[': PUSH_CHAR('\033'); break;
2911  case 'x': {
2912  PUSH_HEX(2,'x');
2913  c += 2;
2914  } break;
2915  case 'u': {
2916  if (isBytes) {
2917  PUSH_CHAR(c[0]);
2918  PUSH_CHAR(c[1]);
2919  } else {
2920  PUSH_HEX(4,'u');
2921  c += 4;
2922  }
2923  } break;
2924  case 'U': {
2925  if (isBytes) {
2926  PUSH_CHAR(c[0]);
2927  PUSH_CHAR(c[1]);
2928  } else {
2929  PUSH_HEX(8,'U');
2930  c += 8;
2931  }
2932  } break;
2933  case '\n': break;
2934  default:
2935  if (c[1] >= '0' && c[1] <= '7') {
2936  int out = c[1] - '0';
2937  if (c + 2 != end && (c[2] >= '0' && c[2] <= '7')) {
2938  out <<= 3;
2939  out += c[2] - '0';
2940  c++;
2941  if (c + 1 != end && (c[2] >= '0' && c[2] <= '7')) {
2942  out <<= 3;
2943  out += c[2] - '0';
2944  c++;
2945  }
2946  }
2947  if (isBytes) {
2948  out = out % 256;
2949  PUSH_CHAR(out);
2950  } else {
2951  unsigned char bytes[5] = {0};
2952  size_t len = krk_codepointToBytes(out, bytes);
2953  for (size_t i = 0; i < len; i++) PUSH_CHAR(bytes[i]);
2954  }
2955  } else {
2956  PUSH_CHAR(c[0]);
2957  c++;
2958  continue;
2959  }
2960  }
2961  c += 2;
2962  } else if (isFormat && *c == '}') {
2963  if (c[1] != '}') {
2964  error("single '}' not allowed in f-string");
2965  goto _cleanupError;
2966  }
2967  PUSH_CHAR('}');
2968  c += 2;
2969  continue;
2970  } else if (isFormat && *c == '{') {
2971  if (c[1] == '{') {
2972  PUSH_CHAR('{');
2973  c += 2;
2974  continue;
2975  }
2976  if (sb.length) { /* Make sure there's a string for coersion reasons */
2977  emitConstant(krk_finishStringBuilder(&sb));
2978  formatElements++;
2979  }
2980  const char * start = c+1;
2981  KrkScanner beforeExpression = krk_tellScanner(&state->scanner);
2982  Parser parserBefore = state->parser;
2983  KrkScanner inner = (KrkScanner){.start=c+1, .cur=c+1, .linePtr=lineBefore, .line=lineNo, .startOfLine = 0, .hasUnget = 0};
2984  krk_rewindScanner(&state->scanner, inner);
2985  advance();
2986  parsePrecedence(state, PREC_COMMA); /* allow unparen'd tuples, but not assignments, as expressions in f-strings */
2987  if (state->parser.hadError) goto _cleanupError;
2988  inner = krk_tellScanner(&state->scanner); /* To figure out how far to advance c */
2989  krk_rewindScanner(&state->scanner, beforeExpression); /* To get us back to where we were with a string token */
2990  state->parser = parserBefore;
2991  c = inner.start;
2992 
2993  int formatType = 0;
2994 
2995  while (*c == ' ') c++;
2996  if (*c == '=') {
2997  c++;
2998  while (*c == ' ') c++;
2999  emitConstant(OBJECT_VAL(krk_copyString(start,c-start)));
3000  formatElements++;
3001  formatType |= FORMAT_OP_EQ;
3002  }
3003 
3004  if (*c == '!') {
3005  c++;
3006  /* Conversion specifiers, must only be one */
3007  if (*c == 'r') {
3008  formatType |= FORMAT_OP_REPR;
3009  } else if (*c == 's') {
3010  formatType |= FORMAT_OP_STR;
3011  } else {
3012  error("Unsupported conversion flag '%c' for f-string expression.", *c);
3013  goto _cleanupError;
3014  }
3015  c++;
3016  }
3017 
3018  if (*c == ':') {
3019  /* TODO format specs */
3020  const char * formatStart = c+1;
3021  c++;
3022  while (c < end && *c != '}') c++;
3023  emitConstant(OBJECT_VAL(krk_copyString(formatStart,c-formatStart)));
3024  formatType |= FORMAT_OP_FORMAT;
3025  }
3026 
3027  /* Default to !r if '=' was present but neither was specified. */
3028  if (!(formatType & (FORMAT_OP_FORMAT | FORMAT_OP_STR)) && (formatType & FORMAT_OP_EQ)) {
3029  formatType |= FORMAT_OP_REPR;
3030  }
3031 
3032  EMIT_OPERAND_OP(OP_FORMAT_VALUE, formatType);
3033 
3034  if (*c != '}') {
3035  error("Expected closing '}' after expression in f-string");
3036  goto _cleanupError;
3037  }
3038 
3039  formatElements++;
3040  c++;
3041  } else {
3042  if (*(unsigned char*)c > 127 && isBytes) {
3043  error("bytes literal can only contain ASCII characters");
3044  goto _cleanupError;
3045  }
3046  PUSH_CHAR(*c);
3047  c++;
3048  }
3049  }
3050 
3051 _nextStr:
3052  (void)0;
3053  isRaw = 0;
3054  isFormat = 0;
3055  if (!isBytes) {
3056  if (match(TOKEN_PREFIX_F)) {
3057  isFormat = 1;
3058  } else if (match(TOKEN_PREFIX_R)) {
3059  isRaw = 1;
3060  }
3061  }
3062  } while ((!isBytes || match(TOKEN_PREFIX_B)) && (match(TOKEN_STRING) || match(TOKEN_BIG_STRING)));
3063  if (isBytes && (match(TOKEN_STRING) || match(TOKEN_BIG_STRING))) {
3064  error("Can not mix bytes and string literals");
3065  goto _cleanupError;
3066  }
3067  if (isBytes) {
3068  emitConstant(krk_finishStringBuilderBytes(&sb));
3069  return;
3070  }
3071  if (sb.length || !formatElements) {
3072  emitConstant(krk_finishStringBuilder(&sb));
3073  formatElements++;
3074  }
3075  if (formatElements != 1) {
3076  EMIT_OPERAND_OP(OP_MAKE_STRING, formatElements);
3077  }
3078 _cleanupError:
3080 #undef PUSH_CHAR
3081 }
3082 
3083 static size_t addUpvalue(struct GlobalState * state, Compiler * compiler, ssize_t index, int isLocal, KrkToken name) {
3084  size_t upvalueCount = compiler->codeobject->upvalueCount;
3085  for (size_t i = 0; i < upvalueCount; ++i) {
3086  Upvalue * upvalue = &compiler->upvalues[i];
3087  if ((ssize_t)upvalue->index == index && upvalue->isLocal == isLocal) {
3088  return i;
3089  }
3090  }
3091  if (upvalueCount + 1 > compiler->upvaluesSpace) {
3092  size_t old = compiler->upvaluesSpace;
3093  compiler->upvaluesSpace = KRK_GROW_CAPACITY(old);
3094  compiler->upvalues = KRK_GROW_ARRAY(Upvalue,compiler->upvalues,old,compiler->upvaluesSpace);
3095  }
3096  compiler->upvalues[upvalueCount].isLocal = isLocal;
3097  compiler->upvalues[upvalueCount].index = index;
3098  compiler->upvalues[upvalueCount].name = name;
3099  return compiler->codeobject->upvalueCount++;
3100 }
3101 
3102 static ssize_t resolveUpvalue(struct GlobalState * state, Compiler * compiler, KrkToken * name) {
3103  size_t upvalueCount = compiler->codeobject->upvalueCount;
3104  for (size_t i = 0; i < upvalueCount; ++i) {
3105  if (identifiersEqual(name, &compiler->upvalues[i].name)) {
3106  return i;
3107  }
3108  }
3109 
3110  if (compiler->enclosing == NULL) return -1;
3111 
3112  ssize_t local = resolveLocal(state, compiler->enclosing, name);
3113  if (local != -1) {
3114  compiler->enclosing->locals[local].isCaptured = 1;
3115  return addUpvalue(state, compiler, local, 1, *name);
3116  }
3117  ssize_t upvalue = resolveUpvalue(state, compiler->enclosing, name);
3118  if (upvalue != -1) {
3119  return addUpvalue(state, compiler, upvalue, 0, *name);
3120  }
3121  return -1;
3122 }
3123 
3124 #define OP_NONE_LONG -1
3125 #define DO_VARIABLE(opset,opget,opdel) do { \
3126  if (exprType == EXPR_ASSIGN_TARGET) { \
3127  if (matchComplexEnd(state)) { \
3128  EMIT_OPERAND_OP(opset, arg); \
3129  break; \
3130  } \
3131  exprType = EXPR_NORMAL; \
3132  } \
3133  if (exprType == EXPR_CAN_ASSIGN && match(TOKEN_EQUAL)) { \
3134  parsePrecedence(state, PREC_ASSIGNMENT); \
3135  EMIT_OPERAND_OP(opset, arg); \
3136  } else if (exprType == EXPR_CAN_ASSIGN && matchAssignment(state)) { \
3137  EMIT_OPERAND_OP(opget, arg); \
3138  assignmentValue(state); \
3139  EMIT_OPERAND_OP(opset, arg); \
3140  } else if (exprType == EXPR_DEL_TARGET && checkEndOfDel(state)) {\
3141  if (opdel == OP_NONE) { emitByte(OP_NONE); EMIT_OPERAND_OP(opset, arg); } \
3142  else { EMIT_OPERAND_OP(opdel, arg); } \
3143  } else { \
3144  EMIT_OPERAND_OP(opget, arg); \
3145  } } while (0)
3146 
3147 static void namedVariable(struct GlobalState * state, KrkToken name, int exprType) {
3148  if (state->current->type == TYPE_CLASS) {
3149  /* Only at the class body level, see if this is a class property. */
3150  struct IndexWithNext * properties = state->current->properties;
3151  while (properties) {
3152  KrkString * constant = AS_STRING(currentChunk()->constants.values[properties->ind]);
3153  if (constant->length == name.length && !memcmp(constant->chars, name.start, name.length)) {
3154  ssize_t arg = properties->ind;
3155  DO_VARIABLE(OP_SET_NAME, OP_GET_NAME, OP_NONE);
3156  return;
3157  }
3158  properties = properties->next;
3159  }
3160  }
3161  ssize_t arg = resolveLocal(state, state->current, &name);
3162  if (arg != -1) {
3163  DO_VARIABLE(OP_SET_LOCAL, OP_GET_LOCAL, OP_NONE);
3164  } else if ((arg = resolveUpvalue(state, state->current, &name)) != -1) {
3165  DO_VARIABLE(OP_SET_UPVALUE, OP_GET_UPVALUE, OP_NONE);
3166  } else {
3167  if ((state->current->optionsFlags & OPTIONS_FLAG_COMPILE_TIME_BUILTINS) && *name.start != '_') {
3168  KrkValue value;
3169  if (krk_tableGet_fast(&vm.builtins->fields, krk_copyString(name.start, name.length), &value)) {
3170  if ((exprType == EXPR_ASSIGN_TARGET && matchComplexEnd(state)) ||
3171  (exprType == EXPR_CAN_ASSIGN && match(TOKEN_EQUAL)) ||
3172  (exprType == EXPR_CAN_ASSIGN && matchAssignment(state))) {
3173  error("Can not assign to '%.*s' when 'compile_time_builtins' is enabled.",
3174  (int)name.length, name.start);
3175  } else if (exprType == EXPR_DEL_TARGET && checkEndOfDel(state)) {
3176  error("Can not delete '%.*s' when 'compile_time_builtins' is enabled.",
3177  (int)name.length, name.start);
3178  } else {
3179  emitConstant(value);
3180  }
3181  return;
3182  }
3183  }
3184  arg = identifierConstant(state, &name);
3185  DO_VARIABLE(OP_SET_GLOBAL, OP_GET_GLOBAL, OP_DEL_GLOBAL);
3186  }
3187 }
3188 #undef DO_VARIABLE
3189 
3190 static void variable(struct GlobalState * state, int exprType, RewindState *rewind) {
3191  namedVariable(state, state->parser.previous, exprType);
3192 }
3193 
3194 static int isClassOrStaticMethod(FunctionType type) {
3195  return (type == TYPE_STATIC || type == TYPE_CLASSMETHOD);
3196 }
3197 
3198 static void super_(struct GlobalState * state, int exprType, RewindState *rewind) {
3199  consume(TOKEN_LEFT_PAREN, "Expected 'super' to be called.");
3200 
3201  /* Argument time */
3202  if (match(TOKEN_RIGHT_PAREN)) {
3203  if (!isMethod(state->current->type) && !isClassOrStaticMethod(state->current->type)) {
3204  error("super() outside of a method body requires arguments");
3205  return;
3206  }
3207  if (!state->current->codeobject->potentialPositionals) {
3208  error("super() is not valid in a function with no arguments");
3209  return;
3210  }
3211  namedVariable(state, state->currentClass->name, 0);
3212  EMIT_OPERAND_OP(OP_GET_LOCAL, 0);
3213  } else {
3214  expression(state);
3215  if (match(TOKEN_COMMA)) {
3216  expression(state);
3217  } else {
3218  emitByte(OP_UNSET);
3219  }
3220  consume(TOKEN_RIGHT_PAREN, "Expected ')' after argument list");
3221  }
3222  consume(TOKEN_DOT, "Expected a field of 'super()' to be referenced.");
3223  consume(TOKEN_IDENTIFIER, "Expected a field name.");
3224  size_t ind = identifierConstant(state, &state->parser.previous);
3225  EMIT_OPERAND_OP(OP_GET_SUPER, ind);
3226 }
3227 
3228 static void comprehensionInner(struct GlobalState * state, KrkScanner scannerBefore, Parser parserBefore, void (*body)(struct GlobalState*,size_t), size_t arg) {
3229  ssize_t loopInd = state->current->localCount;
3230  ssize_t varCount = 0;
3231  int sawComma = 0;
3232  if (!check(TOKEN_IDENTIFIER)) {
3233  errorAtCurrent("Empty variable list in comprehension");
3234  return;
3235  }
3236  do {
3237  if (!check(TOKEN_IDENTIFIER)) break;
3238  defineVariable(state, parseVariable(state, "Expected name for iteration variable."));
3239  if (state->parser.hadError) return;
3240  emitByte(OP_NONE);
3241  defineVariable(state, loopInd);
3242  varCount++;
3243  if (check(TOKEN_COMMA)) sawComma = 1;
3244  } while (match(TOKEN_COMMA));
3245 
3246  consume(TOKEN_IN, "Only iterator loops (for ... in ...) are allowed in generator expressions.");
3247 
3248  beginScope(state);
3249  parsePrecedence(state, PREC_OR); /* Otherwise we can get trapped on a ternary */
3250  endScope(state);
3251 
3252  anonymousLocal(state);
3253  emitByte(OP_INVOKE_ITER);
3254  int loopStart = currentChunk()->count;
3255  int exitJump = emitJump(OP_CALL_ITER);
3256 
3257  if (varCount > 1 || sawComma) {
3258  EMIT_OPERAND_OP(OP_UNPACK, varCount);
3259  for (ssize_t i = loopInd + varCount - 1; i >= loopInd; i--) {
3260  EMIT_OPERAND_OP(OP_SET_LOCAL_POP, i);
3261  }
3262  } else {
3263  EMIT_OPERAND_OP(OP_SET_LOCAL_POP, loopInd);
3264  }
3265 
3266  if (match(TOKEN_IF)) {
3267  parsePrecedence(state, PREC_OR);
3268  int acceptJump = emitJump(OP_JUMP_IF_TRUE_OR_POP);
3269  emitLoop(state, loopStart, OP_LOOP);
3270  patchJump(acceptJump);
3271  emitByte(OP_POP); /* Pop condition */
3272  }
3273 
3274  beginScope(state);
3275  if (match(TOKEN_FOR)) {
3276  comprehensionInner(state, scannerBefore, parserBefore, body, arg);
3277  } else {
3278  KrkScanner scannerAfter = krk_tellScanner(&state->scanner);
3279  Parser parserAfter = state->parser;
3280  krk_rewindScanner(&state->scanner, scannerBefore);
3281  state->parser = parserBefore;
3282 
3283  body(state, arg);
3284 
3285  krk_rewindScanner(&state->scanner, scannerAfter);
3286  state->parser = parserAfter;
3287  }
3288  endScope(state);
3289 
3290  emitLoop(state, loopStart, OP_LOOP_ITER);
3291  patchJump(exitJump);
3292  emitByte(OP_POP);
3293 }
3294 
3295 static void yieldInner(struct GlobalState * state, size_t arg) {
3296  (void)arg;
3297  expression(state);
3298  emitBytes(OP_YIELD, OP_POP);
3299 }
3300 
3310 static void generatorExpression(struct GlobalState * state, KrkScanner scannerBefore, Parser parserBefore, void (*body)(struct GlobalState*,size_t)) {
3311  state->parser.previous = syntheticToken("<genexpr>");
3312  Compiler subcompiler;
3313  initCompiler(state, &subcompiler, TYPE_FUNCTION);
3314  subcompiler.codeobject->chunk.filename = subcompiler.enclosing->codeobject->chunk.filename;
3315  subcompiler.codeobject->obj.flags |= KRK_OBJ_FLAGS_CODEOBJECT_IS_GENERATOR;
3316 
3317  beginScope(state);
3318  comprehensionInner(state, scannerBefore, parserBefore, body, 0);
3319  endScope(state);
3320 
3321  KrkCodeObject *subfunction = endCompiler(state);
3322  size_t indFunc = krk_addConstant(currentChunk(), OBJECT_VAL(subfunction));
3323  EMIT_OPERAND_OP(OP_CLOSURE, indFunc);
3324  doUpvalues(state, &subcompiler, subfunction);
3325  freeCompiler(&subcompiler);
3326  emitBytes(OP_CALL, 0);
3327 }
3328 
3339 static void comprehensionExpression(struct GlobalState * state, KrkScanner scannerBefore, Parser parserBefore, void (*body)(struct GlobalState *,size_t), int type) {
3340  Compiler subcompiler;
3341  initCompiler(state, &subcompiler, TYPE_LAMBDA);
3342  subcompiler.codeobject->chunk.filename = subcompiler.enclosing->codeobject->chunk.filename;
3343 
3344  beginScope(state);
3345 
3346  /* Build an empty collection to fill up. */
3347  emitBytes(type,0);
3348  size_t ind = anonymousLocal(state);
3349 
3350  beginScope(state);
3351  comprehensionInner(state, scannerBefore, parserBefore, body, ind);
3352  endScope(state);
3353 
3354  KrkCodeObject *subfunction = endCompiler(state);
3355  size_t indFunc = krk_addConstant(currentChunk(), OBJECT_VAL(subfunction));
3356  EMIT_OPERAND_OP(OP_CLOSURE, indFunc);
3357  doUpvalues(state, &subcompiler, subfunction);
3358  freeCompiler(&subcompiler);
3359  emitBytes(OP_CALL, 0);
3360 }
3361 
3362 static size_t finishStarComma(struct GlobalState * state, size_t arg, size_t * argBefore, size_t *argAfter) {
3363  *argBefore = arg;
3364  *argAfter = 1;
3365  EMIT_OPERAND_OP(OP_MAKE_LIST,arg);
3366  parsePrecedence(state, PREC_BITOR);
3367  emitByte(OP_LIST_EXTEND_TOP);
3368 
3369  if (arg == 0 && !check(TOKEN_COMMA)) {
3370  /* While we don't really need to, we disallow a lone @c *expr
3371  * or @c (*expr) without a trailing comma because Python does.
3372  * Catch that here specifically. */
3373  error("* expression not valid here");
3374  return 0;
3375  }
3376 
3377  arg++;
3378 
3379  while (match(TOKEN_COMMA)) {
3380  if (!getRule(state->parser.current.type)->prefix) break;
3381  if (match(TOKEN_ASTERISK)) {
3382  parsePrecedence(state, PREC_BITOR);
3383  emitByte(OP_LIST_EXTEND_TOP);
3384  } else {
3385  parsePrecedence(state, PREC_TERNARY);
3386  emitByte(OP_LIST_APPEND_TOP);
3387  (*argAfter)++;
3388  }
3389  arg++;
3390  }
3391 
3392 
3393  emitByte(OP_TUPLE_FROM_LIST);
3394  return arg;
3395 }
3396 
3404 static void parens(struct GlobalState * state, int exprType, RewindState *rewind) {
3405  /* Record parser state before processing contents. */
3406  ChunkRecorder before = recordChunk(currentChunk());
3407  KrkScanner scannerBefore = krk_tellScanner(&state->scanner);
3408  Parser parserBefore = state->parser;
3409 
3410  /*
3411  * Generator expressions are not valid assignment targets, nor are
3412  * an empty set of parentheses (empty tuple). A single target in
3413  * parens, or a list of targets can be assigned to.
3414  */
3415  int maybeValidAssignment = 0;
3416 
3417  size_t argCount = 0;
3418  size_t argAfter = 0;
3419  size_t argBefore = 0;
3420 
3421  /* Whitespace is ignored inside of parens */
3422  startEatingWhitespace();
3423 
3424  if (check(TOKEN_RIGHT_PAREN)) {
3425  /* Empty paren pair () is an empty tuple. */
3426  emitBytes(OP_TUPLE,0);
3427  } else if (match(TOKEN_ASTERISK)) {
3428  argCount = finishStarComma(state, 0, &argBefore, &argAfter);
3429  maybeValidAssignment = 1;
3430  } else {
3431  parsePrecedence(state, PREC_CAN_ASSIGN);
3432  maybeValidAssignment = 1;
3433  argCount = 1;
3434 
3435  if (match(TOKEN_FOR)) {
3436  /* Parse generator expression. */
3437  maybeValidAssignment = 0;
3438  rewindChunk(currentChunk(), before);
3439  generatorExpression(state, scannerBefore, parserBefore, yieldInner);
3440  } else if (match(TOKEN_COMMA)) {
3441  /* Parse as tuple literal. */
3442  if (!check(TOKEN_RIGHT_PAREN)) {
3443  /* (expr,) is a valid single-element tuple, so we need to check for that. */
3444  do {
3445  if (match(TOKEN_ASTERISK)) {
3446  argCount = finishStarComma(state, argCount, &argBefore, &argAfter);
3447  goto _done;
3448  }
3449  expression(state);
3450  argCount++;
3451  } while (match(TOKEN_COMMA) && !check(TOKEN_RIGHT_PAREN));
3452  }
3453  EMIT_OPERAND_OP(OP_TUPLE, argCount);
3454  }
3455  }
3456 
3457 _done:
3458  stopEatingWhitespace();
3459 
3460  if (!match(TOKEN_RIGHT_PAREN)) {
3461  switch (state->parser.current.type) {
3462  case TOKEN_EQUAL: error("Assignment value expression must be enclosed in parentheses."); break;
3463  default: error("Expected ')' at end of parenthesized expression."); break;
3464  }
3465  }
3466 
3467  if (exprType == EXPR_CAN_ASSIGN && match(TOKEN_EQUAL)) {
3468  if (!argCount) {
3469  error("Can not assign to empty target list.");
3470  } else if (!maybeValidAssignment) {
3471  error("Can not assign to generator expression.");
3472  } else {
3473  complexAssignment(state, before, scannerBefore, parserBefore, argCount, 1, argBefore, argAfter);
3474  }
3475  } else if (exprType == EXPR_ASSIGN_TARGET && (check(TOKEN_EQUAL) || check(TOKEN_COMMA) || check(TOKEN_RIGHT_PAREN))) {
3476  if (!argCount) {
3477  error("Can not assign to empty target list.");
3478  } else if (!maybeValidAssignment) {
3479  error("Can not assign to generator expression.");
3480  } else {
3481  rewindChunk(currentChunk(), before);
3482  complexAssignmentTargets(state, scannerBefore, parserBefore, argCount, 2, argBefore, argAfter);
3483  if (!matchComplexEnd(state)) {
3484  errorAtCurrent("Unexpected end of nested target list");
3485  }
3486  }
3487  }
3488 }
3489 
3495 static void listInner(struct GlobalState * state, size_t arg) {
3496  expression(state);
3497  EMIT_OPERAND_OP(OP_LIST_APPEND, arg);
3498 }
3499 
3500 static void finishStarList(struct GlobalState * state, size_t arg) {
3501  EMIT_OPERAND_OP(OP_MAKE_LIST, arg);
3502 
3503  parsePrecedence(state, PREC_BITOR);
3504  emitByte(OP_LIST_EXTEND_TOP);
3505 
3506  while (match(TOKEN_COMMA) && !check(TOKEN_RIGHT_SQUARE)) {
3507  if (match(TOKEN_ASTERISK)) {
3508  parsePrecedence(state, PREC_BITOR);
3509  emitByte(OP_LIST_EXTEND_TOP);
3510  } else {
3511  expression(state);
3512  emitByte(OP_LIST_APPEND_TOP);
3513  }
3514  }
3515 
3516  stopEatingWhitespace();
3517 
3518  consume(TOKEN_RIGHT_SQUARE,"Expected ']' at end of list expression.");
3519 }
3520 
3526 static void list(struct GlobalState * state, int exprType, RewindState *rewind) {
3527  ChunkRecorder before = recordChunk(currentChunk());
3528 
3529  startEatingWhitespace();
3530 
3531  if (!check(TOKEN_RIGHT_SQUARE)) {
3532  KrkScanner scannerBefore = krk_tellScanner(&state->scanner);
3533  Parser parserBefore = state->parser;
3534  if (match(TOKEN_ASTERISK)) {
3535  finishStarList(state, 0);
3536  return;
3537  }
3538  expression(state);
3539 
3540  if (match(TOKEN_FOR)) {
3541  /* Roll back the earlier compiler */
3542  rewindChunk(currentChunk(), before);
3543  /* Nested fun times */
3544  state->parser.previous = syntheticToken("<listcomp>");
3545  comprehensionExpression(state, scannerBefore, parserBefore, listInner, OP_MAKE_LIST);
3546  } else {
3547  size_t argCount = 1;
3548  while (match(TOKEN_COMMA) && !check(TOKEN_RIGHT_SQUARE)) {
3549  if (match(TOKEN_ASTERISK)) {
3550  finishStarList(state, argCount);
3551  return;
3552  }
3553  expression(state);
3554  argCount++;
3555  }
3556  EMIT_OPERAND_OP(OP_MAKE_LIST, argCount);
3557  }
3558  } else {
3559  /* Empty list expression */
3560  EMIT_OPERAND_OP(OP_MAKE_LIST, 0);
3561  }
3562 
3563  stopEatingWhitespace();
3564 
3565  consume(TOKEN_RIGHT_SQUARE,"Expected ']' at end of list expression.");
3566 }
3567 
3571 static void dictInner(struct GlobalState * state, size_t arg) {
3572  expression(state); /* Key */
3573  consume(TOKEN_COLON, "Expected ':' after dict key.");
3574  expression(state); /* Value */
3575  EMIT_OPERAND_OP(OP_DICT_SET, arg);
3576 }
3577 
3581 static void setInner(struct GlobalState * state, size_t arg) {
3582  expression(state);
3583  EMIT_OPERAND_OP(OP_SET_ADD, arg);
3584 }
3585 
3586 static void finishStarSet(struct GlobalState * state, size_t args) {
3587  EMIT_OPERAND_OP(OP_MAKE_SET, args);
3588 
3589  parsePrecedence(state, PREC_BITOR);
3590  emitByte(OP_SET_UPDATE_TOP);
3591 
3592  while (match(TOKEN_COMMA) && !check(TOKEN_RIGHT_BRACE)) {
3593  if (match(TOKEN_ASTERISK)) {
3594  parsePrecedence(state, PREC_BITOR);
3595  emitByte(OP_SET_UPDATE_TOP);
3596  } else {
3597  expression(state);
3598  emitByte(OP_SET_ADD_TOP);
3599  }
3600  }
3601 
3602  stopEatingWhitespace();
3603  consume(TOKEN_RIGHT_BRACE,"Expected '}' at end of dict expression.");
3604 }
3605 
3606 static void finishStarDict(struct GlobalState * state, size_t args) {
3607  EMIT_OPERAND_OP(OP_MAKE_DICT, args);
3608 
3609  parsePrecedence(state, PREC_BITOR);
3610  emitByte(OP_DICT_UPDATE_TOP);
3611 
3612  while (match(TOKEN_COMMA) && !check(TOKEN_RIGHT_BRACE)) {
3613  if (match(TOKEN_POW)) {
3614  parsePrecedence(state, PREC_BITOR);
3615  emitByte(OP_DICT_UPDATE_TOP);
3616  } else {
3617  expression(state);
3618  consume(TOKEN_COLON, "Expected ':' after dict key.");
3619  expression(state);
3620  emitByte(OP_DICT_SET_TOP);
3621  }
3622  }
3623 
3624  stopEatingWhitespace();
3625  consume(TOKEN_RIGHT_BRACE,"Expected '}' at end of dict expression.");
3626 }
3627 
3639 static void dict(struct GlobalState * state, int exprType, RewindState *rewind) {
3640  ChunkRecorder before = recordChunk(currentChunk());
3641 
3642  startEatingWhitespace();
3643 
3644  if (!check(TOKEN_RIGHT_BRACE)) {
3645  KrkScanner scannerBefore = krk_tellScanner(&state->scanner);
3646  Parser parserBefore = state->parser;
3647 
3648  if (match(TOKEN_ASTERISK)) {
3649  finishStarSet(state, 0);
3650  return;
3651  } else if (match(TOKEN_POW)) {
3652  finishStarDict(state, 0);
3653  return;
3654  }
3655 
3656  expression(state);
3657  if (check(TOKEN_COMMA) || check(TOKEN_RIGHT_BRACE)) {
3658  /* One expression, must be a set literal. */
3659  size_t argCount = 1;
3660  while (match(TOKEN_COMMA) && !check(TOKEN_RIGHT_BRACE)) {
3661  if (match(TOKEN_ASTERISK)) {
3662  finishStarSet(state, argCount);
3663  return;
3664  }
3665  expression(state);
3666  argCount++;
3667  }
3668  EMIT_OPERAND_OP(OP_MAKE_SET, argCount);
3669  } else if (match(TOKEN_FOR)) {
3670  /* One expression followed by 'for': set comprehension. */
3671  rewindChunk(currentChunk(), before);
3672  state->parser.previous = syntheticToken("<setcomp>");
3673  comprehensionExpression(state, scannerBefore, parserBefore, setInner, OP_MAKE_SET);
3674  } else {
3675  /* Anything else must be a colon indicating a dictionary. */
3676  consume(TOKEN_COLON, "Expected ':' after dict key.");
3677  expression(state);
3678 
3679  if (match(TOKEN_FOR)) {
3680  /* Dictionary comprehension */
3681  rewindChunk(currentChunk(), before);
3682  state->parser.previous = syntheticToken("<dictcomp>");
3683  comprehensionExpression(state, scannerBefore, parserBefore, dictInner, OP_MAKE_DICT);
3684  } else {
3685  /*
3686  * The operand to MAKE_DICT is double the number of entries,
3687  * as it is the number of stack slots to consume to produce
3688  * the dict: one for each key, one for each value.
3689  */
3690  size_t argCount = 2;
3691  while (match(TOKEN_COMMA) && !check(TOKEN_RIGHT_BRACE)) {
3692  if (match(TOKEN_POW)) {
3693  finishStarDict(state, argCount);
3694  return;
3695  }
3696  expression(state);
3697  consume(TOKEN_COLON, "Expected ':' after dict key.");
3698  expression(state);
3699  argCount += 2;
3700  }
3701  EMIT_OPERAND_OP(OP_MAKE_DICT, argCount);
3702  }
3703  }
3704  } else {
3705  /* Empty braces, empty dictionary. */
3706  EMIT_OPERAND_OP(OP_MAKE_DICT, 0);
3707  }
3708 
3709  stopEatingWhitespace();
3710 
3711  consume(TOKEN_RIGHT_BRACE,"Expected '}' at end of dict expression.");
3712 }
3713 
3714 static void ternary(struct GlobalState * state, int exprType, RewindState *rewind) {
3715  Parser before = state->parser;
3716  rewindChunk(currentChunk(), rewind->before);
3717 
3718  parsePrecedence(state, PREC_OR);
3719 
3720  int thenJump = emitJump(OP_JUMP_IF_TRUE_OR_POP);
3721  consume(TOKEN_ELSE, "Expected 'else' after ternary condition");
3722 
3723  parsePrecedence(state, PREC_TERNARY);
3724 
3725  KrkScanner outScanner = krk_tellScanner(&state->scanner);
3726  Parser outParser = state->parser;
3727 
3728  int elseJump = emitJump(OP_JUMP);
3729  patchJump(thenJump);
3730  emitByte(OP_POP);
3731 
3732  krk_rewindScanner(&state->scanner, rewind->oldScanner);
3733  state->parser = rewind->oldParser;
3734  parsePrecedence(state, PREC_OR);
3735  patchJump(elseJump);
3736 
3737  if (!check(TOKEN_IF)) {
3738  state->parser = before;
3739  error("syntax error");
3740  }
3741 
3742  krk_rewindScanner(&state->scanner, outScanner);
3743  state->parser = outParser;
3744 }
3745 
3746 static void complexAssignmentTargets(struct GlobalState * state, KrkScanner oldScanner, Parser oldParser, size_t targetCount, int parenthesized, size_t argBefore, size_t argAfter) {
3747  emitBytes(OP_DUP, 0);
3748 
3749  if (argAfter) {
3750  if (argBefore > 255 || argAfter > 256) {
3751  error("Too many assignment targets");
3752  return;
3753  }
3754  EMIT_OPERAND_OP(OP_UNPACK_EX,((argBefore << 8) | (argAfter-1)));
3755  } else {
3756  EMIT_OPERAND_OP(OP_UNPACK,targetCount);
3757  }
3758  EMIT_OPERAND_OP(OP_REVERSE,targetCount);
3759 
3760  /* Rewind */
3761  krk_rewindScanner(&state->scanner, oldScanner);
3762  state->parser = oldParser;
3763 
3764  /* Parse assignment targets */
3765  size_t checkTargetCount = 0;
3766  int seenStar = 0;
3767  do {
3768  checkTargetCount++;
3769  if (match(TOKEN_ASTERISK)) {
3770  if (seenStar) {
3771  errorAtCurrent("multiple *expr in assignment");
3772  return;
3773  }
3774  seenStar = 1;
3775  }
3776  parsePrecedence(state, PREC_MUST_ASSIGN);
3777  emitByte(OP_POP);
3778 
3779  if (checkTargetCount == targetCount && state->parser.previous.type == TOKEN_COMMA) {
3780  if (!match(parenthesized ? TOKEN_RIGHT_PAREN : TOKEN_EQUAL)) {
3781  goto _errorAtCurrent;
3782  }
3783  }
3784 
3785  if (checkTargetCount == targetCount && parenthesized) {
3786  if (state->parser.previous.type != TOKEN_RIGHT_PAREN) {
3787  goto _errorAtCurrent;
3788  }
3789  }
3790 
3791  if (checkTargetCount == targetCount && parenthesized) {
3792  if (parenthesized == 1 && !match(TOKEN_EQUAL)) {
3793  goto _errorAtCurrent;
3794  }
3795  }
3796 
3797  if (checkTargetCount == targetCount) return;
3798 
3799  if (check(TOKEN_COMMA) || check(TOKEN_EQUAL) || check(TOKEN_RIGHT_PAREN)) {
3800  goto _errorAtCurrent;
3801  }
3802 
3803  } while (state->parser.previous.type != TOKEN_EQUAL && !state->parser.hadError);
3804 
3805 _errorAtCurrent:
3806  errorAtCurrent("Invalid complex assignment target");
3807 }
3808 
3809 static void complexAssignment(struct GlobalState * state, ChunkRecorder before, KrkScanner oldScanner, Parser oldParser, size_t targetCount, int parenthesized, size_t argBefore, size_t argAfter) {
3810 
3811  rewindChunk(currentChunk(), before);
3812  parsePrecedence(state, PREC_ASSIGNMENT);
3813 
3814  /* Store end state */
3815  KrkScanner outScanner = krk_tellScanner(&state->scanner);
3816  Parser outParser = state->parser;
3817 
3818  complexAssignmentTargets(state, oldScanner,oldParser,targetCount,parenthesized, argBefore, argAfter);
3819 
3820  /* Restore end state */
3821  krk_rewindScanner(&state->scanner, outScanner);
3822  state->parser = outParser;
3823 }
3824 
3825 static void comma(struct GlobalState * state, int exprType, RewindState *rewind) {
3826  size_t expressionCount = 1;
3827  size_t argBefore = 0;
3828  size_t argAfter = 0;
3829  do {
3830  if (!getRule(state->parser.current.type)->prefix) break;
3831  if (match(TOKEN_ASTERISK)) {
3832  expressionCount = finishStarComma(state, expressionCount, &argBefore, &argAfter);
3833  goto _maybeassign;
3834  return;
3835  }
3836  expressionCount++;
3837  parsePrecedence(state, PREC_TERNARY);
3838  } while (match(TOKEN_COMMA));
3839 
3840  EMIT_OPERAND_OP(OP_TUPLE,expressionCount);
3841 
3842 _maybeassign: (void)0;
3843  if (exprType == EXPR_CAN_ASSIGN && match(TOKEN_EQUAL)) {
3844  complexAssignment(state, rewind->before, rewind->oldScanner, rewind->oldParser, expressionCount, 0, argBefore, argAfter);
3845  }
3846 }
3847 
3848 static void pstar(struct GlobalState * state, int exprType, RewindState *rewind) {
3849  size_t argBefore = 0;
3850  size_t argAfter = 0;
3851  size_t totalArgs = finishStarComma(state,0, &argBefore, &argAfter);
3852  if (exprType == EXPR_CAN_ASSIGN && match(TOKEN_EQUAL)) {
3853  complexAssignment(state, rewind->before, rewind->oldScanner, rewind->oldParser, totalArgs, 0, argBefore, argAfter);
3854  }
3855 }
3856 
3857 static void call(struct GlobalState * state, int exprType, RewindState *rewind) {
3858  KrkToken left = rewind ? rewind->oldParser.current : state->parser.previous;
3859  KrkToken this = state->parser.previous;
3860  startEatingWhitespace();
3861  size_t argCount = 0, specialArgs = 0, keywordArgs = 0, seenKeywordUnpacking = 0;
3862  if (!check(TOKEN_RIGHT_PAREN)) {
3863  size_t chunkBefore = currentChunk()->count;
3864  KrkScanner scannerBefore = krk_tellScanner(&state->scanner);
3865  Parser parserBefore = state->parser;
3866  do {
3867  if (check(TOKEN_RIGHT_PAREN)) break;
3868  if (match(TOKEN_ASTERISK) || check(TOKEN_POW)) {
3869  specialArgs++;
3870  if (match(TOKEN_POW)) {
3871  seenKeywordUnpacking = 1;
3872  emitBytes(OP_EXPAND_ARGS, 2); /* creates a KWARGS_DICT */
3873  expression(state); /* Expect dict */
3874  continue;
3875  } else {
3876  if (seenKeywordUnpacking) {
3877  error("Iterable expansion follows keyword argument unpacking.");
3878  return;
3879  }
3880  emitBytes(OP_EXPAND_ARGS, 1); /* creates a KWARGS_LIST */
3881  expression(state);
3882  continue;
3883  }
3884  }
3885  if (match(TOKEN_IDENTIFIER)) {
3886  KrkToken argName = state->parser.previous;
3887  if (check(TOKEN_EQUAL)) {
3888  /* This is a keyword argument. */
3889  advance();
3890  /* Output the name */
3891  size_t ind = identifierConstant(state, &argName);
3892  EMIT_OPERAND_OP(OP_CONSTANT, ind);
3893  expression(state);
3894  keywordArgs++;
3895  specialArgs++;
3896  continue;
3897  } else {
3898  /*
3899  * This is a regular argument that happened to start with an identifier,
3900  * roll it back so we can process it that way.
3901  */
3902  krk_ungetToken(&state->scanner, state->parser.current);
3903  state->parser.current = argName;
3904  }
3905  } else if (seenKeywordUnpacking) {
3906  error("Positional argument follows keyword argument unpacking");
3907  return;
3908  } else if (keywordArgs) {
3909  error("Positional argument follows keyword argument");
3910  return;
3911  }
3912  if (specialArgs) {
3913  emitBytes(OP_EXPAND_ARGS, 0); /* creates a KWARGS_SINGLE */
3914  expression(state);
3915  specialArgs++;
3916  continue;
3917  }
3918  expression(state);
3919  if (argCount == 0 && match(TOKEN_FOR)) {
3920  currentChunk()->count = chunkBefore;
3921  generatorExpression(state, scannerBefore, parserBefore, yieldInner);
3922  argCount = 1;
3923  if (match(TOKEN_COMMA)) {
3924  error("Generator expression must be parenthesized");
3925  return;
3926  }
3927  break;
3928  }
3929  argCount++;
3930  } while (match(TOKEN_COMMA));
3931  }
3932  stopEatingWhitespace();
3933  consume(TOKEN_RIGHT_PAREN, "Expected ')' after arguments.");
3934  if (specialArgs) {
3935  /*
3936  * Creates a sentinel at the top of the stack to tell the CALL instruction
3937  * how many keyword arguments are at the top of the stack. This value
3938  * triggers special handling in the CALL that processes the keyword arguments,
3939  * which is relatively slow, so only use keyword arguments if you have to!
3940  */
3941  EMIT_OPERAND_OP(OP_KWARGS, specialArgs);
3942  /*
3943  * We added two elements - name and value - for each keyword arg,
3944  * plus the sentinel object that will show up at the end after the
3945  * OP_KWARGS instruction complets, so make sure we have the
3946  * right depth into the stack when we execute CALL
3947  */
3948  argCount += 1 /* for the sentinel */ + 2 * specialArgs;
3949  }
3950 
3951  if (exprType == EXPR_METHOD_CALL) {
3952  EMIT_OPERAND_OP(OP_CALL_METHOD, argCount);
3953  } else if (exprType == EXPR_CLASS_PARAMETERS) {
3954  EMIT_OPERAND_OP(OP_CALL, (argCount + 2));
3955  } else {
3956  EMIT_OPERAND_OP(OP_CALL, argCount);
3957  }
3958  writeExpressionLocation(&left,&state->parser.previous,&this,state);
3959 
3960  invalidTarget(state, exprType, "function call");
3961 }
3962 
3963 static void and_(struct GlobalState * state, int exprType, RewindState *rewind) {
3964  int endJump = emitJump(OP_JUMP_IF_FALSE_OR_POP);
3965  parsePrecedence(state, PREC_AND);
3966  patchJump(endJump);
3967  invalidTarget(state, exprType, "operator");
3968 }
3969 
3970 static void or_(struct GlobalState * state, int exprType, RewindState *rewind) {
3971  int endJump = emitJump(OP_JUMP_IF_TRUE_OR_POP);
3972  parsePrecedence(state, PREC_OR);
3973  patchJump(endJump);
3974  invalidTarget(state, exprType, "operator");
3975 }
3976 
3977 static void parsePrecedence(struct GlobalState * state, Precedence precedence) {
3978  RewindState rewind = {recordChunk(currentChunk()), krk_tellScanner(&state->scanner), state->parser};
3979 
3980  advance();
3981  ParseFn prefixRule = getRule(state->parser.previous.type)->prefix;
3982 
3983  /* Only allow *expr parsing where we would allow comma expressions,
3984  * otherwise pretend this prefix rule doesn't exist. */
3985  if (prefixRule == pstar && precedence > PREC_COMMA) prefixRule = NULL;
3986 
3987  if (prefixRule == NULL) {
3988  switch (state->parser.previous.type) {
3989  case TOKEN_RIGHT_BRACE:
3990  case TOKEN_RIGHT_PAREN:
3991  case TOKEN_RIGHT_SQUARE:
3992  error("Unmatched '%.*s'",
3993  (int)state->parser.previous.length, state->parser.previous.start);
3994  break;
3995  case TOKEN_EOL:
3996  /* TODO: This should definitely be tripping the REPL to ask for more input. */
3997  error("Unexpected end of line");
3998  break;
3999  case TOKEN_EOF:
4000  error("Unexpected end of input");
4001  break;
4002  default:
4003  error("'%.*s' does not start an expression",
4004  (int)state->parser.previous.length, state->parser.previous.start);
4005  }
4006  return;
4007  }
4008  int exprType = 0;
4009  if (precedence <= PREC_ASSIGNMENT || precedence == PREC_CAN_ASSIGN) exprType = EXPR_CAN_ASSIGN;
4010  if (precedence == PREC_MUST_ASSIGN) exprType = EXPR_ASSIGN_TARGET;
4011  if (precedence == PREC_DEL_TARGET) exprType = EXPR_DEL_TARGET;
4012  prefixRule(state, exprType, &rewind);
4013  while (precedence <= getRule(state->parser.current.type)->precedence) {
4014  if (state->parser.hadError) {
4015  skipToEnd();
4016  return;
4017  }
4018 
4019  if (exprType == EXPR_ASSIGN_TARGET && (state->parser.previous.type == TOKEN_COMMA ||
4020  state->parser.previous.type == TOKEN_EQUAL)) break;
4021  advance();
4022  ParseFn infixRule = getRule(state->parser.previous.type)->infix;
4023  infixRule(state, exprType, &rewind);
4024  }
4025 
4026  if (exprType == EXPR_CAN_ASSIGN && matchAssignment(state)) {
4027  error("Invalid assignment target");
4028  }
4029 }
4030 
4031 
4032 static int maybeSingleExpression(struct GlobalState * state) {
4033  /* We're only going to use this to reset if we found a string, and it turns out
4034  * not to be a docstring. */
4035  RewindState rewind = {recordChunk(currentChunk()), krk_tellScanner(&state->scanner), state->parser};
4036 
4044  if (check(TOKEN_STRING) || check(TOKEN_BIG_STRING)) {
4045  advance();
4046  if (match(TOKEN_EOL)) {
4047  /* We found just a string on the first line, but is it the only line?
4048  * We should treat that as a regular string expression, eg. in a repl. */
4049  int isEof = check(TOKEN_EOF);
4050  /* Regardless, restore the scanner/parser so we can actually parse the string. */
4051  krk_rewindScanner(&state->scanner, rewind.oldScanner);
4052  state->parser = rewind.oldParser;
4053  advance();
4054  /* Parse the string. */
4055  string(state, EXPR_NORMAL, NULL);
4056  /* If we did see end of input, it's a simple string expression. */
4057  if (isEof) return 1;
4058  /* Otherwise, it's a docstring, and there's more code following it.
4059  * Emit the instructions to assign the docstring to the current globals. */
4060  KrkToken doc = syntheticToken("__doc__");
4061  size_t ind = identifierConstant(state, &doc);
4062  EMIT_OPERAND_OP(OP_DEFINE_GLOBAL, ind);
4063  return 0;
4064  } else {
4065  /* There was something other than a line feed after the string token,
4066  * rewind so we can parse as an expression next. */
4067  krk_rewindScanner(&state->scanner, rewind.oldScanner);
4068  state->parser = rewind.oldParser;
4069  }
4070  }
4071 
4072  /* Try to parse one single expression */
4073  ParseRule * rule = getRule(state->parser.current.type);
4074  if (rule->prefix) {
4075  parsePrecedence(state, PREC_ASSIGNMENT);
4076 
4077  /* Semicolon after expression statement, finish this one and continue
4078  * parsing only more simple statements, as we would normally. */
4079  if (match(TOKEN_SEMICOLON)) {
4080  emitByte(OP_POP);
4081  simpleStatement(state);
4082  return 0;
4083  }
4084 
4085  /* Expression statement that isn't the end of the input, finish it
4086  * and let the declaration loop handle the rest. */
4087  if (match(TOKEN_EOL) && !check(TOKEN_EOF)) {
4088  emitByte(OP_POP);
4089  return 0;
4090  }
4091 
4092  /* End of input after expression, must be just the single expression;
4093  * using check rather than match makes the return emit on the same
4094  * line, which produces cleaner disassembly when -d tracing is enabled. */
4095  if (check(TOKEN_EOF))
4096  return 1;
4097 
4098  /* Must be an error. */
4099  errorAfterStatement(state);
4100  return 0;
4101  }
4102 
4103  return 0;
4104 }
4105 
4106 _noexport
4107 void _createAndBind_compilerClass(void) {
4108  KrkClass * CompilerState = ADD_BASE_CLASS(KRK_BASE_CLASS(CompilerState), "CompilerState", KRK_BASE_CLASS(object));
4109  CompilerState->allocSize = sizeof(struct GlobalState);
4110  CompilerState->_ongcscan = _GlobalState_gcscan;
4111  CompilerState->_ongcsweep = _GlobalState_gcsweep;
4112  CompilerState->obj.flags |= KRK_OBJ_FLAGS_NO_INHERIT;
4113  krk_finalizeClass(CompilerState);
4114 }
4115 
4129 KrkCodeObject * krk_compile(const char * src, const char * fileName) {
4130  struct GlobalState * state = (void*)krk_newInstance(KRK_BASE_CLASS(CompilerState));
4131  krk_push(OBJECT_VAL(state));
4132 
4133  /* Point a new scanner at the source. */
4134  state->scanner = krk_initScanner(src);
4135 
4136  /* Reset parser state. */
4137  memset(&state->parser, 0, sizeof(state->parser));
4138 
4139  /* Start compiling a new function. */
4140  Compiler compiler;
4141  initCompiler(state, &compiler, TYPE_MODULE);
4142  compiler.codeobject->chunk.filename = krk_copyString(fileName, strlen(fileName));
4143  compiler.codeobject->name = krk_copyString("<module>",8);
4144 
4145  /* Start reading tokens from the scanner... */
4146  advance();
4147 
4148  /* The first line of an input may be a doc string. */
4149  if (maybeSingleExpression(state)) {
4150  state->current->type = TYPE_LAMBDA;
4151  } else {
4152  /* Parse top-level declarations... */
4153  while (!match(TOKEN_EOF)) {
4154  declaration(state);
4155 
4156  /* Skip over redundant whitespace */
4157  if (check(TOKEN_EOL) || check(TOKEN_INDENTATION) || check(TOKEN_EOF)) {
4158  advance();
4159  }
4160  }
4161  }
4162 
4163  KrkCodeObject * function = endCompiler(state);
4164  freeCompiler(&compiler);
4165 
4166  /*
4167  * We'll always get something out of endCompiler even if it
4168  * wasn't fully compiled, so be sure to check for a syntax
4169  * error and return NULL
4170  */
4171  if (state->parser.hadError) function = NULL;
4172 
4173  krk_pop();
4174  return function;
4175 }
4176 
4177 #define RULE(token, a, b, c) [TOKEN_ ## token] = {a, b, c}
4178 
4190  RULE(DOT, NULL, dot, PREC_PRIMARY),
4191  RULE(LEFT_PAREN, parens, call, PREC_PRIMARY),
4192  RULE(LEFT_SQUARE, list, getitem, PREC_PRIMARY),
4193  RULE(LEFT_BRACE, dict, NULL, PREC_NONE),
4194  RULE(RIGHT_PAREN, NULL, NULL, PREC_NONE),
4195  RULE(RIGHT_SQUARE, NULL, NULL, PREC_NONE),
4196  RULE(RIGHT_BRACE, NULL, NULL, PREC_NONE),
4197  RULE(COLON, NULL, NULL, PREC_NONE),
4198  RULE(SEMICOLON, NULL, NULL, PREC_NONE),
4199  RULE(EQUAL, NULL, NULL, PREC_NONE),
4200  RULE(WALRUS, NULL, NULL, PREC_NONE),
4201  RULE(PLUS_EQUAL, NULL, NULL, PREC_NONE),
4202  RULE(MINUS_EQUAL, NULL, NULL, PREC_NONE),
4203  RULE(PLUS_PLUS, NULL, NULL, PREC_NONE),
4204  RULE(MINUS_MINUS, NULL, NULL, PREC_NONE),
4205  RULE(CARET_EQUAL, NULL, NULL, PREC_NONE),
4206  RULE(PIPE_EQUAL, NULL, NULL, PREC_NONE),
4207  RULE(LSHIFT_EQUAL, NULL, NULL, PREC_NONE),
4208  RULE(RSHIFT_EQUAL, NULL, NULL, PREC_NONE),
4209  RULE(AMP_EQUAL, NULL, NULL, PREC_NONE),
4210  RULE(SOLIDUS_EQUAL, NULL, NULL, PREC_NONE),
4211  RULE(DSOLIDUS_EQUAL,NULL, NULL, PREC_NONE),
4212  RULE(ASTERISK_EQUAL,NULL, NULL, PREC_NONE),
4213  RULE(MODULO_EQUAL, NULL, NULL, PREC_NONE),
4214  RULE(AT_EQUAL, NULL, NULL, PREC_NONE),
4215  RULE(POW_EQUAL, NULL, NULL, PREC_NONE),
4216  RULE(ARROW, NULL, NULL, PREC_NONE),
4217  RULE(MINUS, unary, binary, PREC_SUM),
4218  RULE(PLUS, unary, binary, PREC_SUM),
4219  RULE(TILDE, unary, NULL, PREC_NONE),
4220  RULE(BANG, unary, NULL, PREC_NONE),
4221  RULE(SOLIDUS, NULL, binary, PREC_TERM),
4222  RULE(DOUBLE_SOLIDUS,NULL, binary, PREC_TERM),
4223  RULE(ASTERISK, pstar, binary, PREC_TERM),
4224  RULE(MODULO, NULL, binary, PREC_TERM),
4225  RULE(AT, NULL, binary, PREC_TERM),
4226  RULE(POW, NULL, binary, PREC_EXPONENT),
4227  RULE(PIPE, NULL, binary, PREC_BITOR),
4228  RULE(CARET, NULL, binary, PREC_BITXOR),
4229  RULE(AMPERSAND, NULL, binary, PREC_BITAND),
4230  RULE(LEFT_SHIFT, NULL, binary, PREC_SHIFT),
4231  RULE(RIGHT_SHIFT, NULL, binary, PREC_SHIFT),
4232  RULE(BANG_EQUAL, NULL, compare, PREC_COMPARISON),
4233  RULE(EQUAL_EQUAL, NULL, compare, PREC_COMPARISON),
4234  RULE(GREATER, NULL, compare, PREC_COMPARISON),
4235  RULE(GREATER_EQUAL, NULL, compare, PREC_COMPARISON),
4236  RULE(LESS, NULL, compare, PREC_COMPARISON),
4237  RULE(LESS_EQUAL, NULL, compare, PREC_COMPARISON),
4238  RULE(IN, NULL, compare, PREC_COMPARISON),
4239  RULE(IS, NULL, compare, PREC_COMPARISON),
4240  RULE(NOT, unot_, compare, PREC_COMPARISON),
4241  RULE(IDENTIFIER, variable, NULL, PREC_NONE),
4242  RULE(STRING, string, NULL, PREC_NONE),
4243  RULE(BIG_STRING, string, NULL, PREC_NONE),
4244  RULE(PREFIX_B, string, NULL, PREC_NONE),
4245  RULE(PREFIX_F, string, NULL, PREC_NONE),
4246  RULE(PREFIX_R, string, NULL, PREC_NONE),
4247  RULE(NUMBER, number, NULL, PREC_NONE),
4248  RULE(AND, NULL, and_, PREC_AND),
4249  RULE(OR, NULL, or_, PREC_OR),
4250  RULE(FALSE, literal, NULL, PREC_NONE),
4251  RULE(NONE, literal, NULL, PREC_NONE),
4252  RULE(TRUE, literal, NULL, PREC_NONE),
4253  RULE(ELLIPSIS, ellipsis, NULL, PREC_NONE),
4254  RULE(YIELD, yield, NULL, PREC_NONE),
4255  RULE(AWAIT, await, NULL, PREC_NONE),
4256  RULE(LAMBDA, lambda, NULL, PREC_NONE),
4257  RULE(SUPER, super_, NULL, PREC_NONE),
4258  RULE(CLASS, NULL, NULL, PREC_NONE),
4259  RULE(ELSE, NULL, NULL, PREC_NONE),
4260  RULE(FOR, NULL, NULL, PREC_NONE),
4261  RULE(DEF, NULL, NULL, PREC_NONE),
4262  RULE(DEL, NULL, NULL, PREC_NONE),
4263  RULE(LET, NULL, NULL, PREC_NONE),
4264  RULE(RETURN, NULL, NULL, PREC_NONE),
4265  RULE(WHILE, NULL, NULL, PREC_NONE),
4266  RULE(BREAK, NULL, NULL, PREC_NONE),
4267  RULE(CONTINUE, NULL, NULL, PREC_NONE),
4268  RULE(IMPORT, NULL, NULL, PREC_NONE),
4269  RULE(RAISE, NULL, NULL, PREC_NONE),
4270  RULE(ASYNC, NULL, NULL, PREC_NONE),
4271  RULE(PASS, NULL, NULL, PREC_NONE),
4272  RULE(ASSERT, NULL, NULL, PREC_NONE),
4273  RULE(FINALLY, NULL, NULL, PREC_NONE),
4274  RULE(ELIF, NULL, NULL, PREC_NONE),
4275  RULE(TRY, NULL, NULL, PREC_NONE),
4276  RULE(EXCEPT, NULL, NULL, PREC_NONE),
4277  RULE(AS, NULL, NULL, PREC_NONE),
4278  RULE(FROM, NULL, NULL, PREC_NONE),
4279  RULE(WITH, NULL, NULL, PREC_NONE),
4280 
4281  RULE(COMMA, NULL, comma, PREC_COMMA),
4282  RULE(IF, NULL, ternary, PREC_TERNARY),
4283 
4284  RULE(INDENTATION, NULL, NULL, PREC_NONE),
4285  RULE(ERROR, NULL, NULL, PREC_NONE),
4286  RULE(EOL, NULL, NULL, PREC_NONE),
4287  RULE(EOF, NULL, NULL, PREC_NONE),
4288  RULE(RETRY, NULL, NULL, PREC_NONE),
4289 };
4290 
4291 static ParseRule * getRule(KrkTokenType type) {
4292  return &krk_parseRules[type];
4293 }
4294 
ParseRule krk_parseRules[]
Parse rules table.
Definition: compiler.c:4189
FunctionType
Function compilation type.
Definition: compiler.c:160
@ TYPE_COROUTINE_METHOD
Definition: compiler.c:170
@ TYPE_INIT
Definition: compiler.c:164
@ TYPE_COROUTINE
Definition: compiler.c:169
@ TYPE_MODULE
Definition: compiler.c:162
@ TYPE_STATIC
Definition: compiler.c:166
@ TYPE_CLASSMETHOD
Definition: compiler.c:168
@ TYPE_LAMBDA
Definition: compiler.c:165
@ TYPE_METHOD
Definition: compiler.c:163
@ TYPE_CLASS
Definition: compiler.c:167
@ TYPE_FUNCTION
Definition: compiler.c:161
ExpressionType
Expression type.
Definition: compiler.c:97
@ EXPR_METHOD_CALL
Definition: compiler.c:102
@ EXPR_NORMAL
Definition: compiler.c:98
@ EXPR_ASSIGN_TARGET
Definition: compiler.c:100
@ EXPR_DEL_TARGET
Definition: compiler.c:101
@ EXPR_CAN_ASSIGN
Definition: compiler.c:99
struct Compiler Compiler
Subcompiler state.
KrkCodeObject * krk_compile(const char *src, const char *fileName)
Compile a source string to bytecode.
Definition: compiler.c:4129
struct RewindState RewindState
Compiler emit and parse state prior to this expression.
Precedence
Parse precedence ladder.
Definition: compiler.c:69
@ PREC_TERNARY
Definition: compiler.c:76
@ PREC_CAN_ASSIGN
Definition: compiler.c:74
@ PREC_BITXOR
Definition: compiler.c:82
@ PREC_SUM
Definition: compiler.c:85
@ PREC_DEL_TARGET
Definition: compiler.c:75
@ PREC_SHIFT
Definition: compiler.c:84
@ PREC_OR
Definition: compiler.c:77
@ PREC_FACTOR
Definition: compiler.c:87
@ PREC_ASSIGNMENT
Definition: compiler.c:71
@ PREC_PRIMARY
Definition: compiler.c:89
@ PREC_EXPONENT
Definition: compiler.c:88
@ PREC_MUST_ASSIGN
Definition: compiler.c:73
@ PREC_COMPARISON
Definition: compiler.c:80
@ PREC_COMMA
Definition: compiler.c:72
@ PREC_NOT
Definition: compiler.c:79
@ PREC_BITOR
Definition: compiler.c:81
@ PREC_BITAND
Definition: compiler.c:83
@ PREC_TERM
Definition: compiler.c:86
@ PREC_AND
Definition: compiler.c:78
struct ClassCompiler ClassCompiler
Class compilation context.
void(* ParseFn)(struct GlobalState *, int, struct RewindState *)
Subexpression parser function.
Definition: compiler.c:116
struct ChunkRecorder ChunkRecorder
Bytecode emitter backtracking breadcrumb.
Exported methods for the source compiler.
Functions for debugging bytecode execution.
void krk_disassembleCodeObject(FILE *f, KrkCodeObject *func, const char *name)
Print a disassembly of 'func' to the stream 'f'.
Definition: debug.c:163
Top-level header with configuration macros.
Functions for dealing with garbage collection and memory allocation.
void krk_markObject(KrkObj *object)
During a GC scan cycle, mark an object as used.
Definition: memory.c:321
KrkValue krk_parse_float(const char *s, size_t l)
Parse a string into a float.
Definition: obj_long.c:2820
Struct definitions for core object types.
Internal header.
Definitions used by the token scanner.
void krk_ungetToken(KrkScanner *, KrkToken token)
Push a token back to the scanner to be reprocessed.
Definition: scanner.c:285
KrkToken krk_scanToken(KrkScanner *)
Read the next token from the scanner.
Definition: scanner.c:301
KrkScanner krk_initScanner(const char *src)
Initialize the compiler to scan tokens from 'src'.
Definition: scanner.c:11
KrkScanner krk_tellScanner(KrkScanner *)
Retreive a copy of the current scanner state.
Definition: scanner.c:293
void krk_rewindScanner(KrkScanner *, KrkScanner to)
Rewind the scanner to a previous state.
Definition: scanner.c:297
Bytecode emitter backtracking breadcrumb.
Definition: compiler.c:258
size_t count
Offset into the bytecode.
Definition: compiler.c:259
size_t constants
Number of constants in the constants table.
Definition: compiler.c:261
size_t lines
Offset into the line map.
Definition: compiler.c:260
size_t expressions
Number of expression mapping entries.
Definition: compiler.c:262
Class compilation context.
Definition: compiler.c:240
struct ClassCompiler * enclosing
Enclosing class scope.
Definition: compiler.c:241
int hasAnnotations
Flag indicating if an annotation dictionary has been attached to this class.
Definition: compiler.c:243
KrkToken name
Name of the current class.
Definition: compiler.c:242
Subcompiler state.
Definition: compiler.c:199
size_t continueCount
Number of continue statements.
Definition: compiler.c:214
size_t localNameCapacity
How much space is available in the codeobject's local names table.
Definition: compiler.c:218
struct IndexWithNext * properties
Linked list of class property constant indices.
Definition: compiler.c:220
FunctionType type
Type of function being compiled.
Definition: compiler.c:202
size_t continueSpace
Space in continues array.
Definition: compiler.c:215
size_t optionsFlags
Special options imports; similar to future in Python.
Definition: compiler.c:226
size_t localsSpace
Space in the locals array.
Definition: compiler.c:205
int delSatisfied
Flag indicating if a 'del' target has been completed.
Definition: compiler.c:224
struct Compiler * enclosed
Subcompiler we are enclosing, need for type annotation compilation.
Definition: compiler.c:221
size_t loopLocalCount
Tracks how many locals to pop off the stack when exiting a loop.
Definition: compiler.c:210
size_t localCount
Total number of local variables.
Definition: compiler.c:204
size_t scopeDepth
Depth of nested scope blocks.
Definition: compiler.c:203
size_t breakSpace
Space in breaks array.
Definition: compiler.c:212
size_t upvaluesSpace
Space in the upvalues array.
Definition: compiler.c:207
Local * locals
Array of local variable references.
Definition: compiler.c:206
struct LoopExit * continues
Array of loop exit instruction indices for continue statements.
Definition: compiler.c:216
struct LoopExit * breaks
Array of loop exit instruction indices for break statements.
Definition: compiler.c:213
size_t annotationCount
Number of type annotations found while compiling function signature.
Definition: compiler.c:222
KrkCodeObject * codeobject
Bytecode emitter.
Definition: compiler.c:201
size_t breakCount
Number of break statements.
Definition: compiler.c:211
struct Compiler * enclosing
Enclosing function compiler, or NULL for a module.
Definition: compiler.c:200
Upvalue * upvalues
Array of upvalue references. Count is stored in the codeobject.
Definition: compiler.c:208
int unnamedArgs
Number of positional arguments that will not be assignable through keywords.
Definition: compiler.c:227
KrkScanner scanner
Scanner state.
Definition: compiler.c:279
Parser parser
Parser state.
Definition: compiler.c:278
ClassCompiler * currentClass
Current in-progress class definition (or NULL)
Definition: compiler.c:281
Compiler * current
Current compiler (in-progress code object) state.
Definition: compiler.c:280
KrkInstance inst
Base instance.
Definition: compiler.c:277
Linked list of indices.
Definition: compiler.c:180
size_t ind
Index of an identifier constant.
Definition: compiler.c:181
struct IndexWithNext * next
Linked list next pointer.
Definition: compiler.c:182
Opcode chunk of a code object.
Definition: chunk.h:36
size_t krk_writeConstant(KrkChunk *chunk, KrkValue value, size_t line)
Add a new constant and write an instruction for it.
Definition: chunk.c:69
size_t krk_addConstant(KrkChunk *chunk, KrkValue value)
Add a new constant value to an opcode chunk.
Definition: chunk.c:50
void krk_writeChunk(KrkChunk *chunk, uint8_t byte, size_t line)
Append a byte to an opcode chunk.
Definition: chunk.c:31
Type object.
Definition: object.h:215
KrkCleanupCallback _ongcsweep
C function to call when the garbage collector is discarding an instance of this class.
Definition: object.h:224
KrkCleanupCallback _ongcscan
C function to call when the garbage collector visits an instance of this class in the scan phase.
Definition: object.h:223
size_t allocSize
Size to allocate when creating instances of this class.
Definition: object.h:222
KrkObj obj
Base.
Definition: object.h:216
void krk_finalizeClass(KrkClass *_class)
Finalize a class by collecting pointers to core methods.
Definition: vm.c:189
Code object.
Definition: object.h:163
unsigned short potentialPositionals
Precalculated positional arguments for complex argument processing.
Definition: object.h:167
size_t overlongJumpsCount
Number of entries in pessimal jump table.
Definition: object.h:185
KrkChunk chunk
Bytecode data.
Definition: object.h:170
size_t upvalueCount
Number of upvalues this function collects as a closure.
Definition: object.h:169
void krk_debug_addExpression(KrkCodeObject *codeobject, uint8_t start, uint8_t midStart, uint8_t midEnd, uint8_t end)
Add an expression mapping to the bytecode chunk.
Definition: debug.c:705
KrkLocalEntry * localNames
Stores the names of local variables used in the function, for debugging.
Definition: object.h:177
unsigned short keywordArgs
Arity of keyword (default) arguments.
Definition: object.h:166
KrkOverlongJump * overlongJumps
Pessimal overlong jump container.
Definition: object.h:183
unsigned short requiredArgs
Arity of required (non-default) arguments.
Definition: object.h:165
KrkString * docstring
Docstring attached to the function.
Definition: object.h:172
KrkObj obj
Base.
Definition: object.h:164
KrkCodeObject * krk_newCodeObject(void)
Create a new, uninitialized code object.
Definition: object.c:264
size_t localNameCount
Number of entries in localNames.
Definition: object.h:176
KrkString * name
Name of the function.
Definition: object.h:171
size_t overlongJumpsCapacity
Number of possible entries in pessimal jump table.
Definition: object.h:184
KrkString * qualname
The dotted name of the function.
Definition: object.h:178
Map entry of opcode offsets to expressions spans.
Definition: object.h:141
An object of a class.
Definition: object.h:281
KrkInstance * krk_newInstance(KrkClass *_class)
Create a new instance of the given class.
Definition: object.c:343
KrkTable fields
Attributes table.
Definition: object.h:284
Map entry of instruction offsets to line numbers.
Definition: chunk.h:19
Metadata on a local variable name in a function.
Definition: object.h:129
size_t birthday
Instruction offset that this local name became valid on.
Definition: object.h:131
KrkString * name
Name of the local.
Definition: object.h:133
size_t id
Local ID as used by opcodes; offset from the frame's stack base.
Definition: object.h:130
size_t deathday
Instruction offset that this local name becomes invalid on.
Definition: object.h:132
The most basic object type.
Definition: object.h:41
uint16_t flags
General object flags, mostly related to garbage collection.
Definition: object.h:43
uint16_t intendedTarget
High bytes of the intended target.
Definition: object.h:153
uint8_t originalOpcode
Original jump opcode to execute.
Definition: object.h:154
uint32_t instructionOffset
Instruction (operand offset) this jump target applies to.
Definition: object.h:152
Token scanner state.
Definition: scanner.h:140
Immutable sequence of Unicode codepoints.
Definition: object.h:93
KrkString * krk_copyString(const char *chars, size_t length)
Obtain a string object representation of the given C string.
Definition: object.c:224
char * chars
UTF8 canonical data.
Definition: object.h:97
size_t length
String length in bytes.
Definition: object.h:95
size_t krk_codepointToBytes(krk_integer_type value, unsigned char *out)
Convert an integer codepoint to a UTF-8 byte representation.
Definition: object.c:38
int krk_tableGet(KrkTable *table, KrkValue key, KrkValue *value)
Obtain the value associated with a key in a table.
Definition: table.c:211
void krk_attachNamedObject(KrkTable *table, const char name[], KrkObj *obj)
Attach an object to an attribute table.
Definition: vm.c:808
void krk_attachNamedValue(KrkTable *table, const char name[], KrkValue obj)
Attach a value to an attribute table.
Definition: vm.c:794
int krk_tableGet_fast(KrkTable *table, struct KrkString *str, KrkValue *value)
Obtain the value associated with a string key in a table.
Definition: table.c:219
KrkValue currentException
Definition: vm.h:164
int flags
Definition: vm.h:165
KrkInstance * module
Definition: vm.h:163
A token from the scanner.
Definition: scanner.h:124
void krk_writeValueArray(KrkValueArray *array, KrkValue value)
Add a value to a value array.
Definition: value.c:17
size_t count
Definition: value.h:77
Stack reference or primative value.
Local variable reference.
Definition: compiler.c:137
KrkToken name
Token that provided the name for this variable.
Definition: compiler.c:138
ssize_t depth
Stack depth, or -1 if uninitialized.
Definition: compiler.c:139
char isCaptured
Flag indicating if the variable is captured by a closure.
Definition: compiler.c:140
Tracks 'break' and 'continue' statements.
Definition: compiler.c:188
KrkToken token
Token for this exit statement, so its location can be printed in an error message.
Definition: compiler.c:190
int offset
Offset of the jump expression to patch.
Definition: compiler.c:189
Parse rule table entry.
Definition: compiler.c:124
ParseFn infix
Parse function to call when this token appears after an expression.
Definition: compiler.c:126
Precedence precedence
Precedence ordering for Pratt parsing, Precedence.
Definition: compiler.c:127
ParseFn prefix
Parse function to call when this token appears at the start of an expression.
Definition: compiler.c:125
Token parser state.
Definition: compiler.c:56
KrkToken previous
Last token matched, consumed, or advanced over.
Definition: compiler.c:58
KrkToken current
Token to be parsed.
Definition: compiler.c:57
unsigned int eatingWhitespace
Depth of whitespace-ignoring parse functions.
Definition: compiler.c:60
char hadError
Flag indicating if the parser encountered an error.
Definition: compiler.c:59
Compiler emit and parse state prior to this expression.
Definition: compiler.c:270
ChunkRecorder before
Bytecode and constant table output offsets.
Definition: compiler.c:271
Parser oldParser
Previous/current tokens.
Definition: compiler.c:273
KrkScanner oldScanner
Scanner cursor state.
Definition: compiler.c:272
Inline flexible string array.
Definition: util.h:162
Closure upvalue reference.
Definition: compiler.c:148
char isLocal
Flag indicating if Kuroko Language Reference is a local or upvalue index.
Definition: compiler.c:150
KrkToken name
Name for direct lookup. Mainly for non-automatically-populated upvalue cells.
Definition: compiler.c:151
size_t index
Enclosing local index or upvalue index.
Definition: compiler.c:149
Utilities for creating native bindings.
void krk_pushStringBuilderStr(struct StringBuilder *sb, const char *str, size_t len)
Append a string to the end of a string builder.
Definition: obj_str.c:1091
KrkValue krk_finishStringBuilderBytes(struct StringBuilder *sb)
Finalize a string builder in a bytes object.
Definition: obj_str.c:1117
KrkValue krk_discardStringBuilder(struct StringBuilder *sb)
Discard the contents of a string builder.
Definition: obj_str.c:1123
KrkValue krk_finishStringBuilder(struct StringBuilder *sb)
Finalize a string builder into a string object.
Definition: obj_str.c:1111
void krk_pushStringBuilder(struct StringBuilder *sb, char c)
Add a character to the end of a string builder.
Definition: obj_str.c:1082
Core API for the bytecode virtual machine.
krk_threadLocal KrkThreadState krk_currentThread
Thread-local VM state.
#define vm
Convenience macro for namespacing.
Definition: vm.h:257
KrkValue krk_pop(void)
Pop the top of the stack.
Definition: vm.c:131
void krk_push(KrkValue value)
Push a stack value.
Definition: vm.c:118