Merge remote-tracking branch 'upstream/master' into register-corefile

README.md

@@ -98,7 +98,7 @@ janet:1:> (+ 1 2 3)
 janet:2:> (print "Hello, World!")
 Hello, World!
 nil
-janet:3:> (os.exit)
+janet:3:> (os/exit)
 $ ./janet -h
 usage: ./janet [options] scripts...
 Options are:

src/core/core.janet

@@ -1434,7 +1434,7 @@ value, one key will be ignored."
               " around byte "
               (string (parser/where p))
               ": "
-              (or (parser/error p) "unmatched delimiter")
+              (parser/error p)
               "\n"))
 
 (defn bad-compile
@@ -1514,7 +1514,9 @@ value, one key will be ignored."
     (var pindex 0)
     (var pstatus nil)
     (def len (length buf))
-    (if (= len 0) (set going false))
+    (when (= len 0) 
+      (parser/eof p)
+      (set going false))
     (while (> len pindex)
       (+= pindex (parser/consume p buf pindex))
       (while (parser/has-more p)
@@ -1522,7 +1524,10 @@ value, one key will be ignored."
       (when (= (parser/status p) :error)
         (on-parse-error p where))))
 
-  (if (= (parser/status p) :pending)
+  # Check final parser state
+  (while (parser/has-more p)
+    (eval1 (parser/produce p)))
+  (when (= (parser/status p) :error)
     (on-parse-error p where))
 
   (set *env* oldenv)

src/core/parse.c

@@ -531,20 +531,34 @@ static int root(JanetParser *p, JanetParseState *state, uint8_t c) {
     }
 }
 
-int janet_parser_consume(JanetParser *parser, uint8_t c) {
+static void janet_parser_checkdead(JanetParser *parser) {
+    if (parser->flag) janet_panic("parser is dead, cannot consume");
+    if (parser->error) janet_panic("parser has unchecked error, cannot consume");
+}
+
+/* Public API */
+
+void janet_parser_consume(JanetParser *parser, uint8_t c) {
     int consumed = 0;
-    if (parser->error) return 0;
+    janet_parser_checkdead(parser);
     parser->offset++;
     while (!consumed && !parser->error) {
         JanetParseState *state = parser->states + parser->statecount - 1;
         consumed = state->consumer(parser, state, c);
     }
     parser->lookback = c;
-    return 1;
+}
+
+void janet_parser_eof(JanetParser *parser) {
+    janet_parser_checkdead(parser);
+    janet_parser_consume(parser, '\n');
+    parser->offset--;
+    parser->flag = 1;
 }
 
 enum JanetParserStatus janet_parser_status(JanetParser *parser) {
     if (parser->error) return JANET_PARSE_ERROR;
+    if (parser->flag) return JANET_PARSE_DEAD;
     if (parser->statecount > 1) return JANET_PARSE_PENDING;
     return JANET_PARSE_ROOT;
 }
@@ -594,6 +608,7 @@ void janet_parser_init(JanetParser *parser) {
     parser->lookback = -1;
     parser->offset = 0;
     parser->pending = 0;
+    parser->flag = 0;
 
     pushstate(parser, root, PFLAG_CONTAINER);
 }
@@ -669,6 +684,13 @@ static Janet cfun_parse_consume(int32_t argc, Janet *argv) {
     return janet_wrap_integer(i);
 }
 
+static Janet cfun_parse_eof(int32_t argc, Janet *argv) {
+    janet_fixarity(argc, 1);
+    JanetParser *p = janet_getabstract(argv, 0, &janet_parse_parsertype);
+    janet_parser_eof(p);
+    return argv[0];
+}
+
 static Janet cfun_parse_insert(int32_t argc, Janet *argv) {
     janet_fixarity(argc, 2);
     JanetParser *p = janet_getabstract(argv, 0, &janet_parse_parsertype);
@@ -730,6 +752,9 @@ static Janet cfun_parse_status(int32_t argc, Janet *argv) {
         case JANET_PARSE_ROOT:
             stat = "root";
             break;
+        case JANET_PARSE_DEAD:
+            stat = "dead";
+            break;
     }
     return janet_ckeywordv(stat);
 }
@@ -801,6 +826,7 @@ static const JanetMethod parser_methods[] = {
     {"state", cfun_parse_state},
     {"status", cfun_parse_status},
     {"where", cfun_parse_where},
+    {"eof", cfun_parse_eof},
     {NULL, NULL}
 };
 
@@ -880,6 +906,11 @@ static const JanetReg parse_cfuns[] = {
              "in the byte stream as a tuple (line, column). Lines and columns are counted from "
              "1, (the first byte is line 1, column 1) and a newline is considered ASCII 0x0A.")
     },
+    {
+        "parser/eof", cfun_parse_eof,
+        JDOC("(parser/insert parser)\n\n"
+             "Indicate that the end of file was reached to the parser. This puts the parser in the :dead state.")
+    },
     {
         "parser/insert", cfun_parse_insert,
         JDOC("(parser/insert parser value)\n\n"

src/core/run.c

@@ -28,17 +28,17 @@
 /* Run a string */
 int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char *sourcePath, Janet *out) {
     JanetParser parser;
-    int errflags = 0;
+    int errflags = 0, done = 0;
     int32_t index = 0;
-    int dudeol = 0;
-    int done = 0;
     Janet ret = janet_wrap_nil();
     const uint8_t *where = sourcePath ? janet_cstring(sourcePath) : NULL;
+
     if (where) janet_gcroot(janet_wrap_string(where));
     if (NULL == sourcePath) sourcePath = "<unknown>";
     janet_parser_init(&parser);
 
-    while (!errflags && !done) {
+    /* While we haven't seen an error */
+    while (!done) {
 
         /* Evaluate parsed values */
         while (janet_parser_has_more(&parser)) {
@@ -51,38 +51,37 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
                 if (status != JANET_SIGNAL_OK) {
                     janet_stacktrace(fiber, ret);
                     errflags |= 0x01;
+                    done = 1;
                 }
             } else {
                 fprintf(stderr, "compile error in %s: %s\n", sourcePath,
                         (const char *)cres.error);
                 errflags |= 0x02;
+                done = 1;
             }
         }
 
         /* Dispatch based on parse state */
         switch (janet_parser_status(&parser)) {
+            case JANET_PARSE_DEAD:
+                done = 1;
+                break;
             case JANET_PARSE_ERROR:
                 errflags |= 0x04;
                 fprintf(stderr, "parse error in %s: %s\n",
                         sourcePath, janet_parser_error(&parser));
+                done = 1;
                 break;
             case JANET_PARSE_PENDING:
-                if (index >= len) {
+                if (index == len) {
-                    if (dudeol) {
+                    janet_parser_eof(&parser);
-                        errflags |= 0x04;
-                        fprintf(stderr, "internal parse error in %s: unexpected end of source\n",
-                                sourcePath);
-                    } else {
-                        dudeol = 1;
-                        janet_parser_consume(&parser, '\n');
-                    }
                 } else {
                     janet_parser_consume(&parser, bytes[index++]);
                 }
                 break;
             case JANET_PARSE_ROOT:
                 if (index >= len) {
-                    done = 1;
+                    janet_parser_eof(&parser);
                 } else {
                     janet_parser_consume(&parser, bytes[index++]);
                 }
@@ -90,6 +89,8 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
         }
 
     }
+
+    /* Clean up and return errors */
     janet_parser_deinit(&parser);
     if (where) janet_gcunroot(janet_wrap_string(where));
     if (out) *out = ret;

src/include/janet.h

@@ -178,7 +178,9 @@ extern "C" {
 /* Define max stack size for stacks before raising a stack overflow error.
  * If this is not defined, fiber stacks can grow without limit (until memory
  * runs out) */
-#define JANET_STACK_MAX 8192
+#ifndef JANET_STACK_MAX
+#define JANET_STACK_MAX 16384
+#endif
 
 /* Use nanboxed values - uses 8 bytes per value instead of 12 or 16.
  * To turn of nanboxing, for debugging purposes or for certain
@@ -784,7 +786,8 @@ typedef struct JanetParser JanetParser;
 enum JanetParserStatus {
     JANET_PARSE_ROOT,
     JANET_PARSE_ERROR,
-    JANET_PARSE_PENDING
+    JANET_PARSE_PENDING,
+    JANET_PARSE_DEAD
 };
 
 /* A janet parser */
@@ -802,6 +805,7 @@ struct JanetParser {
     size_t offset;
     size_t pending;
     int lookback;
+    int flag;
 };
 
 typedef struct {
@@ -970,12 +974,12 @@ extern enum JanetInstructionType janet_instructions[JOP_INSTRUCTION_COUNT];
 /* Parsing */
 JANET_API void janet_parser_init(JanetParser *parser);
 JANET_API void janet_parser_deinit(JanetParser *parser);
-JANET_API int janet_parser_consume(JanetParser *parser, uint8_t c);
+JANET_API void janet_parser_consume(JanetParser *parser, uint8_t c);
 JANET_API enum JanetParserStatus janet_parser_status(JanetParser *parser);
 JANET_API Janet janet_parser_produce(JanetParser *parser);
 JANET_API const char *janet_parser_error(JanetParser *parser);
 JANET_API void janet_parser_flush(JanetParser *parser);
-JANET_API JanetParser *janet_check_parser(Janet x);
+JANET_API void janet_parser_eof(JanetParser *parser);
 #define janet_parser_has_more(P) ((P)->pending)
 
 /* Assembly */