Fix parse error with comment on last line.

If a comment is not followed by a newline character, then
we got a false parse error. This is because the comment
state is left on the parse stack when we finished parsing, and
since the parse stack was not emtpy, we assumed an error.

This commit adds the parser/eof function, which lets the parser know
that an eof was reached. Before, we simply added a fake newline
character in some cases, and in the case of reading a file, we did
nothing, hence the bug.

src/core/core.janet

@@ -1434,7 +1434,7 @@ value, one key will be ignored."
               " around byte "
               (string (parser/where p))
               ": "
-              (or (parser/error p) "unmatched delimiter")
+              (parser/error p)
               "\n"))
 
 (defn bad-compile
@@ -1514,7 +1514,9 @@ value, one key will be ignored."
     (var pindex 0)
     (var pstatus nil)
     (def len (length buf))
-    (if (= len 0) (set going false))
+    (when (= len 0) 
+      (parser/eof p)
+      (set going false))
     (while (> len pindex)
       (+= pindex (parser/consume p buf pindex))
       (while (parser/has-more p)
@@ -1522,7 +1524,10 @@ value, one key will be ignored."
       (when (= (parser/status p) :error)
         (on-parse-error p where))))
 
-  (if (= (parser/status p) :pending)
+  # Check final parser state
+  (while (parser/has-more p)
+    (eval1 (parser/produce p)))
+  (when (= (parser/status p) :error)
     (on-parse-error p where))
 
   (set *env* oldenv)

src/core/parse.c

@@ -531,20 +531,34 @@ static int root(JanetParser *p, JanetParseState *state, uint8_t c) {
     }
 }
 
-int janet_parser_consume(JanetParser *parser, uint8_t c) {
+static void janet_parser_checkdead(JanetParser *parser) {
+    if (parser->flag) janet_panic("parser is dead, cannot consume");
+    if (parser->error) janet_panic("parser has unchecked error, cannot consume");
+}
+
+/* Public API */
+
+void janet_parser_consume(JanetParser *parser, uint8_t c) {
     int consumed = 0;
-    if (parser->error) return 0;
+    janet_parser_checkdead(parser);
     parser->offset++;
     while (!consumed && !parser->error) {
         JanetParseState *state = parser->states + parser->statecount - 1;
         consumed = state->consumer(parser, state, c);
     }
     parser->lookback = c;
-    return 1;
+}
+
+void janet_parser_eof(JanetParser *parser) {
+    janet_parser_checkdead(parser);
+    janet_parser_consume(parser, '\n');
+    parser->offset--;
+    parser->flag = 1;
 }
 
 enum JanetParserStatus janet_parser_status(JanetParser *parser) {
     if (parser->error) return JANET_PARSE_ERROR;
+    if (parser->flag) return JANET_PARSE_DEAD;
     if (parser->statecount > 1) return JANET_PARSE_PENDING;
     return JANET_PARSE_ROOT;
 }
@@ -594,6 +608,7 @@ void janet_parser_init(JanetParser *parser) {
     parser->lookback = -1;
     parser->offset = 0;
     parser->pending = 0;
+    parser->flag = 0;
 
     pushstate(parser, root, PFLAG_CONTAINER);
 }
@@ -669,6 +684,13 @@ static Janet cfun_parse_consume(int32_t argc, Janet *argv) {
     return janet_wrap_integer(i);
 }
 
+static Janet cfun_parse_eof(int32_t argc, Janet *argv) {
+    janet_fixarity(argc, 1);
+    JanetParser *p = janet_getabstract(argv, 0, &janet_parse_parsertype);
+    janet_parser_eof(p);
+    return argv[0];
+}
+
 static Janet cfun_parse_insert(int32_t argc, Janet *argv) {
     janet_fixarity(argc, 2);
     JanetParser *p = janet_getabstract(argv, 0, &janet_parse_parsertype);
@@ -730,6 +752,9 @@ static Janet cfun_parse_status(int32_t argc, Janet *argv) {
         case JANET_PARSE_ROOT:
             stat = "root";
             break;
+        case JANET_PARSE_DEAD:
+            stat = "dead";
+            break;
     }
     return janet_ckeywordv(stat);
 }
@@ -801,6 +826,7 @@ static const JanetMethod parser_methods[] = {
     {"state", cfun_parse_state},
     {"status", cfun_parse_status},
     {"where", cfun_parse_where},
+    {"eof", cfun_parse_eof},
     {NULL, NULL}
 };
 
@@ -880,6 +906,11 @@ static const JanetReg parse_cfuns[] = {
              "in the byte stream as a tuple (line, column). Lines and columns are counted from "
              "1, (the first byte is line 1, column 1) and a newline is considered ASCII 0x0A.")
     },
+    {
+        "parser/eof", cfun_parse_eof,
+        JDOC("(parser/insert parser)\n\n"
+             "Indicate that the end of file was reached to the parser. This puts the parser in the :dead state.")
+    },
     {
         "parser/insert", cfun_parse_insert,
         JDOC("(parser/insert parser value)\n\n"

src/core/run.c

@@ -28,17 +28,17 @@
 /* Run a string */
 int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char *sourcePath, Janet *out) {
     JanetParser parser;
-    int errflags = 0;
+    int errflags = 0, done = 0;
     int32_t index = 0;
-    int dudeol = 0;
-    int done = 0;
     Janet ret = janet_wrap_nil();
     const uint8_t *where = sourcePath ? janet_cstring(sourcePath) : NULL;
+
     if (where) janet_gcroot(janet_wrap_string(where));
     if (NULL == sourcePath) sourcePath = "<unknown>";
     janet_parser_init(&parser);
 
-    while (!errflags && !done) {
+    /* While we haven't seen an error */
+    while (!done) {
 
         /* Evaluate parsed values */
         while (janet_parser_has_more(&parser)) {
@@ -51,38 +51,37 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
                 if (status != JANET_SIGNAL_OK) {
                     janet_stacktrace(fiber, ret);
                     errflags |= 0x01;
+                    done = 1;
                 }
             } else {
                 fprintf(stderr, "compile error in %s: %s\n", sourcePath,
                         (const char *)cres.error);
                 errflags |= 0x02;
+                done = 1;
             }
         }
 
         /* Dispatch based on parse state */
         switch (janet_parser_status(&parser)) {
+            case JANET_PARSE_DEAD:
+                done = 1;
+                break;
             case JANET_PARSE_ERROR:
                 errflags |= 0x04;
                 fprintf(stderr, "parse error in %s: %s\n",
                         sourcePath, janet_parser_error(&parser));
+                done = 1;
                 break;
             case JANET_PARSE_PENDING:
-                if (index >= len) {
+                if (index == len) {
-                    if (dudeol) {
+                    janet_parser_eof(&parser);
-                        errflags |= 0x04;
-                        fprintf(stderr, "internal parse error in %s: unexpected end of source\n",
-                                sourcePath);
-                    } else {
-                        dudeol = 1;
-                        janet_parser_consume(&parser, '\n');
-                    }
                 } else {
                     janet_parser_consume(&parser, bytes[index++]);
                 }
                 break;
             case JANET_PARSE_ROOT:
                 if (index >= len) {
-                    done = 1;
+                    janet_parser_eof(&parser);
                 } else {
                     janet_parser_consume(&parser, bytes[index++]);
                 }
@@ -90,6 +89,8 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
         }
 
     }
+
+    /* Clean up and return errors */
     janet_parser_deinit(&parser);
     if (where) janet_gcunroot(janet_wrap_string(where));
     if (out) *out = ret;

src/include/janet.h

@@ -784,7 +784,8 @@ typedef struct JanetParser JanetParser;
 enum JanetParserStatus {
     JANET_PARSE_ROOT,
     JANET_PARSE_ERROR,
-    JANET_PARSE_PENDING
+    JANET_PARSE_PENDING,
+    JANET_PARSE_DEAD
 };
 
 /* A janet parser */
@@ -802,6 +803,7 @@ struct JanetParser {
     size_t offset;
     size_t pending;
     int lookback;
+    int flag;
 };
 
 typedef struct {
@@ -970,12 +972,12 @@ extern enum JanetInstructionType janet_instructions[JOP_INSTRUCTION_COUNT];
 /* Parsing */
 JANET_API void janet_parser_init(JanetParser *parser);
 JANET_API void janet_parser_deinit(JanetParser *parser);
-JANET_API int janet_parser_consume(JanetParser *parser, uint8_t c);
+JANET_API void janet_parser_consume(JanetParser *parser, uint8_t c);
 JANET_API enum JanetParserStatus janet_parser_status(JanetParser *parser);
 JANET_API Janet janet_parser_produce(JanetParser *parser);
 JANET_API const char *janet_parser_error(JanetParser *parser);
 JANET_API void janet_parser_flush(JanetParser *parser);
-JANET_API JanetParser *janet_check_parser(Janet x);
+JANET_API void janet_parser_eof(JanetParser *parser);
 #define janet_parser_has_more(P) ((P)->pending)
 
 /* Assembly */