Make parser API more robust - the value queue is now

distinct from the parse state, and is queried separately.

src/core/core.janet

@@ -1437,9 +1437,9 @@ value, one key will be ignored."
     (if (= len 0) (set going false))
     (while (> len pindex)
       (+= pindex (parser/consume p buf pindex))
-      (while (= (set pstatus (parser/status p)) :full)
+      (while (parser/has-more p)
         (eval1 (parser/produce p)))
-      (when (= pstatus :error)
+      (when (= (parser/status p) :error)
         (onstatus :parse
                   (string (parser/error p)
                           " around byte " (parser/where p))

src/core/parse.c

@@ -161,6 +161,8 @@ static void popstate(JanetParser *p, Janet val) {
                 janet_tuple_sm_end(janet_unwrap_tuple(val)) = (int32_t) p->offset;
             }
             newtop->argn++;
+            /* Keep track of number of values in the root state */
+            if (p->statecount == 1) p->pending++;
             push_arg(p, val);
             return;
         } else if (newtop->flags & PFLAG_READERMAC) {
@@ -534,7 +536,6 @@ int janet_parser_consume(JanetParser *parser, uint8_t c) {
 enum JanetParserStatus janet_parser_status(JanetParser *parser) {
     if (parser->error) return JANET_PARSE_ERROR;
     if (parser->statecount > 1) return JANET_PARSE_PENDING;
-    if (parser->argcount) return JANET_PARSE_FULL;
     return JANET_PARSE_ROOT;
 }
 
@@ -542,6 +543,7 @@ void janet_parser_flush(JanetParser *parser) {
     parser->argcount = 0;
     parser->statecount = 1;
     parser->bufcount = 0;
+    parser->pending = 0;
 }
 
 const char *janet_parser_error(JanetParser *parser) {
@@ -558,12 +560,12 @@ const char *janet_parser_error(JanetParser *parser) {
 Janet janet_parser_produce(JanetParser *parser) {
     Janet ret;
     size_t i;
-    enum JanetParserStatus status = janet_parser_status(parser);
-    if (status != JANET_PARSE_FULL) return janet_wrap_nil();
+    if (parser->pending == 0) return janet_wrap_nil();
     ret = parser->args[0];
     for (i = 1; i < parser->argcount; i++) {
         parser->args[i - 1] = parser->args[i];
     }
+    parser->pending--;
     parser->argcount--;
     return ret;
 }
@@ -581,6 +583,7 @@ void janet_parser_init(JanetParser *parser) {
     parser->error = NULL;
     parser->lookback = -1;
     parser->offset = 0;
+    parser->pending = 0;
 
     pushstate(parser, root, PFLAG_CONTAINER);
 }
@@ -664,6 +667,14 @@ static int cfun_consume(JanetArgs args) {
     JANET_RETURN_INTEGER(args, i);
 }
 
+static int cfun_has_more(JanetArgs args) {
+    JanetParser *p;
+    JANET_FIXARITY(args, 1);
+    JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
+    p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
+    JANET_RETURN_BOOLEAN(args, janet_parser_has_more(p));
+}
+
 static int cfun_byte(JanetArgs args) {
     int32_t i;
     JanetParser *p;
@@ -682,9 +693,6 @@ static int cfun_status(JanetArgs args) {
     JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
     p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
     switch (janet_parser_status(p)) {
-        case JANET_PARSE_FULL:
-            stat = "full";
-            break;
         case JANET_PARSE_PENDING:
             stat = "pending";
             break;
@@ -776,6 +784,10 @@ static const JanetReg cfuns[] = {
         "Creates and returns a new parser object. Parsers are state machines "
         "that can receive bytes, and generate a stream of janet values. "
     },
+    {"parser/has-more", cfun_has_more,
+        "(parser/has-more parser)\n\n"
+        "Check if the parser has more values in the value queue."
+    },
     {"parser/produce", cfun_produce,
         "(parser/produce parser)\n\n"
         "Dequeue the next value in the parse queue. Will return nil if "
@@ -795,14 +807,15 @@ static const JanetReg cfuns[] = {
     {"parser/error", cfun_error,
         "(parser/error parser)\n\n"
         "If the parser is in the error state, returns the message asscoiated with "
-        "that error. Otherwise, returns nil."
+        "that error. Otherwise, returns nil. Also flushes the parser state and parser "
+        "queue, so be sure to handle everything in the queue before calling "
+        "parser/error."
     },
     {"parser/status", cfun_status,
         "(parser/status parser)\n\n"
         "Gets the current status of the parser state machine. The status will "
         "be one of:\n\n"
-        "\t:full - there are values in the parse queue to be consumed.\n"
-        "\t:pending - no values in the queue but a value is being parsed.\n"
+        "\t:pending - a value is being parsed.\n"
         "\t:error - a parsing error was encountered.\n"
         "\t:root - the parser can either read more values or safely terminate."
     },

src/core/run.c

@@ -92,26 +92,29 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
     janet_parser_init(&parser);
 
     while (!errflags && !done) {
-        switch (janet_parser_status(&parser)) {
-            case JANET_PARSE_FULL:
-                {
-                    Janet form = janet_parser_produce(&parser);
-                    JanetCompileResult cres = janet_compile(form, env, where);
-                    if (cres.status == JANET_COMPILE_OK) {
-                        JanetFunction *f = janet_thunk(cres.funcdef);
-                        JanetFiber *fiber = janet_fiber(f, 64);
-                        JanetSignal status = janet_continue(fiber, janet_wrap_nil(), &ret);
-                        if (status != JANET_SIGNAL_OK) {
-                            janet_stacktrace(fiber, "runtime", ret);
-                            errflags |= 0x01;
-                        }
-                    } else {
-                        janet_stacktrace(cres.macrofiber, "compile",
-                                janet_wrap_string(cres.error));
-                        errflags |= 0x02;
-                    }
+
+        /* Evaluate parsed values */
+        while (janet_parser_has_more(&parser)) {
+            Janet form = janet_parser_produce(&parser);
+            JanetCompileResult cres = janet_compile(form, env, where);
+            if (cres.status == JANET_COMPILE_OK) {
+                JanetFunction *f = janet_thunk(cres.funcdef);
+                JanetFiber *fiber = janet_fiber(f, 64);
+                JanetSignal status = janet_continue(fiber, janet_wrap_nil(), &ret);
+                if (status != JANET_SIGNAL_OK) {
+                    janet_stacktrace(fiber, "runtime", ret);
+                    errflags |= 0x01;
                 }
-                break;
+            } else {
+                fprintf(stderr, "source path: %s\n", sourcePath);
+                janet_stacktrace(cres.macrofiber, "compile",
+                        janet_wrap_string(cres.error));
+                errflags |= 0x02;
+            }
+        }
+
+        /* Dispatch based on parse state */
+        switch (janet_parser_status(&parser)) {
             case JANET_PARSE_ERROR:
                 errflags |= 0x04;
                 fprintf(stderr, "parse error: %s\n", janet_parser_error(&parser));
@@ -137,6 +140,7 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
                 }
                 break;
         }
+
     }
     janet_parser_deinit(&parser);
     if (where) janet_gcunroot(janet_wrap_string(where));

src/include/janet/janet.h

@@ -717,7 +717,6 @@ typedef struct JanetParser JanetParser;
 enum JanetParserStatus {
     JANET_PARSE_ROOT,
     JANET_PARSE_ERROR,
-    JANET_PARSE_FULL,
     JANET_PARSE_PENDING
 };
 
@@ -734,6 +733,7 @@ struct JanetParser {
     size_t bufcount;
     size_t bufcap;
     size_t offset;
+    size_t pending;
     int lookback;
 };
 
@@ -878,6 +878,7 @@ JANET_API Janet janet_parser_produce(JanetParser *parser);
 JANET_API const char *janet_parser_error(JanetParser *parser);
 JANET_API void janet_parser_flush(JanetParser *parser);
 JANET_API JanetParser *janet_check_parser(Janet x);
+#define janet_parser_has_more(P) ((P)->pending)
 
 /* Assembly */
 #ifdef JANET_ASSEMBLER