mirror of
https://github.com/janet-lang/janet
synced 2024-12-25 07:50:27 +00:00
Make parser API more robust - the value queue is now
distinct from the parse state, and is queried separately.
This commit is contained in:
Calvin Rose
parent
5fa96a6f8c
commit
d8b0a5ed01
@ -216,7 +216,7 @@
|
||||
<key>corelib</key>
|
||||
<dict>
|
||||
<key>match</key>
|
||||
<string>(?<![\.:\w_\-=!@\$%^&?|\\/<>*])(%|%=|\*|\*=|\*doc\-width\*|\*env\*|\+|\+\+|\+=|\-|\-\-|\-=|\->|\->>|\-\?>|\-\?>>|/|/=|<|<=|=|==|>|>=|_env|abstract\?|all|all\-symbols|allsyms|and|apply|array|array/concat|array/ensure|array/insert|array/new|array/peek|array/pop|array/push|array/slice|array\?|as\->|as\?\->|asm|band|blshift|bnot|boolean\?|bor|brshift|brushift|buffer|buffer/clear|buffer/new|buffer/popn|buffer/push\-byte|buffer/push\-string|buffer/push\-word|buffer/slice|buffer\?|bxor|bytes\?|callable\?|case|cfunction\?|comment|comp|compile|complement|cond|coro|count|debug|debug/arg\-stack|debug/break|debug/fbreak|debug/lineage|debug/stack|debug/unbreak|debug/unfbreak|dec|deep\-not=|deep=|def\-|default|defglobal|defmacro|defmacro\-|defn|defn\-|describe|dictionary\?|disasm|distinct|doc|doc\*|doc\-format|drop\-until|drop\-while|each|empty\?|env\-lookup|error|eval|eval\-string|even\?|every\?|extreme|false\?|fiber/current|fiber/maxstack|fiber/new|fiber/setmaxstack|fiber/status|fiber\?|file/close|file/flush|file/open|file/popen|file/read|file/seek|file/write|filter|find|find\-index|first|flatten|flatten\-into|for|frequencies|function\?|gccollect|gcinterval|gcsetinterval|generate|gensym|get|getline|hash|idempotent\?|identity|if\-let|if\-not|import|import\*|inc|indexed\?|interleave|interpose|invert|janet/build|janet/version|juxt|juxt\*|keep|keys|keyword|keyword\?|kvs|last|length|let|loop|macex|macex1|make\-env|map|mapcat|marshal|match|match\-1|math/acos|math/asin|math/atan|math/ceil|math/cos|math/e|math/exp|math/floor|math/inf|math/log|math/log10|math/pi|math/pow|math/random|math/seedrandom|math/sin|math/sqrt|math/tan|max|max\-order|merge|merge\-into|min|min\-order|module/find|module/native\-paths|module/paths|native|neg\?|next|nil\?|not|not=|not==|number\?|odd\?|one\?|or|order<|order<=|order>|order>=|os/clock|os/cwd|os/execute|os/exit|os/getenv|os/setenv|os/shell|os/sleep|os/time|os/which|pairs|parser/byte|parser/consume|parser/error|parser/flush|parser/new|parser/produce|parser/state|parser/status|parser/where|partial|pos\?|postwalk|prewalk|print|process/args|product|put|range|reduce|repl|require|resume|reverse|run\-context|scan\-number|sentinel|seq|some|sort|sorted|status\-pp|stderr|stdin|stdout|string|string/ascii\-lower|string/ascii\-upper|string/bytes|string/check\-set|string/find|string/find\-all|string/from\-bytes|string/join|string/number|string/pretty|string/repeat|string/replace|string/replace\-all|string/reverse|string/slice|string/split|string\?|struct|struct\?|sum|symbol|symbol\?|table|table/getproto|table/new|table/rawget|table/setproto|table/to\-struct|table\?|take\-until|take\-while|true\?|try|tuple|tuple/append|tuple/prepend|tuple/slice|tuple\?|type|unless|unmarshal|update|values|varglobal|walk|when|when\-let|with\-idemp|yield|zero\?|zipcoll)(?![\.:\w_\-=!@\$%^&?|\\/<>*])</string>
|
||||
<string>(?<![\.:\w_\-=!@\$%^&?|\\/<>*])(%|%=|\*|\*=|\*doc\-width\*|\*env\*|\+|\+\+|\+=|\-|\-\-|\-=|\->|\->>|\-\?>|\-\?>>|/|/=|<|<=|=|==|>|>=|_env|abstract\?|all|all\-symbols|allsyms|and|apply|array|array/concat|array/ensure|array/insert|array/new|array/peek|array/pop|array/push|array/slice|array\?|as\->|as\?\->|asm|band|blshift|bnot|boolean\?|bor|brshift|brushift|buffer|buffer/clear|buffer/new|buffer/popn|buffer/push\-byte|buffer/push\-string|buffer/push\-word|buffer/slice|buffer\?|bxor|bytes\?|callable\?|case|cfunction\?|comment|comp|compile|complement|cond|coro|count|debug|debug/arg\-stack|debug/break|debug/fbreak|debug/lineage|debug/stack|debug/unbreak|debug/unfbreak|dec|deep\-not=|deep=|def\-|default|defglobal|defmacro|defmacro\-|defn|defn\-|describe|dictionary\?|disasm|distinct|doc|doc\*|doc\-format|drop\-until|drop\-while|each|empty\?|env\-lookup|error|eval|eval\-string|even\?|every\?|extreme|false\?|fiber/current|fiber/maxstack|fiber/new|fiber/setmaxstack|fiber/status|fiber\?|file/close|file/flush|file/open|file/popen|file/read|file/seek|file/write|filter|find|find\-index|first|flatten|flatten\-into|for|frequencies|function\?|gccollect|gcinterval|gcsetinterval|generate|gensym|get|getline|hash|idempotent\?|identity|if\-let|if\-not|import|import\*|inc|indexed\?|interleave|interpose|invert|janet/build|janet/version|juxt|juxt\*|keep|keys|keyword|keyword\?|kvs|last|length|let|loop|macex|macex1|make\-env|map|mapcat|marshal|match|match\-1|math/acos|math/asin|math/atan|math/ceil|math/cos|math/e|math/exp|math/floor|math/inf|math/log|math/log10|math/pi|math/pow|math/random|math/seedrandom|math/sin|math/sqrt|math/tan|max|max\-order|merge|merge\-into|min|min\-order|module/find|module/native\-paths|module/paths|native|neg\?|next|nil\?|not|not=|not==|number\?|odd\?|one\?|or|order<|order<=|order>|order>=|os/clock|os/cwd|os/execute|os/exit|os/getenv|os/setenv|os/shell|os/sleep|os/time|os/which|pairs|parser/byte|parser/consume|parser/error|parser/flush|parser/has\-more|parser/new|parser/produce|parser/state|parser/status|parser/where|partial|pos\?|postwalk|prewalk|print|process/args|product|put|range|reduce|repl|require|resume|reverse|run\-context|scan\-number|sentinel|seq|some|sort|sorted|status\-pp|stderr|stdin|stdout|string|string/ascii\-lower|string/ascii\-upper|string/bytes|string/check\-set|string/find|string/find\-all|string/from\-bytes|string/join|string/number|string/pretty|string/repeat|string/replace|string/replace\-all|string/reverse|string/slice|string/split|string\?|struct|struct\?|sum|symbol|symbol\?|table|table/getproto|table/new|table/rawget|table/setproto|table/to\-struct|table\?|take\-until|take\-while|true\?|try|tuple|tuple/append|tuple/prepend|tuple/slice|tuple\?|type|unless|unmarshal|update|values|varglobal|walk|when|when\-let|with\-idemp|yield|zero\?|zipcoll)(?![\.:\w_\-=!@\$%^&?|\\/<>*])</string>
|
||||
<key>name</key>
|
||||
<string>keyword.control.janet</string>
|
||||
</dict>
|
||||
|
||||
@ -1437,9 +1437,9 @@ value, one key will be ignored."
|
||||
(if (= len 0) (set going false))
|
||||
(while (> len pindex)
|
||||
(+= pindex (parser/consume p buf pindex))
|
||||
(while (= (set pstatus (parser/status p)) :full)
|
||||
(while (parser/has-more p)
|
||||
(eval1 (parser/produce p)))
|
||||
(when (= pstatus :error)
|
||||
(when (= (parser/status p) :error)
|
||||
(onstatus :parse
|
||||
(string (parser/error p)
|
||||
" around byte " (parser/where p))
|
||||
|
||||
@ -161,6 +161,8 @@ static void popstate(JanetParser *p, Janet val) {
|
||||
janet_tuple_sm_end(janet_unwrap_tuple(val)) = (int32_t) p->offset;
|
||||
}
|
||||
newtop->argn++;
|
||||
/* Keep track of number of values in the root state */
|
||||
if (p->statecount == 1) p->pending++;
|
||||
push_arg(p, val);
|
||||
return;
|
||||
} else if (newtop->flags & PFLAG_READERMAC) {
|
||||
@ -534,7 +536,6 @@ int janet_parser_consume(JanetParser *parser, uint8_t c) {
|
||||
enum JanetParserStatus janet_parser_status(JanetParser *parser) {
|
||||
if (parser->error) return JANET_PARSE_ERROR;
|
||||
if (parser->statecount > 1) return JANET_PARSE_PENDING;
|
||||
if (parser->argcount) return JANET_PARSE_FULL;
|
||||
return JANET_PARSE_ROOT;
|
||||
}
|
||||
|
||||
@ -542,6 +543,7 @@ void janet_parser_flush(JanetParser *parser) {
|
||||
parser->argcount = 0;
|
||||
parser->statecount = 1;
|
||||
parser->bufcount = 0;
|
||||
parser->pending = 0;
|
||||
}
|
||||
|
||||
const char *janet_parser_error(JanetParser *parser) {
|
||||
@ -558,12 +560,12 @@ const char *janet_parser_error(JanetParser *parser) {
|
||||
Janet janet_parser_produce(JanetParser *parser) {
|
||||
Janet ret;
|
||||
size_t i;
|
||||
enum JanetParserStatus status = janet_parser_status(parser);
|
||||
if (status != JANET_PARSE_FULL) return janet_wrap_nil();
|
||||
if (parser->pending == 0) return janet_wrap_nil();
|
||||
ret = parser->args[0];
|
||||
for (i = 1; i < parser->argcount; i++) {
|
||||
parser->args[i - 1] = parser->args[i];
|
||||
}
|
||||
parser->pending--;
|
||||
parser->argcount--;
|
||||
return ret;
|
||||
}
|
||||
@ -581,6 +583,7 @@ void janet_parser_init(JanetParser *parser) {
|
||||
parser->error = NULL;
|
||||
parser->lookback = -1;
|
||||
parser->offset = 0;
|
||||
parser->pending = 0;
|
||||
|
||||
pushstate(parser, root, PFLAG_CONTAINER);
|
||||
}
|
||||
@ -664,6 +667,14 @@ static int cfun_consume(JanetArgs args) {
|
||||
JANET_RETURN_INTEGER(args, i);
|
||||
}
|
||||
|
||||
static int cfun_has_more(JanetArgs args) {
|
||||
JanetParser *p;
|
||||
JANET_FIXARITY(args, 1);
|
||||
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
|
||||
p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
|
||||
JANET_RETURN_BOOLEAN(args, janet_parser_has_more(p));
|
||||
}
|
||||
|
||||
static int cfun_byte(JanetArgs args) {
|
||||
int32_t i;
|
||||
JanetParser *p;
|
||||
@ -682,9 +693,6 @@ static int cfun_status(JanetArgs args) {
|
||||
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
|
||||
p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
|
||||
switch (janet_parser_status(p)) {
|
||||
case JANET_PARSE_FULL:
|
||||
stat = "full";
|
||||
break;
|
||||
case JANET_PARSE_PENDING:
|
||||
stat = "pending";
|
||||
break;
|
||||
@ -776,6 +784,10 @@ static const JanetReg cfuns[] = {
|
||||
"Creates and returns a new parser object. Parsers are state machines "
|
||||
"that can receive bytes, and generate a stream of janet values. "
|
||||
},
|
||||
{"parser/has-more", cfun_has_more,
|
||||
"(parser/has-more parser)\n\n"
|
||||
"Check if the parser has more values in the value queue."
|
||||
},
|
||||
{"parser/produce", cfun_produce,
|
||||
"(parser/produce parser)\n\n"
|
||||
"Dequeue the next value in the parse queue. Will return nil if "
|
||||
@ -795,14 +807,15 @@ static const JanetReg cfuns[] = {
|
||||
{"parser/error", cfun_error,
|
||||
"(parser/error parser)\n\n"
|
||||
"If the parser is in the error state, returns the message asscoiated with "
|
||||
"that error. Otherwise, returns nil."
|
||||
"that error. Otherwise, returns nil. Also flushes the parser state and parser "
|
||||
"queue, so be sure to handle everything in the queue before calling "
|
||||
"parser/error."
|
||||
},
|
||||
{"parser/status", cfun_status,
|
||||
"(parser/status parser)\n\n"
|
||||
"Gets the current status of the parser state machine. The status will "
|
||||
"be one of:\n\n"
|
||||
"\t:full - there are values in the parse queue to be consumed.\n"
|
||||
"\t:pending - no values in the queue but a value is being parsed.\n"
|
||||
"\t:pending - a value is being parsed.\n"
|
||||
"\t:error - a parsing error was encountered.\n"
|
||||
"\t:root - the parser can either read more values or safely terminate."
|
||||
},
|
||||
|
||||
@ -92,26 +92,29 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
|
||||
janet_parser_init(&parser);
|
||||
|
||||
while (!errflags && !done) {
|
||||
switch (janet_parser_status(&parser)) {
|
||||
case JANET_PARSE_FULL:
|
||||
{
|
||||
Janet form = janet_parser_produce(&parser);
|
||||
JanetCompileResult cres = janet_compile(form, env, where);
|
||||
if (cres.status == JANET_COMPILE_OK) {
|
||||
JanetFunction *f = janet_thunk(cres.funcdef);
|
||||
JanetFiber *fiber = janet_fiber(f, 64);
|
||||
JanetSignal status = janet_continue(fiber, janet_wrap_nil(), &ret);
|
||||
if (status != JANET_SIGNAL_OK) {
|
||||
janet_stacktrace(fiber, "runtime", ret);
|
||||
errflags |= 0x01;
|
||||
}
|
||||
} else {
|
||||
janet_stacktrace(cres.macrofiber, "compile",
|
||||
janet_wrap_string(cres.error));
|
||||
errflags |= 0x02;
|
||||
}
|
||||
|
||||
/* Evaluate parsed values */
|
||||
while (janet_parser_has_more(&parser)) {
|
||||
Janet form = janet_parser_produce(&parser);
|
||||
JanetCompileResult cres = janet_compile(form, env, where);
|
||||
if (cres.status == JANET_COMPILE_OK) {
|
||||
JanetFunction *f = janet_thunk(cres.funcdef);
|
||||
JanetFiber *fiber = janet_fiber(f, 64);
|
||||
JanetSignal status = janet_continue(fiber, janet_wrap_nil(), &ret);
|
||||
if (status != JANET_SIGNAL_OK) {
|
||||
janet_stacktrace(fiber, "runtime", ret);
|
||||
errflags |= 0x01;
|
||||
}
|
||||
break;
|
||||
} else {
|
||||
fprintf(stderr, "source path: %s\n", sourcePath);
|
||||
janet_stacktrace(cres.macrofiber, "compile",
|
||||
janet_wrap_string(cres.error));
|
||||
errflags |= 0x02;
|
||||
}
|
||||
}
|
||||
|
||||
/* Dispatch based on parse state */
|
||||
switch (janet_parser_status(&parser)) {
|
||||
case JANET_PARSE_ERROR:
|
||||
errflags |= 0x04;
|
||||
fprintf(stderr, "parse error: %s\n", janet_parser_error(&parser));
|
||||
@ -137,6 +140,7 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
janet_parser_deinit(&parser);
|
||||
if (where) janet_gcunroot(janet_wrap_string(where));
|
||||
|
||||
@ -717,7 +717,6 @@ typedef struct JanetParser JanetParser;
|
||||
enum JanetParserStatus {
|
||||
JANET_PARSE_ROOT,
|
||||
JANET_PARSE_ERROR,
|
||||
JANET_PARSE_FULL,
|
||||
JANET_PARSE_PENDING
|
||||
};
|
||||
|
||||
@ -734,6 +733,7 @@ struct JanetParser {
|
||||
size_t bufcount;
|
||||
size_t bufcap;
|
||||
size_t offset;
|
||||
size_t pending;
|
||||
int lookback;
|
||||
};
|
||||
|
||||
@ -878,6 +878,7 @@ JANET_API Janet janet_parser_produce(JanetParser *parser);
|
||||
JANET_API const char *janet_parser_error(JanetParser *parser);
|
||||
JANET_API void janet_parser_flush(JanetParser *parser);
|
||||
JANET_API JanetParser *janet_check_parser(Janet x);
|
||||
#define janet_parser_has_more(P) ((P)->pending)
|
||||
|
||||
/* Assembly */
|
||||
#ifdef JANET_ASSEMBLER
|
||||
|
||||
Loading…
Reference in New Issue
Block a user