mirrors/janet
1
0
Fork 0
mirror of https://github.com/janet-lang/janet synced 2024-11-24 17:27:18 +00:00
Code Issues 1 Releases Wiki Activity

Make parser API more robust - the value queue is now

Browse Source 
distinct from the parse state, and is queried separately.
This commit is contained in:
Calvin Rose 2019-01-03 20:44:58 -05:00
parent 5fa96a6f8c
commit d8b0a5ed01
5 changed files with 50 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
grammar/janet.tmLanguage
View File

@ -216,7 +216,7 @@
<key>corelib</key> <key>corelib</key>
<dict> <dict>
<key>match</key> <key>match</key>
<string>(?&lt;![\.:\w_\-=!@\$%^&amp;?|\\/&lt;&gt;*])(%|%=|\*|\*=|\*doc\-width\*|\*env\*|\+|\+\+|\+=|\-|\-\-|\-=|\-&gt;|\-&gt;&gt;|\-\?&gt;|\-\?&gt;&gt;|&#47;|&#47;=|&lt;|&lt;=|=|==|&gt;|&gt;=|_env|abstract\?|all|all\-symbols|allsyms|and|apply|array|array&#47;concat|array&#47;ensure|array&#47;insert|array&#47;new|array&#47;peek|array&#47;pop|array&#47;push|array&#47;slice|array\?|as\-&gt;|as\?\-&gt;|asm|band|blshift|bnot|boolean\?|bor|brshift|brushift|buffer|buffer&#47;clear|buffer&#47;new|buffer&#47;popn|buffer&#47;push\-byte|buffer&#47;push\-string|buffer&#47;push\-word|buffer&#47;slice|buffer\?|bxor|bytes\?|callable\?|case|cfunction\?|comment|comp|compile|complement|cond|coro|count|debug|debug&#47;arg\-stack|debug&#47;break|debug&#47;fbreak|debug&#47;lineage|debug&#47;stack|debug&#47;unbreak|debug&#47;unfbreak|dec|deep\-not=|deep=|def\-|default|defglobal|defmacro|defmacro\-|defn|defn\-|describe|dictionary\?|disasm|distinct|doc|doc\*|doc\-format|drop\-until|drop\-while|each|empty\?|env\-lookup|error|eval|eval\-string|even\?|every\?|extreme|false\?|fiber&#47;current|fiber&#47;maxstack|fiber&#47;new|fiber&#47;setmaxstack|fiber&#47;status|fiber\?|file&#47;close|file&#47;flush|file&#47;open|file&#47;popen|file&#47;read|file&#47;seek|file&#47;write|filter|find|find\-index|first|flatten|flatten\-into|for|frequencies|function\?|gccollect|gcinterval|gcsetinterval|generate|gensym|get|getline|hash|idempotent\?|identity|if\-let|if\-not|import|import\*|inc|indexed\?|interleave|interpose|invert|janet&#47;build|janet&#47;version|juxt|juxt\*|keep|keys|keyword|keyword\?|kvs|last|length|let|loop|macex|macex1|make\-env|map|mapcat|marshal|match|match\-1|math&#47;acos|math&#47;asin|math&#47;atan|math&#47;ceil|math&#47;cos|math&#47;e|math&#47;exp|math&#47;floor|math&#47;inf|math&#47;log|math&#47;log10|math&#47;pi|math&#47;pow|math&#47;random|math&#47;seedrandom|math&#47;sin|math&#47;sqrt|math&#47;tan|max|max\-order|merge|merge\-into|min|min\-order|module&#47;find|module&#47;native\-paths|module&#47;paths|native|neg\?|next|nil\?|not|not=|not==|number\?|odd\?|one\?|or|order&lt;|order&lt;=|order&gt;|order&gt;=|os&#47;clock|os&#47;cwd|os&#47;execute|os&#47;exit|os&#47;getenv|os&#47;setenv|os&#47;shell|os&#47;sleep|os&#47;time|os&#47;which|pairs|parser&#47;byte|parser&#47;consume|parser&#47;error|parser&#47;flush|parser&#47;new|parser&#47;produce|parser&#47;state|parser&#47;status|parser&#47;where|partial|pos\?|postwalk|prewalk|print|process&#47;args|product|put|range|reduce|repl|require|resume|reverse|run\-context|scan\-number|sentinel|seq|some|sort|sorted|status\-pp|stderr|stdin|stdout|string|string&#47;ascii\-lower|string&#47;ascii\-upper|string&#47;bytes|string&#47;check\-set|string&#47;find|string&#47;find\-all|string&#47;from\-bytes|string&#47;join|string&#47;number|string&#47;pretty|string&#47;repeat|string&#47;replace|string&#47;replace\-all|string&#47;reverse|string&#47;slice|string&#47;split|string\?|struct|struct\?|sum|symbol|symbol\?|table|table&#47;getproto|table&#47;new|table&#47;rawget|table&#47;setproto|table&#47;to\-struct|table\?|take\-until|take\-while|true\?|try|tuple|tuple&#47;append|tuple&#47;prepend|tuple&#47;slice|tuple\?|type|unless|unmarshal|update|values|varglobal|walk|when|when\-let|with\-idemp|yield|zero\?|zipcoll)(?![\.:\w_\-=!@\$%^&amp;?|\\/&lt;&gt;*])</string> <string>(?&lt;![\.:\w_\-=!@\$%^&amp;?|\\/&lt;&gt;*])(%|%=|\*|\*=|\*doc\-width\*|\*env\*|\+|\+\+|\+=|\-|\-\-|\-=|\-&gt;|\-&gt;&gt;|\-\?&gt;|\-\?&gt;&gt;|&#47;|&#47;=|&lt;|&lt;=|=|==|&gt;|&gt;=|_env|abstract\?|all|all\-symbols|allsyms|and|apply|array|array&#47;concat|array&#47;ensure|array&#47;insert|array&#47;new|array&#47;peek|array&#47;pop|array&#47;push|array&#47;slice|array\?|as\-&gt;|as\?\-&gt;|asm|band|blshift|bnot|boolean\?|bor|brshift|brushift|buffer|buffer&#47;clear|buffer&#47;new|buffer&#47;popn|buffer&#47;push\-byte|buffer&#47;push\-string|buffer&#47;push\-word|buffer&#47;slice|buffer\?|bxor|bytes\?|callable\?|case|cfunction\?|comment|comp|compile|complement|cond|coro|count|debug|debug&#47;arg\-stack|debug&#47;break|debug&#47;fbreak|debug&#47;lineage|debug&#47;stack|debug&#47;unbreak|debug&#47;unfbreak|dec|deep\-not=|deep=|def\-|default|defglobal|defmacro|defmacro\-|defn|defn\-|describe|dictionary\?|disasm|distinct|doc|doc\*|doc\-format|drop\-until|drop\-while|each|empty\?|env\-lookup|error|eval|eval\-string|even\?|every\?|extreme|false\?|fiber&#47;current|fiber&#47;maxstack|fiber&#47;new|fiber&#47;setmaxstack|fiber&#47;status|fiber\?|file&#47;close|file&#47;flush|file&#47;open|file&#47;popen|file&#47;read|file&#47;seek|file&#47;write|filter|find|find\-index|first|flatten|flatten\-into|for|frequencies|function\?|gccollect|gcinterval|gcsetinterval|generate|gensym|get|getline|hash|idempotent\?|identity|if\-let|if\-not|import|import\*|inc|indexed\?|interleave|interpose|invert|janet&#47;build|janet&#47;version|juxt|juxt\*|keep|keys|keyword|keyword\?|kvs|last|length|let|loop|macex|macex1|make\-env|map|mapcat|marshal|match|match\-1|math&#47;acos|math&#47;asin|math&#47;atan|math&#47;ceil|math&#47;cos|math&#47;e|math&#47;exp|math&#47;floor|math&#47;inf|math&#47;log|math&#47;log10|math&#47;pi|math&#47;pow|math&#47;random|math&#47;seedrandom|math&#47;sin|math&#47;sqrt|math&#47;tan|max|max\-order|merge|merge\-into|min|min\-order|module&#47;find|module&#47;native\-paths|module&#47;paths|native|neg\?|next|nil\?|not|not=|not==|number\?|odd\?|one\?|or|order&lt;|order&lt;=|order&gt;|order&gt;=|os&#47;clock|os&#47;cwd|os&#47;execute|os&#47;exit|os&#47;getenv|os&#47;setenv|os&#47;shell|os&#47;sleep|os&#47;time|os&#47;which|pairs|parser&#47;byte|parser&#47;consume|parser&#47;error|parser&#47;flush|parser&#47;has\-more|parser&#47;new|parser&#47;produce|parser&#47;state|parser&#47;status|parser&#47;where|partial|pos\?|postwalk|prewalk|print|process&#47;args|product|put|range|reduce|repl|require|resume|reverse|run\-context|scan\-number|sentinel|seq|some|sort|sorted|status\-pp|stderr|stdin|stdout|string|string&#47;ascii\-lower|string&#47;ascii\-upper|string&#47;bytes|string&#47;check\-set|string&#47;find|string&#47;find\-all|string&#47;from\-bytes|string&#47;join|string&#47;number|string&#47;pretty|string&#47;repeat|string&#47;replace|string&#47;replace\-all|string&#47;reverse|string&#47;slice|string&#47;split|string\?|struct|struct\?|sum|symbol|symbol\?|table|table&#47;getproto|table&#47;new|table&#47;rawget|table&#47;setproto|table&#47;to\-struct|table\?|take\-until|take\-while|true\?|try|tuple|tuple&#47;append|tuple&#47;prepend|tuple&#47;slice|tuple\?|type|unless|unmarshal|update|values|varglobal|walk|when|when\-let|with\-idemp|yield|zero\?|zipcoll)(?![\.:\w_\-=!@\$%^&amp;?|\\/&lt;&gt;*])</string>
<key>name</key> <key>name</key>
<string>keyword.control.janet</string> <string>keyword.control.janet</string>
</dict> </dict>

4
src/core/core.janet
View File

@ -1437,9 +1437,9 @@ value, one key will be ignored."
(if (= len 0) (set going false)) (if (= len 0) (set going false))
(while (> len pindex) (while (> len pindex)
(+= pindex (parser/consume p buf pindex)) (+= pindex (parser/consume p buf pindex))
(while (= (set pstatus (parser/status p)) :full) (while (parser/has-more p)
(eval1 (parser/produce p))) (eval1 (parser/produce p)))
(when (= pstatus :error) (when (= (parser/status p) :error)
(onstatus :parse (onstatus :parse
(string (parser/error p) (string (parser/error p)
" around byte " (parser/where p)) " around byte " (parser/where p))

31
src/core/parse.c
View File

@ -161,6 +161,8 @@ static void popstate(JanetParser *p, Janet val) {
janet_tuple_sm_end(janet_unwrap_tuple(val)) = (int32_t) p->offset; janet_tuple_sm_end(janet_unwrap_tuple(val)) = (int32_t) p->offset;
} }
newtop->argn++; newtop->argn++;
/* Keep track of number of values in the root state */
if (p->statecount == 1) p->pending++;
push_arg(p, val); push_arg(p, val);
return; return;
} else if (newtop->flags & PFLAG_READERMAC) { } else if (newtop->flags & PFLAG_READERMAC) {
@ -534,7 +536,6 @@ int janet_parser_consume(JanetParser *parser, uint8_t c) {
enum JanetParserStatus janet_parser_status(JanetParser *parser) { enum JanetParserStatus janet_parser_status(JanetParser *parser) {
if (parser->error) return JANET_PARSE_ERROR; if (parser->error) return JANET_PARSE_ERROR;
if (parser->statecount > 1) return JANET_PARSE_PENDING; if (parser->statecount > 1) return JANET_PARSE_PENDING;
if (parser->argcount) return JANET_PARSE_FULL;
return JANET_PARSE_ROOT; return JANET_PARSE_ROOT;
} }
@ -542,6 +543,7 @@ void janet_parser_flush(JanetParser *parser) {
parser->argcount = 0; parser->argcount = 0;
parser->statecount = 1; parser->statecount = 1;
parser->bufcount = 0; parser->bufcount = 0;
parser->pending = 0;
} }
const char *janet_parser_error(JanetParser *parser) { const char *janet_parser_error(JanetParser *parser) {
@ -558,12 +560,12 @@ const char *janet_parser_error(JanetParser *parser) {
Janet janet_parser_produce(JanetParser *parser) { Janet janet_parser_produce(JanetParser *parser) {
Janet ret; Janet ret;
size_t i; size_t i;
enum JanetParserStatus status = janet_parser_status(parser); if (parser->pending == 0) return janet_wrap_nil();
if (status != JANET_PARSE_FULL) return janet_wrap_nil();
ret = parser->args[0]; ret = parser->args[0];
for (i = 1; i < parser->argcount; i++) { for (i = 1; i < parser->argcount; i++) {
parser->args[i - 1] = parser->args[i]; parser->args[i - 1] = parser->args[i];
} }
parser->pending--;
parser->argcount--; parser->argcount--;
return ret; return ret;
} }
@ -581,6 +583,7 @@ void janet_parser_init(JanetParser *parser) {
parser->error = NULL; parser->error = NULL;
parser->lookback = -1; parser->lookback = -1;
parser->offset = 0; parser->offset = 0;
parser->pending = 0;
pushstate(parser, root, PFLAG_CONTAINER); pushstate(parser, root, PFLAG_CONTAINER);
} }
@ -664,6 +667,14 @@ static int cfun_consume(JanetArgs args) {
JANET_RETURN_INTEGER(args, i); JANET_RETURN_INTEGER(args, i);
} }
static int cfun_has_more(JanetArgs args) {
JanetParser *p;
JANET_FIXARITY(args, 1);
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
JANET_RETURN_BOOLEAN(args, janet_parser_has_more(p));
}
static int cfun_byte(JanetArgs args) { static int cfun_byte(JanetArgs args) {
int32_t i; int32_t i;
JanetParser *p; JanetParser *p;
@ -682,9 +693,6 @@ static int cfun_status(JanetArgs args) {
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype); JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
p = (JanetParser *) janet_unwrap_abstract(args.v[0]); p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
switch (janet_parser_status(p)) { switch (janet_parser_status(p)) {
case JANET_PARSE_FULL:
stat = "full";
break;
case JANET_PARSE_PENDING: case JANET_PARSE_PENDING:
stat = "pending"; stat = "pending";
break; break;
@ -776,6 +784,10 @@ static const JanetReg cfuns[] = {
"Creates and returns a new parser object. Parsers are state machines " "Creates and returns a new parser object. Parsers are state machines "
"that can receive bytes, and generate a stream of janet values. " "that can receive bytes, and generate a stream of janet values. "
}, },
{"parser/has-more", cfun_has_more,
"(parser/has-more parser)\n\n"
"Check if the parser has more values in the value queue."
},
{"parser/produce", cfun_produce, {"parser/produce", cfun_produce,
"(parser/produce parser)\n\n" "(parser/produce parser)\n\n"
"Dequeue the next value in the parse queue. Will return nil if " "Dequeue the next value in the parse queue. Will return nil if "
@ -795,14 +807,15 @@ static const JanetReg cfuns[] = {
{"parser/error", cfun_error, {"parser/error", cfun_error,
"(parser/error parser)\n\n" "(parser/error parser)\n\n"
"If the parser is in the error state, returns the message asscoiated with " "If the parser is in the error state, returns the message asscoiated with "
"that error. Otherwise, returns nil." "that error. Otherwise, returns nil. Also flushes the parser state and parser "
"queue, so be sure to handle everything in the queue before calling "
"parser/error."
}, },
{"parser/status", cfun_status, {"parser/status", cfun_status,
"(parser/status parser)\n\n" "(parser/status parser)\n\n"
"Gets the current status of the parser state machine. The status will " "Gets the current status of the parser state machine. The status will "
"be one of:\n\n" "be one of:\n\n"
"\t:full - there are values in the parse queue to be consumed.\n" "\t:pending - a value is being parsed.\n"
"\t:pending - no values in the queue but a value is being parsed.\n"
"\t:error - a parsing error was encountered.\n" "\t:error - a parsing error was encountered.\n"
"\t:root - the parser can either read more values or safely terminate." "\t:root - the parser can either read more values or safely terminate."
}, },

42
src/core/run.c
View File

@ -92,26 +92,29 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
janet_parser_init(&parser); janet_parser_init(&parser);
while (!errflags && !done) { while (!errflags && !done) {
switch (janet_parser_status(&parser)) {
case JANET_PARSE_FULL: /* Evaluate parsed values */
{ while (janet_parser_has_more(&parser)) {
Janet form = janet_parser_produce(&parser); Janet form = janet_parser_produce(&parser);
JanetCompileResult cres = janet_compile(form, env, where); JanetCompileResult cres = janet_compile(form, env, where);
if (cres.status == JANET_COMPILE_OK) { if (cres.status == JANET_COMPILE_OK) {
JanetFunction *f = janet_thunk(cres.funcdef); JanetFunction *f = janet_thunk(cres.funcdef);
JanetFiber *fiber = janet_fiber(f, 64); JanetFiber *fiber = janet_fiber(f, 64);
JanetSignal status = janet_continue(fiber, janet_wrap_nil(), &ret); JanetSignal status = janet_continue(fiber, janet_wrap_nil(), &ret);
if (status != JANET_SIGNAL_OK) { if (status != JANET_SIGNAL_OK) {
janet_stacktrace(fiber, "runtime", ret); janet_stacktrace(fiber, "runtime", ret);
errflags |= 0x01; errflags |= 0x01;
}
} else {
janet_stacktrace(cres.macrofiber, "compile",
janet_wrap_string(cres.error));
errflags |= 0x02;
}
} }
break; } else {
fprintf(stderr, "source path: %s\n", sourcePath);
janet_stacktrace(cres.macrofiber, "compile",
janet_wrap_string(cres.error));
errflags |= 0x02;
}
}
/* Dispatch based on parse state */
switch (janet_parser_status(&parser)) {
case JANET_PARSE_ERROR: case JANET_PARSE_ERROR:
errflags |= 0x04; errflags |= 0x04;
fprintf(stderr, "parse error: %s\n", janet_parser_error(&parser)); fprintf(stderr, "parse error: %s\n", janet_parser_error(&parser));
@ -137,6 +140,7 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
} }
break; break;
} }
} }
janet_parser_deinit(&parser); janet_parser_deinit(&parser);
if (where) janet_gcunroot(janet_wrap_string(where)); if (where) janet_gcunroot(janet_wrap_string(where));

3
src/include/janet/janet.h
View File

@ -717,7 +717,6 @@ typedef struct JanetParser JanetParser;
enum JanetParserStatus { enum JanetParserStatus {
JANET_PARSE_ROOT, JANET_PARSE_ROOT,
JANET_PARSE_ERROR, JANET_PARSE_ERROR,
JANET_PARSE_FULL,
JANET_PARSE_PENDING JANET_PARSE_PENDING
}; };
@ -734,6 +733,7 @@ struct JanetParser {
size_t bufcount; size_t bufcount;
size_t bufcap; size_t bufcap;
size_t offset; size_t offset;
size_t pending;
int lookback; int lookback;
}; };
@ -878,6 +878,7 @@ JANET_API Janet janet_parser_produce(JanetParser *parser);
JANET_API const char *janet_parser_error(JanetParser *parser); JANET_API const char *janet_parser_error(JanetParser *parser);
JANET_API void janet_parser_flush(JanetParser *parser); JANET_API void janet_parser_flush(JanetParser *parser);
JANET_API JanetParser *janet_check_parser(Janet x); JANET_API JanetParser *janet_check_parser(Janet x);
#define janet_parser_has_more(P) ((P)->pending)
/* Assembly */ /* Assembly */
#ifdef JANET_ASSEMBLER #ifdef JANET_ASSEMBLER