mirror of
https://github.com/janet-lang/janet
synced 2024-06-26 15:13:16 +00:00
6b95326d7c
remove some complexity and unexpected behavior around numbers in general as all numbers are the same number type, IEEE 754 double precision numbers. Also update examples and tests, some of which were out of date. Some more testing may be needed for new changes to numbers.
838 lines
25 KiB
C
838 lines
25 KiB
C
/*
|
|
* Copyright (c) 2018 Calvin Rose
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to
|
|
* deal in the Software without restriction, including without limitation the
|
|
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
|
* sell copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
* IN THE SOFTWARE.
|
|
*/
|
|
|
|
#include <janet/janet.h>
|
|
|
|
/* Check if a character is whitespace */
|
|
static int is_whitespace(uint8_t c) {
|
|
return c == ' '
|
|
|| c == '\t'
|
|
|| c == '\n'
|
|
|| c == '\r'
|
|
|| c == '\0'
|
|
|| c == '\f';
|
|
}
|
|
|
|
/* Code generated by tools/symcharsgen.c.
|
|
* The table contains 256 bits, where each bit is 1
|
|
* if the corresponding ascci code is a symbol char, and 0
|
|
* if not. The upper characters are also considered symbol
|
|
* chars and are then checked for utf-8 compliance. */
|
|
static const uint32_t symchars[8] = {
|
|
0x00000000, 0xf7ffec72, 0xc7ffffff, 0x17fffffe,
|
|
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
|
|
};
|
|
|
|
/* Check if a character is a valid symbol character
|
|
* symbol chars are A-Z, a-z, 0-9, or one of !$&*+-./:<=>@\^_~| */
|
|
static int is_symbol_char(uint8_t c) {
|
|
return symchars[c >> 5] & (1 << (c & 0x1F));
|
|
}
|
|
|
|
/* Validate some utf8. Useful for identifiers. Only validates
|
|
* the encoding, does not check for valid codepoints (they
|
|
* are less well defined than the encoding). */
|
|
static int valid_utf8(const uint8_t *str, int32_t len) {
|
|
int32_t i = 0;
|
|
int32_t j;
|
|
while (i < len) {
|
|
int32_t nexti;
|
|
uint8_t c = str[i];
|
|
|
|
/* Check the number of bytes in code point */
|
|
if (c < 0x80) nexti = i + 1;
|
|
else if ((c >> 5) == 0x06) nexti = i + 2;
|
|
else if ((c >> 4) == 0x0E) nexti = i + 3;
|
|
else if ((c >> 3) == 0x1E) nexti = i + 4;
|
|
/* Don't allow 5 or 6 byte code points */
|
|
else return 0;
|
|
|
|
/* No overflow */
|
|
if (nexti > len) return 0;
|
|
|
|
/* Ensure trailing bytes are well formed (10XX XXXX) */
|
|
for (j = i + 1; j < nexti; j++) {
|
|
if ((str[j] >> 6) != 2) return 0;
|
|
}
|
|
|
|
/* Check for overlong encodings */
|
|
if ((nexti == i + 2) && str[i] < 0xC2) return 0;
|
|
if ((str[i] == 0xE0) && str[i + 1] < 0xA0) return 0;
|
|
if ((str[i] == 0xF0) && str[i + 1] < 0x90) return 0;
|
|
|
|
i = nexti;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
/* Get hex digit from a letter */
|
|
static int to_hex(uint8_t c) {
|
|
if (c >= '0' && c <= '9') {
|
|
return c - '0';
|
|
} else if (c >= 'A' && c <= 'F') {
|
|
return 10 + c - 'A';
|
|
} else if (c >= 'a' && c <= 'f') {
|
|
return 10 + c - 'a';
|
|
} else {
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
typedef int (*Consumer)(JanetParser *p, JanetParseState *state, uint8_t c);
|
|
struct JanetParseState {
|
|
int32_t counter;
|
|
int32_t argn;
|
|
int flags;
|
|
size_t start;
|
|
Consumer consumer;
|
|
};
|
|
|
|
/* Define a stack on the main parser struct */
|
|
#define DEF_PARSER_STACK(NAME, T, STACK, STACKCOUNT, STACKCAP) \
|
|
static void NAME(JanetParser *p, T x) { \
|
|
size_t oldcount = p->STACKCOUNT; \
|
|
size_t newcount = oldcount + 1; \
|
|
if (newcount > p->STACKCAP) { \
|
|
T *next; \
|
|
size_t newcap = 2 * newcount; \
|
|
next = realloc(p->STACK, sizeof(T) * newcap); \
|
|
if (NULL == next) { \
|
|
JANET_OUT_OF_MEMORY; \
|
|
} \
|
|
p->STACK = next; \
|
|
p->STACKCAP = newcap; \
|
|
} \
|
|
p->STACK[oldcount] = x; \
|
|
p->STACKCOUNT = newcount; \
|
|
}
|
|
|
|
DEF_PARSER_STACK(push_buf, uint8_t, buf, bufcount, bufcap)
|
|
DEF_PARSER_STACK(push_arg, Janet, args, argcount, argcap)
|
|
DEF_PARSER_STACK(_pushstate, JanetParseState, states, statecount, statecap)
|
|
|
|
#undef DEF_PARSER_STACK
|
|
|
|
#define PFLAG_CONTAINER 0x100
|
|
#define PFLAG_BUFFER 0x200
|
|
#define PFLAG_PARENS 0x400
|
|
#define PFLAG_SQRBRACKETS 0x800
|
|
#define PFLAG_CURLYBRACKETS 0x1000
|
|
#define PFLAG_STRING 0x2000
|
|
#define PFLAG_LONGSTRING 0x4000
|
|
#define PFLAG_READERMAC 0x8000
|
|
|
|
static void pushstate(JanetParser *p, Consumer consumer, int flags) {
|
|
JanetParseState s;
|
|
s.counter = 0;
|
|
s.argn = 0;
|
|
s.flags = flags;
|
|
s.consumer = consumer;
|
|
s.start = p->offset;
|
|
_pushstate(p, s);
|
|
}
|
|
|
|
static void popstate(JanetParser *p, Janet val) {
|
|
for (;;) {
|
|
JanetParseState top = p->states[--p->statecount];
|
|
JanetParseState *newtop = p->states + p->statecount - 1;
|
|
if (newtop->flags & PFLAG_CONTAINER) {
|
|
/* Source mapping info */
|
|
if (janet_checktype(val, JANET_TUPLE)) {
|
|
janet_tuple_sm_start(janet_unwrap_tuple(val)) = (int32_t) top.start;
|
|
janet_tuple_sm_end(janet_unwrap_tuple(val)) = (int32_t) p->offset;
|
|
}
|
|
newtop->argn++;
|
|
push_arg(p, val);
|
|
return;
|
|
} else if (newtop->flags & PFLAG_READERMAC) {
|
|
Janet *t = janet_tuple_begin(2);
|
|
int c = newtop->flags & 0xFF;
|
|
const char *which =
|
|
(c == '\'') ? "quote" :
|
|
(c == ',') ? "unquote" :
|
|
(c == ';') ? "splice" :
|
|
(c == '~') ? "quasiquote" : "<unknown>";
|
|
t[0] = janet_csymbolv(which);
|
|
t[1] = val;
|
|
/* Quote source mapping info */
|
|
janet_tuple_sm_start(t) = (int32_t) newtop->start;
|
|
janet_tuple_sm_end(t) = (int32_t) p->offset;
|
|
val = janet_wrap_tuple(janet_tuple_end(t));
|
|
} else {
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
static int checkescape(uint8_t c) {
|
|
switch (c) {
|
|
default: return -1;
|
|
case 'x': return 1;
|
|
case 'n': return '\n';
|
|
case 't': return '\t';
|
|
case 'r': return '\r';
|
|
case '0': return '\0';
|
|
case 'z': return '\0';
|
|
case 'f': return '\f';
|
|
case 'e': return 27;
|
|
case '"': return '"';
|
|
case '\\': return '\\';
|
|
}
|
|
}
|
|
|
|
/* Forward declare */
|
|
static int stringchar(JanetParser *p, JanetParseState *state, uint8_t c);
|
|
|
|
static int escapeh(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
int digit = to_hex(c);
|
|
if (digit < 0) {
|
|
p->error = "invalid hex digit in hex escape";
|
|
return 1;
|
|
}
|
|
state->argn = (state->argn << 4) + digit;;
|
|
state->counter--;
|
|
if (!state->counter) {
|
|
push_buf(p, (state->argn & 0xFF));
|
|
state->argn = 0;
|
|
state->consumer = stringchar;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
static int escape1(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
int e = checkescape(c);
|
|
if (e < 0) {
|
|
p->error = "invalid string escape sequence";
|
|
return 1;
|
|
}
|
|
if (c == 'x') {
|
|
state->counter = 2;
|
|
state->argn = 0;
|
|
state->consumer = escapeh;
|
|
} else {
|
|
push_buf(p, (uint8_t) e);
|
|
state->consumer = stringchar;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
static int stringend(JanetParser *p, JanetParseState *state) {
|
|
Janet ret;
|
|
if (state->flags & PFLAG_BUFFER) {
|
|
JanetBuffer *b = janet_buffer((int32_t)p->bufcount);
|
|
janet_buffer_push_bytes(b, p->buf, (int32_t)p->bufcount);
|
|
ret = janet_wrap_buffer(b);
|
|
} else {
|
|
ret = janet_wrap_string(janet_string(p->buf, (int32_t)p->bufcount));
|
|
}
|
|
p->bufcount = 0;
|
|
popstate(p, ret);
|
|
return 1;
|
|
}
|
|
|
|
static int stringchar(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
/* Enter escape */
|
|
if (c == '\\') {
|
|
state->consumer = escape1;
|
|
return 1;
|
|
}
|
|
/* String end */
|
|
if (c == '"') {
|
|
return stringend(p, state);
|
|
}
|
|
/* normal char */
|
|
if (c != '\n')
|
|
push_buf(p, c);
|
|
return 1;
|
|
}
|
|
|
|
/* Check for string equality in the buffer */
|
|
static int check_str_const(const char *cstr, const uint8_t *str, int32_t len) {
|
|
int32_t index;
|
|
for (index = 0; index < len; index++) {
|
|
uint8_t c = str[index];
|
|
uint8_t k = ((const uint8_t *)cstr)[index];
|
|
if (c < k) return -1;
|
|
if (c > k) return 1;
|
|
if (k == '\0') break;
|
|
}
|
|
return (cstr[index] == '\0') ? 0 : -1;
|
|
}
|
|
|
|
static int tokenchar(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
Janet ret;
|
|
double numval;
|
|
int32_t blen;
|
|
int scanerr;
|
|
if (is_symbol_char(c)) {
|
|
push_buf(p, (uint8_t) c);
|
|
if (c > 127) state->argn = 1; /* Use to indicate non ascii */
|
|
return 1;
|
|
}
|
|
/* Token finished */
|
|
blen = (int32_t) p->bufcount;
|
|
scanerr = 0;
|
|
numval = janet_scan_number(p->buf, blen, &scanerr);
|
|
if (!scanerr) {
|
|
ret = janet_wrap_number(numval);
|
|
} else if (!check_str_const("nil", p->buf, blen)) {
|
|
ret = janet_wrap_nil();
|
|
} else if (!check_str_const("false", p->buf, blen)) {
|
|
ret = janet_wrap_false();
|
|
} else if (!check_str_const("true", p->buf, blen)) {
|
|
ret = janet_wrap_true();
|
|
} else if (p->buf) {
|
|
if (p->buf[0] >= '0' && p->buf[0] <= '9') {
|
|
p->error = "symbol literal cannot start with a digit";
|
|
return 0;
|
|
} else {
|
|
/* Don't do full utf8 check unless we have seen non ascii characters. */
|
|
int valid = (!state->argn) || valid_utf8(p->buf, blen);
|
|
if (!valid) {
|
|
p->error = "invalid utf-8 in symbol";
|
|
return 0;
|
|
}
|
|
ret = janet_symbolv(p->buf, blen);
|
|
}
|
|
} else {
|
|
p->error = "empty symbol invalid";
|
|
return 0;
|
|
}
|
|
p->bufcount = 0;
|
|
popstate(p, ret);
|
|
return 0;
|
|
}
|
|
|
|
static int comment(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
(void) state;
|
|
if (c == '\n') p->statecount--;
|
|
return 1;
|
|
}
|
|
|
|
/* Forward declaration */
|
|
static int root(JanetParser *p, JanetParseState *state, uint8_t c);
|
|
|
|
static int dotuple(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
if (state->flags & PFLAG_SQRBRACKETS
|
|
? c == ']'
|
|
: c == ')') {
|
|
int32_t i;
|
|
Janet *ret = janet_tuple_begin(state->argn);
|
|
for (i = state->argn - 1; i >= 0; i--) {
|
|
ret[i] = p->args[--p->argcount];
|
|
}
|
|
popstate(p, janet_wrap_tuple(janet_tuple_end(ret)));
|
|
return 1;
|
|
}
|
|
return root(p, state, c);
|
|
}
|
|
|
|
static int doarray(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
if (state->flags & PFLAG_SQRBRACKETS
|
|
? c == ']'
|
|
: c == ')') {
|
|
int32_t i;
|
|
JanetArray *array = janet_array(state->argn);
|
|
for (i = state->argn - 1; i >= 0; i--) {
|
|
array->data[i] = p->args[--p->argcount];
|
|
}
|
|
array->count = state->argn;
|
|
popstate(p, janet_wrap_array(array));
|
|
return 1;
|
|
}
|
|
return root(p, state, c);
|
|
}
|
|
|
|
static int dostruct(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
if (c == '}') {
|
|
int32_t i;
|
|
JanetKV *st;
|
|
if (state->argn & 1) {
|
|
p->error = "struct literal expects even number of arguments";
|
|
return 1;
|
|
}
|
|
st = janet_struct_begin(state->argn >> 1);
|
|
for (i = state->argn; i > 0; i -= 2) {
|
|
Janet value = p->args[--p->argcount];
|
|
Janet key = p->args[--p->argcount];
|
|
janet_struct_put(st, key, value);
|
|
}
|
|
popstate(p, janet_wrap_struct(janet_struct_end(st)));
|
|
return 1;
|
|
}
|
|
return root(p, state, c);
|
|
}
|
|
|
|
static int dotable(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
if (c == '}') {
|
|
int32_t i;
|
|
JanetTable *table;
|
|
if (state->argn & 1) {
|
|
p->error = "table literal expects even number of arguments";
|
|
return 1;
|
|
}
|
|
table = janet_table(state->argn >> 1);
|
|
for (i = state->argn; i > 0; i -= 2) {
|
|
Janet value = p->args[--p->argcount];
|
|
Janet key = p->args[--p->argcount];
|
|
janet_table_put(table, key, value);
|
|
}
|
|
popstate(p, janet_wrap_table(table));
|
|
return 1;
|
|
}
|
|
return root(p, state, c);
|
|
}
|
|
|
|
#define PFLAG_INSTRING 0x100000
|
|
#define PFLAG_END_CANDIDATE 0x200000
|
|
static int longstring(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
if (state->flags & PFLAG_INSTRING) {
|
|
/* We are inside the long string */
|
|
if (c == '`') {
|
|
state->flags |= PFLAG_END_CANDIDATE;
|
|
state->flags &= ~PFLAG_INSTRING;
|
|
state->counter = 1; /* Use counter to keep track of number of '=' seen */
|
|
return 1;
|
|
}
|
|
push_buf(p, c);
|
|
return 1;
|
|
} else if (state->flags & PFLAG_END_CANDIDATE) {
|
|
int i;
|
|
/* We are checking a potential end of the string */
|
|
if (state->counter == state->argn) {
|
|
stringend(p, state);
|
|
return 0;
|
|
}
|
|
if (c == '`' && state->counter < state->argn) {
|
|
state->counter++;
|
|
return 1;
|
|
}
|
|
/* Failed end candidate */
|
|
for (i = 0; i < state->counter; i++) {
|
|
push_buf(p, '`');
|
|
}
|
|
push_buf(p, c);
|
|
state->counter = 0;
|
|
state->flags &= ~PFLAG_END_CANDIDATE;
|
|
state->flags |= PFLAG_INSTRING;
|
|
return 1;
|
|
} else {
|
|
/* We are at beginning of string */
|
|
state->argn++;
|
|
if (c != '`') {
|
|
state->flags |= PFLAG_INSTRING;
|
|
push_buf(p, c);
|
|
}
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
static int ampersand(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
(void) state;
|
|
p->statecount--;
|
|
switch (c) {
|
|
case '{':
|
|
pushstate(p, dotable, PFLAG_CONTAINER | PFLAG_CURLYBRACKETS);
|
|
return 1;
|
|
case '"':
|
|
pushstate(p, stringchar, PFLAG_BUFFER | PFLAG_STRING);
|
|
return 1;
|
|
case '`':
|
|
pushstate(p, longstring, PFLAG_BUFFER | PFLAG_LONGSTRING);
|
|
return 1;
|
|
case '[':
|
|
pushstate(p, doarray, PFLAG_CONTAINER | PFLAG_SQRBRACKETS);
|
|
return 1;
|
|
case '(':
|
|
pushstate(p, doarray, PFLAG_CONTAINER | PFLAG_PARENS);
|
|
return 1;
|
|
default:
|
|
break;
|
|
}
|
|
pushstate(p, tokenchar, 0);
|
|
push_buf(p, '@'); /* Push the leading ampersand that was dropped */
|
|
return 0;
|
|
}
|
|
|
|
/* The root state of the parser */
|
|
static int root(JanetParser *p, JanetParseState *state, uint8_t c) {
|
|
(void) state;
|
|
switch (c) {
|
|
default:
|
|
if (is_whitespace(c)) return 1;
|
|
if (!is_symbol_char(c)) {
|
|
p->error = "unexpected character";
|
|
return 1;
|
|
}
|
|
pushstate(p, tokenchar, 0);
|
|
return 0;
|
|
case '\'':
|
|
case ',':
|
|
case ';':
|
|
case '~':
|
|
pushstate(p, root, PFLAG_READERMAC | c);
|
|
return 1;
|
|
case '"':
|
|
pushstate(p, stringchar, PFLAG_STRING);
|
|
return 1;
|
|
case '#':
|
|
pushstate(p, comment, 0);
|
|
return 1;
|
|
case '@':
|
|
pushstate(p, ampersand, 0);
|
|
return 1;
|
|
case '`':
|
|
pushstate(p, longstring, PFLAG_LONGSTRING);
|
|
return 1;
|
|
case ')':
|
|
case ']':
|
|
case '}':
|
|
p->error = "mismatched delimiter";
|
|
return 1;
|
|
case '(':
|
|
pushstate(p, dotuple, PFLAG_CONTAINER | PFLAG_PARENS);
|
|
return 1;
|
|
case '[':
|
|
pushstate(p, dotuple, PFLAG_CONTAINER | PFLAG_SQRBRACKETS);
|
|
return 1;
|
|
case '{':
|
|
pushstate(p, dostruct, PFLAG_CONTAINER | PFLAG_CURLYBRACKETS);
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
int janet_parser_consume(JanetParser *parser, uint8_t c) {
|
|
int consumed = 0;
|
|
if (parser->error) return 0;
|
|
parser->offset++;
|
|
while (!consumed && !parser->error) {
|
|
JanetParseState *state = parser->states + parser->statecount - 1;
|
|
consumed = state->consumer(parser, state, c);
|
|
}
|
|
parser->lookback = c;
|
|
return 1;
|
|
}
|
|
|
|
enum JanetParserStatus janet_parser_status(JanetParser *parser) {
|
|
if (parser->error) return JANET_PARSE_ERROR;
|
|
if (parser->statecount > 1) return JANET_PARSE_PENDING;
|
|
if (parser->argcount) return JANET_PARSE_FULL;
|
|
return JANET_PARSE_ROOT;
|
|
}
|
|
|
|
void janet_parser_flush(JanetParser *parser) {
|
|
parser->argcount = 0;
|
|
parser->statecount = 1;
|
|
parser->bufcount = 0;
|
|
}
|
|
|
|
const char *janet_parser_error(JanetParser *parser) {
|
|
enum JanetParserStatus status = janet_parser_status(parser);
|
|
if (status == JANET_PARSE_ERROR) {
|
|
const char *e = parser->error;
|
|
parser->error = NULL;
|
|
janet_parser_flush(parser);
|
|
return e;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
Janet janet_parser_produce(JanetParser *parser) {
|
|
Janet ret;
|
|
size_t i;
|
|
enum JanetParserStatus status = janet_parser_status(parser);
|
|
if (status != JANET_PARSE_FULL) return janet_wrap_nil();
|
|
ret = parser->args[0];
|
|
for (i = 1; i < parser->argcount; i++) {
|
|
parser->args[i - 1] = parser->args[i];
|
|
}
|
|
parser->argcount--;
|
|
return ret;
|
|
}
|
|
|
|
void janet_parser_init(JanetParser *parser) {
|
|
parser->args = NULL;
|
|
parser->states = NULL;
|
|
parser->buf = NULL;
|
|
parser->argcount = 0;
|
|
parser->argcap = 0;
|
|
parser->bufcount = 0;
|
|
parser->bufcap = 0;
|
|
parser->statecount = 0;
|
|
parser->statecap = 0;
|
|
parser->error = NULL;
|
|
parser->lookback = -1;
|
|
parser->offset = 0;
|
|
|
|
pushstate(parser, root, PFLAG_CONTAINER);
|
|
}
|
|
|
|
void janet_parser_deinit(JanetParser *parser) {
|
|
free(parser->args);
|
|
free(parser->buf);
|
|
free(parser->states);
|
|
}
|
|
|
|
/* C functions */
|
|
|
|
static int parsermark(void *p, size_t size) {
|
|
size_t i;
|
|
JanetParser *parser = (JanetParser *)p;
|
|
(void) size;
|
|
for (i = 0; i < parser->argcount; i++) {
|
|
janet_mark(parser->args[i]);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int parsergc(void *p, size_t size) {
|
|
JanetParser *parser = (JanetParser *)p;
|
|
(void) size;
|
|
janet_parser_deinit(parser);
|
|
return 0;
|
|
}
|
|
|
|
static JanetAbstractType janet_parse_parsertype = {
|
|
":core/parser",
|
|
parsergc,
|
|
parsermark
|
|
};
|
|
|
|
JanetParser *janet_check_parser(Janet x) {
|
|
if (!janet_checktype(x, JANET_ABSTRACT))
|
|
return NULL;
|
|
void *abstract = janet_unwrap_abstract(x);
|
|
if (janet_abstract_type(abstract) != &janet_parse_parsertype)
|
|
return NULL;
|
|
return (JanetParser *)abstract;
|
|
}
|
|
|
|
/* C Function parser */
|
|
static int cfun_parser(JanetArgs args) {
|
|
JANET_FIXARITY(args, 0);
|
|
JanetParser *p = janet_abstract(&janet_parse_parsertype, sizeof(JanetParser));
|
|
janet_parser_init(p);
|
|
JANET_RETURN_ABSTRACT(args, p);
|
|
}
|
|
|
|
static int cfun_consume(JanetArgs args) {
|
|
const uint8_t *bytes;
|
|
int32_t len;
|
|
JanetParser *p;
|
|
int32_t i;
|
|
JANET_MINARITY(args, 2);
|
|
JANET_MAXARITY(args, 3);
|
|
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
|
|
p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
|
|
JANET_ARG_BYTES(bytes, len, args, 1);
|
|
if (args.n == 3) {
|
|
int32_t offset;
|
|
JANET_ARG_INTEGER(offset, args, 2);
|
|
if (offset < 0 || offset > len)
|
|
JANET_THROW(args, "invalid offset");
|
|
len -= offset;
|
|
bytes += offset;
|
|
}
|
|
for (i = 0; i < len; i++) {
|
|
janet_parser_consume(p, bytes[i]);
|
|
switch (janet_parser_status(p)) {
|
|
case JANET_PARSE_ROOT:
|
|
case JANET_PARSE_PENDING:
|
|
break;
|
|
default:
|
|
JANET_RETURN_INTEGER(args, i + 1);
|
|
}
|
|
}
|
|
JANET_RETURN_INTEGER(args, i);
|
|
}
|
|
|
|
static int cfun_byte(JanetArgs args) {
|
|
int32_t i;
|
|
JanetParser *p;
|
|
JANET_FIXARITY(args, 2);
|
|
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
|
|
p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
|
|
JANET_ARG_INTEGER(i, args, 1);
|
|
janet_parser_consume(p, 0xFF & i);
|
|
JANET_RETURN(args, args.v[0]);
|
|
}
|
|
|
|
static int cfun_status(JanetArgs args) {
|
|
const char *stat = NULL;
|
|
JanetParser *p;
|
|
JANET_FIXARITY(args, 1);
|
|
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
|
|
p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
|
|
switch (janet_parser_status(p)) {
|
|
case JANET_PARSE_FULL:
|
|
stat = ":full";
|
|
break;
|
|
case JANET_PARSE_PENDING:
|
|
stat = ":pending";
|
|
break;
|
|
case JANET_PARSE_ERROR:
|
|
stat = ":error";
|
|
break;
|
|
case JANET_PARSE_ROOT:
|
|
stat = ":root";
|
|
break;
|
|
}
|
|
JANET_RETURN_CSYMBOL(args, stat);
|
|
}
|
|
|
|
static int cfun_error(JanetArgs args) {
|
|
const char *err;
|
|
JanetParser *p;
|
|
JANET_FIXARITY(args, 1);
|
|
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
|
|
p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
|
|
err = janet_parser_error(p);
|
|
if (err) {
|
|
JANET_RETURN_CSYMBOL(args, err);
|
|
} else {
|
|
JANET_RETURN_NIL(args);
|
|
}
|
|
}
|
|
|
|
static int cfun_produce(JanetArgs args) {
|
|
Janet val;
|
|
JanetParser *p;
|
|
JANET_FIXARITY(args, 1);
|
|
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
|
|
p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
|
|
val = janet_parser_produce(p);
|
|
JANET_RETURN(args, val);
|
|
}
|
|
|
|
static int cfun_flush(JanetArgs args) {
|
|
JanetParser *p;
|
|
JANET_FIXARITY(args, 1);
|
|
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
|
|
p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
|
|
janet_parser_flush(p);
|
|
JANET_RETURN(args, args.v[0]);
|
|
}
|
|
|
|
static int cfun_where(JanetArgs args) {
|
|
JanetParser *p;
|
|
JANET_FIXARITY(args, 1);
|
|
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
|
|
p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
|
|
JANET_RETURN_INTEGER(args, p->offset);
|
|
}
|
|
|
|
static int cfun_state(JanetArgs args) {
|
|
size_t i;
|
|
const uint8_t *str;
|
|
size_t oldcount;
|
|
JanetParser *p;
|
|
JANET_FIXARITY(args, 1);
|
|
JANET_CHECKABSTRACT(args, 0, &janet_parse_parsertype);
|
|
p = (JanetParser *) janet_unwrap_abstract(args.v[0]);
|
|
oldcount = p->bufcount;
|
|
for (i = 0; i < p->statecount; i++) {
|
|
JanetParseState *s = p->states + i;
|
|
if (s->flags & PFLAG_PARENS) {
|
|
push_buf(p, '(');
|
|
} else if (s->flags & PFLAG_SQRBRACKETS) {
|
|
push_buf(p, '[');
|
|
} else if (s->flags & PFLAG_CURLYBRACKETS) {
|
|
push_buf(p, '{');
|
|
} else if (s->flags & PFLAG_STRING) {
|
|
push_buf(p, '"');
|
|
} else if (s->flags & PFLAG_LONGSTRING) {
|
|
int32_t i;
|
|
for (i = 0; i < s->argn; i++) {
|
|
push_buf(p, '`');
|
|
}
|
|
}
|
|
}
|
|
str = janet_string(p->buf + oldcount, (int32_t)(p->bufcount - oldcount));
|
|
p->bufcount = oldcount;
|
|
JANET_RETURN_STRING(args, str);
|
|
}
|
|
|
|
static const JanetReg cfuns[] = {
|
|
{"parser/new", cfun_parser,
|
|
"(parser/new)\n\n"
|
|
"Creates and returns a new parser object. Parsers are state machines "
|
|
"that can receive bytes, and generate a stream of janet values. "
|
|
},
|
|
{"parser/produce", cfun_produce,
|
|
"(parser/produce parser)\n\n"
|
|
"Dequeue the next value in the parse queue. Will return nil if "
|
|
"no parsed values are in the queue, otherwise will dequeue the "
|
|
"next value."
|
|
},
|
|
{"parser/consume", cfun_consume,
|
|
"(parser/consume parser bytes [, index])\n\n"
|
|
"Input bytes into the parser and parse them. Will not throw errors "
|
|
"if there is a parse error. Starts at the byte index given by index. Returns "
|
|
"the number of bytes read."
|
|
},
|
|
{"parser/byte", cfun_byte,
|
|
"(parser/byte parser b)\n\n"
|
|
"Input a single byte into the parser byte stream. Returns the parser."
|
|
},
|
|
{"parser/error", cfun_error,
|
|
"(parser/error parser)\n\n"
|
|
"If the parser is in the error state, returns the message asscoiated with "
|
|
"that error. Otherwise, returns nil."
|
|
},
|
|
{"parser/status", cfun_status,
|
|
"(parser/status parser)\n\n"
|
|
"Gets the current status of the parser state machine. The status will "
|
|
"be one of:\n\n"
|
|
"\t:full - there are values in the parse queue to be consumed.\n"
|
|
"\t:pending - no values in the queue but a value is being parsed.\n"
|
|
"\t:error - a parsing error was encountered.\n"
|
|
"\t:root - the parser can either read more values or safely terminate."
|
|
},
|
|
{"parser/flush", cfun_flush,
|
|
"(parser/flush parser)\n\n"
|
|
"Clears the parser state and parse queue. Can be used to reset the parser "
|
|
"if an error was encountered. Does not reset the line and column counter, so "
|
|
"to begin parsing in a new context, create a new parser."
|
|
},
|
|
{"parser/state", cfun_state,
|
|
"(parser/state parser)\n\n"
|
|
"Returns a string representation of the internal state of the parser. "
|
|
"Each byte in the string represents a nested data structure. For example, "
|
|
"if the parser state is '([\"', then the parser is in the middle of parsing a "
|
|
"string inside of square brackets inside parens. Can be used to augment a repl prompt."
|
|
},
|
|
{"parser/where", cfun_where,
|
|
"(parser/where parser)\n\n"
|
|
"Returns the current line number and column number of the parser's location "
|
|
"in the byte stream as a tuple (line, column). Lines and columns are counted from "
|
|
"1, (the first byte is line1, column 1) and a newline is considered ascii 0x0A."
|
|
},
|
|
{NULL, NULL, NULL}
|
|
};
|
|
|
|
/* Load the library */
|
|
int janet_lib_parse(JanetArgs args) {
|
|
JanetTable *env = janet_env(args);
|
|
janet_cfuns(env, NULL, cfuns);
|
|
return 0;
|
|
}
|