mirror of
https://github.com/janet-lang/janet
synced 2024-12-25 07:50:27 +00:00
Source mapping v3 (performance improvements, better decoupling from parser).
This commit is contained in:
Calvin Rose
parent
cac39aba67
commit
f93c84c21f
@ -937,8 +937,7 @@ onvalue."
|
||||
(var going true)
|
||||
|
||||
# The parser object
|
||||
(def p (parser.new 1))
|
||||
(if where (parser.set-source p where))
|
||||
(def p (parser.new))
|
||||
|
||||
# Fiber stream of characters
|
||||
(var byteindex 0)
|
||||
@ -971,7 +970,7 @@ onvalue."
|
||||
(defn eval1 [source]
|
||||
(var good true)
|
||||
(def f (fiber.new (fn []
|
||||
(def res (compile source env p))
|
||||
(def res (compile source env where))
|
||||
(if (= (type res) :function)
|
||||
(res)
|
||||
(do
|
||||
|
||||
@ -30,38 +30,32 @@
|
||||
#undef DST_V_DEF_COPYMEM
|
||||
#undef DST_V_DEF_FLATTENMEM
|
||||
|
||||
void dstc_ast_push(DstCompiler *c, Dst x) {
|
||||
static void dstc_ast_push(DstCompiler *c, const Dst *tup) {
|
||||
DstSourceMapping mapping;
|
||||
if (c->result.status == DST_COMPILE_ERROR) {
|
||||
return;
|
||||
}
|
||||
if (c->parser) {
|
||||
int found = dst_parser_lookup(c->parser, x, &mapping);
|
||||
if (!found) {
|
||||
/* Duplicate last value */
|
||||
if (dst_v_count(c->ast_stack)) {
|
||||
mapping = dst_v_last(c->ast_stack);
|
||||
} else {
|
||||
mapping.start = -1;
|
||||
mapping.end = -1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
mapping.start = -1;
|
||||
mapping.end = -1;
|
||||
mapping.start = dst_tuple_sm_start(tup);
|
||||
mapping.end = dst_tuple_sm_end(tup);
|
||||
if (mapping.start < 0 || mapping.end < 0) {
|
||||
/* Reuse previous mapping */
|
||||
mapping = dst_v_last(c->ast_stack);
|
||||
}
|
||||
dst_v_push(c->ast_stack, mapping);
|
||||
c->current_mapping = mapping;
|
||||
}
|
||||
|
||||
void dstc_ast_pop(DstCompiler *c) {
|
||||
static void dstc_ast_pop(DstCompiler *c) {
|
||||
if (c->result.status == DST_COMPILE_ERROR) {
|
||||
return;
|
||||
}
|
||||
dst_v_pop(c->ast_stack);
|
||||
}
|
||||
|
||||
DstSourceMapping dstc_ast(DstCompiler *c) {
|
||||
return dst_v_last(c->ast_stack);
|
||||
if (dst_v_count(c->ast_stack)) {
|
||||
c->current_mapping = dst_v_last(c->ast_stack);
|
||||
} else {
|
||||
c->current_mapping.start = -1;
|
||||
c->current_mapping.end = -1;
|
||||
}
|
||||
}
|
||||
|
||||
DstFopts dstc_fopts_default(DstCompiler *c) {
|
||||
@ -368,7 +362,7 @@ DstSlot dstc_resolve(
|
||||
/* Emit a raw instruction with source mapping. */
|
||||
void dstc_emit(DstCompiler *c, uint32_t instr) {
|
||||
dst_v_push(c->buffer, instr);
|
||||
dst_v_push(c->mapbuffer, dstc_ast(c));
|
||||
dst_v_push(c->mapbuffer, c->current_mapping);
|
||||
}
|
||||
|
||||
/* Add a constant to the current scope. Return the index of the constant. */
|
||||
@ -801,7 +795,6 @@ DstSlot dstc_value(DstFopts opts, Dst x) {
|
||||
DstCompiler *c = opts.compiler;
|
||||
int macrorecur = 0;
|
||||
opts.compiler->recursion_guard--;
|
||||
dstc_ast_push(c, x);
|
||||
recur:
|
||||
if (dstc_iserr(&opts)) {
|
||||
return dstc_cslot(dst_wrap_nil());
|
||||
@ -827,6 +820,8 @@ recur:
|
||||
DstSlot head;
|
||||
DstFopts subopts = dstc_fopts_default(c);
|
||||
const Dst *tup = dst_unwrap_tuple(x);
|
||||
if (!macrorecur)
|
||||
dstc_ast_push(c, tup);
|
||||
/* Empty tuple is tuple literal */
|
||||
if (dst_tuple_length(tup) == 0) {
|
||||
compiled = 1;
|
||||
@ -872,6 +867,8 @@ recur:
|
||||
ret = dstc_call(opts, dstc_toslots(c, tup + 1, dst_tuple_length(tup) - 1), head);
|
||||
}
|
||||
}
|
||||
/* Pop source mapping for tuple */
|
||||
dstc_ast_pop(c);
|
||||
}
|
||||
break;
|
||||
case DST_ARRAY:
|
||||
@ -898,8 +895,6 @@ recur:
|
||||
ret = opts.hint;
|
||||
}
|
||||
opts.compiler->recursion_guard++;
|
||||
/* Only pop on good path in case of error. */
|
||||
dstc_ast_pop(c);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -946,9 +941,7 @@ DstFuncDef *dstc_pop_funcdef(DstCompiler *c) {
|
||||
}
|
||||
|
||||
/* Get source from parser */
|
||||
if (c->parser && (c->parser->flags & DST_PARSEFLAG_SOURCEMAP)) {
|
||||
def->source = c->parser->source;
|
||||
}
|
||||
def->source = c->source;
|
||||
|
||||
def->arity = 0;
|
||||
def->flags = 0;
|
||||
@ -963,20 +956,22 @@ DstFuncDef *dstc_pop_funcdef(DstCompiler *c) {
|
||||
}
|
||||
|
||||
/* Initialize a compiler */
|
||||
static void dstc_init(DstCompiler *c, DstTable *env, DstParser *p) {
|
||||
static void dstc_init(DstCompiler *c, DstTable *env, const uint8_t *where) {
|
||||
c->scopes = NULL;
|
||||
c->buffer = NULL;
|
||||
c->mapbuffer = NULL;
|
||||
c->recursion_guard = DST_RECURSION_GUARD;
|
||||
c->env = env;
|
||||
c->parser = p;
|
||||
c->source = where;
|
||||
c->ast_stack = NULL;
|
||||
c->current_mapping.start = -1;
|
||||
c->current_mapping.end = -1;
|
||||
/* Init result */
|
||||
c->result.error = NULL;
|
||||
c->result.status = DST_COMPILE_OK;
|
||||
c->result.funcdef = NULL;
|
||||
c->result.error_start = -1;
|
||||
c->result.error_end = -1;
|
||||
c->result.error_mapping.start = -1;
|
||||
c->result.error_mapping.end = -1;
|
||||
}
|
||||
|
||||
/* Deinitialize a compiler struct */
|
||||
@ -990,12 +985,11 @@ static void dstc_deinit(DstCompiler *c) {
|
||||
}
|
||||
|
||||
/* Compile a form. */
|
||||
DstCompileResult dst_compile(Dst source, DstTable *env, int flags, DstParser *p) {
|
||||
DstCompileResult dst_compile(Dst source, DstTable *env, const uint8_t *where) {
|
||||
DstCompiler c;
|
||||
DstFopts fopts;
|
||||
(void) flags;
|
||||
|
||||
dstc_init(&c, env, p);
|
||||
dstc_init(&c, env, where);
|
||||
|
||||
/* Push a function scope */
|
||||
dstc_scope(&c, DST_SCOPE_FUNCTION | DST_SCOPE_TOP);
|
||||
@ -1013,9 +1007,7 @@ DstCompileResult dst_compile(Dst source, DstTable *env, int flags, DstParser *p)
|
||||
def->name = dst_cstring("_thunk");
|
||||
c.result.funcdef = def;
|
||||
} else {
|
||||
DstSourceMapping m = dstc_ast(&c);
|
||||
c.result.error_start = m.start;
|
||||
c.result.error_end = m.end;
|
||||
c.result.error_mapping = c.current_mapping;
|
||||
}
|
||||
|
||||
dstc_deinit(&c);
|
||||
@ -1031,19 +1023,18 @@ int dst_compile_cfun(DstArgs args) {
|
||||
DST_MINARITY(args, 2);
|
||||
DST_MAXARITY(args, 3);
|
||||
DST_ARG_TABLE(env, args, 1);
|
||||
DstParser *p = NULL;
|
||||
const uint8_t *source = NULL;
|
||||
if (args.n == 3) {
|
||||
p = dst_check_parser(args.v[2]);
|
||||
if (!p) DST_THROW(args, "expected parser for second argument");
|
||||
DST_ARG_STRING(source, args, 2);
|
||||
}
|
||||
res = dst_compile(args.v[0], env, 0, p);
|
||||
res = dst_compile(args.v[0], env, source);
|
||||
if (res.status == DST_COMPILE_OK) {
|
||||
DST_RETURN_FUNCTION(args, dst_thunk(res.funcdef));
|
||||
} else {
|
||||
t = dst_table(2);
|
||||
dst_table_put(t, dst_csymbolv(":error"), dst_wrap_string(res.error));
|
||||
dst_table_put(t, dst_csymbolv(":error-start"), dst_wrap_integer(res.error_start));
|
||||
dst_table_put(t, dst_csymbolv(":error-end"), dst_wrap_integer(res.error_end));
|
||||
dst_table_put(t, dst_csymbolv(":error-start"), dst_wrap_integer(res.error_mapping.start));
|
||||
dst_table_put(t, dst_csymbolv(":error-end"), dst_wrap_integer(res.error_mapping.end));
|
||||
DST_RETURN_TABLE(args, t);
|
||||
}
|
||||
}
|
||||
|
||||
@ -104,12 +104,13 @@ struct DstCompiler {
|
||||
|
||||
/* Keep track of where we are in the source */
|
||||
DstSourceMapping *ast_stack;
|
||||
DstSourceMapping current_mapping;
|
||||
|
||||
/* Hold the environment */
|
||||
DstTable *env;
|
||||
|
||||
/* Hold a reference to a parser if we need it */
|
||||
DstParser *parser;
|
||||
/* Name of source to attach to generated functions */
|
||||
const uint8_t *source;
|
||||
|
||||
DstCompileResult result;
|
||||
};
|
||||
@ -145,11 +146,6 @@ struct DstSpecial {
|
||||
|
||||
/****************************************************/
|
||||
|
||||
/* Manipulate the ast stack for source mapping reasons. */
|
||||
void dstc_ast_push(DstCompiler *c, Dst x);
|
||||
void dstc_ast_pop(DstCompiler *c);
|
||||
DstSourceMapping dstc_ast(DstCompiler *c);
|
||||
|
||||
/* Get a cfunction optimizer. Return NULL if none exists. */
|
||||
const DstCFunOptimizer *dstc_cfunopt(DstCFunction cfun);
|
||||
|
||||
|
||||
@ -31,16 +31,16 @@ int dst_dobytes(DstTable *env, const uint8_t *bytes, int32_t len, const char *so
|
||||
int32_t index = 0;
|
||||
int dudeol = 0;
|
||||
int done = 0;
|
||||
const uint8_t *source = sourcePath ? dst_cstring(sourcePath) : NULL;
|
||||
const uint8_t *where = sourcePath ? dst_cstring(sourcePath) : NULL;
|
||||
if (where) dst_gcroot(dst_wrap_string(where));
|
||||
dst_parser_init(&parser);
|
||||
|
||||
dst_parser_init(&parser, sourcePath ? DST_PARSEFLAG_SOURCEMAP : 0);
|
||||
parser.source = source;
|
||||
while (!errflags && !done) {
|
||||
switch (dst_parser_status(&parser)) {
|
||||
case DST_PARSE_FULL:
|
||||
{
|
||||
Dst form = dst_parser_produce(&parser);
|
||||
DstCompileResult cres = dst_compile(form, env, 0, &parser);
|
||||
DstCompileResult cres = dst_compile(form, env, where);
|
||||
if (cres.status == DST_COMPILE_OK) {
|
||||
DstFunction *f = dst_thunk(cres.funcdef);
|
||||
DstFiber *fiber = dst_fiber(f, 64);
|
||||
@ -83,6 +83,7 @@ int dst_dobytes(DstTable *env, const uint8_t *bytes, int32_t len, const char *so
|
||||
}
|
||||
}
|
||||
dst_parser_deinit(&parser);
|
||||
if (where) dst_gcunroot(dst_wrap_string(where));
|
||||
return errflags;
|
||||
}
|
||||
|
||||
|
||||
@ -22,36 +22,6 @@
|
||||
|
||||
#include <dst/dst.h>
|
||||
|
||||
/* Add a value to the parsemap */
|
||||
static void pm_put(DstParser *p, Dst key, int32_t start, int32_t end) {
|
||||
int32_t nextid = p->pm_count;
|
||||
if (!dst_checktype(key, DST_TUPLE))
|
||||
return;
|
||||
dst_tuple_id(dst_unwrap_tuple(key)) = nextid;
|
||||
p->pm_count++;
|
||||
if (nextid >= p->pm_capacity) {
|
||||
int32_t newCapacity = 2 * nextid + 2;
|
||||
DstSourceMapping *newPms = realloc(p->pms, sizeof(DstSourceMapping) * newCapacity);
|
||||
if (!newPms) {
|
||||
DST_OUT_OF_MEMORY;
|
||||
}
|
||||
p->pms = newPms;
|
||||
}
|
||||
p->pms[nextid].start = start;
|
||||
p->pms[nextid].end = end;
|
||||
}
|
||||
|
||||
/* Get a value from the parse map. The returned pointer
|
||||
* should be read and discarded immediately. */
|
||||
static DstSourceMapping *pm_get(DstParser *p, Dst ast) {
|
||||
if (!dst_checktype(ast, DST_TUPLE))
|
||||
return NULL;
|
||||
int32_t id = dst_tuple_id(dst_unwrap_tuple(ast));
|
||||
if (id < 0 || id >= p->pm_count)
|
||||
return NULL;
|
||||
return p->pms + id;
|
||||
}
|
||||
|
||||
/* Quote a value */
|
||||
static Dst quote(Dst x) {
|
||||
Dst *t = dst_tuple_begin(2);
|
||||
@ -197,15 +167,19 @@ static void popstate(DstParser *p, Dst val) {
|
||||
len = newtop->qcount;
|
||||
/* Quote the returned value qcount times */
|
||||
for (i = 0; i < len; i++) {
|
||||
if (p->flags & DST_PARSEFLAG_SOURCEMAP)
|
||||
pm_put(p, val, (int32_t) top.start, (int32_t) p->index);
|
||||
if (dst_checktype(val, DST_TUPLE)) {
|
||||
dst_tuple_sm_start(dst_unwrap_tuple(val)) = (int32_t) top.start;
|
||||
dst_tuple_sm_end(dst_unwrap_tuple(val)) = (int32_t) p->index;
|
||||
}
|
||||
val = quote(val);
|
||||
}
|
||||
newtop->qcount = 0;
|
||||
|
||||
/* Ast wrap */
|
||||
if (p->flags & DST_PARSEFLAG_SOURCEMAP)
|
||||
pm_put(p, val, (int32_t) top.start, (int32_t) p->index);
|
||||
if (dst_checktype(val, DST_TUPLE)) {
|
||||
dst_tuple_sm_start(dst_unwrap_tuple(val)) = (int32_t) top.start;
|
||||
dst_tuple_sm_end(dst_unwrap_tuple(val)) = (int32_t) p->index;
|
||||
}
|
||||
|
||||
newtop->argn++;
|
||||
push_arg(p, val);
|
||||
@ -590,16 +564,7 @@ Dst dst_parser_produce(DstParser *parser) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
int dst_parser_lookup(DstParser *parser, Dst key, DstSourceMapping *out) {
|
||||
DstSourceMapping *results = pm_get(parser, key);
|
||||
if (results) {
|
||||
*out = *results;
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void dst_parser_init(DstParser *parser, int flags) {
|
||||
void dst_parser_init(DstParser *parser) {
|
||||
parser->args = NULL;
|
||||
parser->states = NULL;
|
||||
parser->buf = NULL;
|
||||
@ -612,12 +577,6 @@ void dst_parser_init(DstParser *parser, int flags) {
|
||||
parser->error = NULL;
|
||||
parser->index = 0;
|
||||
parser->lookback = -1;
|
||||
parser->flags = flags;
|
||||
parser->source = NULL;
|
||||
|
||||
parser->pms = NULL;
|
||||
parser->pm_count = 0;
|
||||
parser->pm_capacity = 0;
|
||||
|
||||
pushstate(parser, root, PFLAG_CONTAINER);
|
||||
}
|
||||
@ -626,7 +585,6 @@ void dst_parser_deinit(DstParser *parser) {
|
||||
free(parser->args);
|
||||
free(parser->buf);
|
||||
free(parser->states);
|
||||
free(parser->pms);
|
||||
}
|
||||
|
||||
/* C functions */
|
||||
@ -635,9 +593,6 @@ static int parsermark(void *p, size_t size) {
|
||||
size_t i;
|
||||
DstParser *parser = (DstParser *)p;
|
||||
(void) size;
|
||||
if (parser->source) {
|
||||
dst_mark(dst_wrap_string(parser->source));
|
||||
}
|
||||
for (i = 0; i < parser->argcount; i++) {
|
||||
dst_mark(parser->args[i]);
|
||||
}
|
||||
@ -668,13 +623,9 @@ DstParser *dst_check_parser(Dst x) {
|
||||
|
||||
/* C Function parser */
|
||||
static int cfun_parser(DstArgs args) {
|
||||
int flags = 0;
|
||||
DST_MAXARITY(args, 1);
|
||||
if (args.n == 1) {
|
||||
DST_ARG_INTEGER(flags, args, 0);
|
||||
}
|
||||
DST_FIXARITY(args, 0);
|
||||
DstParser *p = dst_abstract(&dst_parse_parsertype, sizeof(DstParser));
|
||||
dst_parser_init(p, flags);
|
||||
dst_parser_init(p);
|
||||
DST_RETURN_ABSTRACT(args, p);
|
||||
}
|
||||
|
||||
@ -802,32 +753,6 @@ static int cfun_state(DstArgs args) {
|
||||
DST_RETURN_STRING(args, str);
|
||||
}
|
||||
|
||||
static int cfun_lookup(DstArgs args) {
|
||||
DstParser *p;
|
||||
DST_FIXARITY(args, 2);
|
||||
DST_CHECKABSTRACT(args, 0, &dst_parse_parsertype);
|
||||
p = (DstParser *) dst_unwrap_abstract(args.v[0]);
|
||||
DstSourceMapping *results = pm_get(p, args.v[1]);
|
||||
if (results) {
|
||||
Dst t[2];
|
||||
t[0] = dst_wrap_integer(results->start);
|
||||
t[1] = dst_wrap_integer(results->end);
|
||||
DST_RETURN_TUPLE(args, dst_tuple_n(t, 2));
|
||||
}
|
||||
DST_RETURN_NIL(args);
|
||||
}
|
||||
|
||||
static int cfun_setsource(DstArgs args) {
|
||||
DstParser *p;
|
||||
const uint8_t *source;
|
||||
DST_FIXARITY(args, 2);
|
||||
DST_CHECKABSTRACT(args, 0, &dst_parse_parsertype);
|
||||
p = (DstParser *) dst_unwrap_abstract(args.v[0]);
|
||||
DST_ARG_STRING(source, args, 1);
|
||||
p->source = source;
|
||||
DST_RETURN(args, args.v[0]);
|
||||
}
|
||||
|
||||
static const DstReg cfuns[] = {
|
||||
{"parser.new", cfun_parser},
|
||||
{"parser.produce", cfun_produce},
|
||||
@ -837,8 +762,6 @@ static const DstReg cfuns[] = {
|
||||
{"parser.status", cfun_status},
|
||||
{"parser.flush", cfun_flush},
|
||||
{"parser.state", cfun_state},
|
||||
{"parser.lookup", cfun_lookup},
|
||||
{"parser.set-source", cfun_setsource},
|
||||
{NULL, NULL}
|
||||
};
|
||||
|
||||
|
||||
@ -33,7 +33,8 @@ Dst *dst_tuple_begin(int32_t length) {
|
||||
char *data = dst_gcalloc(DST_MEMORY_TUPLE, 4 * sizeof(int32_t) + length * sizeof(Dst));
|
||||
Dst *tuple = (Dst *)(data + (4 * sizeof(int32_t)));
|
||||
dst_tuple_length(tuple) = length;
|
||||
dst_tuple_id(tuple) = -1;
|
||||
dst_tuple_sm_start(tuple) = -1;
|
||||
dst_tuple_sm_end(tuple) = -1;
|
||||
return tuple;
|
||||
}
|
||||
|
||||
|
||||
@ -37,14 +37,13 @@ extern "C" {
|
||||
#include "dsttypes.h"
|
||||
|
||||
/* Parsing */
|
||||
void dst_parser_init(DstParser *parser, int flags);
|
||||
void dst_parser_init(DstParser *parser);
|
||||
void dst_parser_deinit(DstParser *parser);
|
||||
int dst_parser_consume(DstParser *parser, uint8_t c);
|
||||
enum DstParserStatus dst_parser_status(DstParser *parser);
|
||||
Dst dst_parser_produce(DstParser *parser);
|
||||
const char *dst_parser_error(DstParser *parser);
|
||||
void dst_parser_flush(DstParser *parser);
|
||||
int dst_parser_lookup(DstParser *parser, Dst key, DstSourceMapping *out);
|
||||
DstParser *dst_check_parser(Dst x);
|
||||
|
||||
/* Number scanning */
|
||||
@ -81,7 +80,8 @@ int dst_buffer_push_u64(DstBuffer *buffer, uint64_t x);
|
||||
#define dst_tuple_raw(t) ((int32_t *)(t) - 4)
|
||||
#define dst_tuple_length(t) (dst_tuple_raw(t)[0])
|
||||
#define dst_tuple_hash(t) ((dst_tuple_raw(t)[1]))
|
||||
#define dst_tuple_id(t) ((dst_tuple_raw(t)[2]))
|
||||
#define dst_tuple_sm_start(t) ((dst_tuple_raw(t)[2]))
|
||||
#define dst_tuple_sm_end(t) ((dst_tuple_raw(t)[3]))
|
||||
Dst *dst_tuple_begin(int32_t length);
|
||||
const Dst *dst_tuple_end(Dst *tuple);
|
||||
const Dst *dst_tuple_n(Dst *values, int32_t n);
|
||||
|
||||
@ -39,10 +39,9 @@ struct DstCompileResult {
|
||||
enum DstCompileStatus status;
|
||||
DstFuncDef *funcdef;
|
||||
const uint8_t *error;
|
||||
int32_t error_start;
|
||||
int32_t error_end;
|
||||
DstSourceMapping error_mapping;
|
||||
};
|
||||
DstCompileResult dst_compile(Dst source, DstTable *env, int flags, DstParser *parser);
|
||||
DstCompileResult dst_compile(Dst source, DstTable *env, const uint8_t *where);
|
||||
int dst_compile_cfun(DstArgs args);
|
||||
int dst_lib_compile(DstArgs args);
|
||||
|
||||
|
||||
@ -523,10 +523,6 @@ struct DstParser {
|
||||
const char *error;
|
||||
DstParseState *states;
|
||||
uint8_t *buf;
|
||||
const uint8_t *source; /* optional source path/string */
|
||||
DstSourceMapping *pms;
|
||||
int32_t pm_capacity;
|
||||
int32_t pm_count;
|
||||
size_t argcount;
|
||||
size_t argcap;
|
||||
size_t statecount;
|
||||
@ -535,7 +531,6 @@ struct DstParser {
|
||||
size_t bufcap;
|
||||
size_t index;
|
||||
int lookback;
|
||||
int flags;
|
||||
};
|
||||
|
||||
/* Defines an abstract type */
|
||||
|
||||
Loading…
Reference in New Issue
Block a user