mirrors/janet
1
0
Fork 0
mirror of https://github.com/janet-lang/janet synced 2025-10-31 07:33:01 +00:00
Code Issues 1 Releases Wiki Activity

Source mapping v3 (performance improvements, better decoupling from parser).

Browse Source
This commit is contained in:
Calvin Rose
2018-06-29 10:37:50 -04:00
parent cac39aba67
commit f93c84c21f
9 changed files with 62 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/compiler/boot.dst
View File

@@ -937,8 +937,7 @@ onvalue."
(var going true)
# The parser object
(def p (parser.new 1))
(if where (parser.set-source p where))
(def p (parser.new))
# Fiber stream of characters
(var byteindex 0)
@@ -971,7 +970,7 @@ onvalue."
(defn eval1 [source]
(var good true)
(def f (fiber.new (fn []
(def res (compile source env p))
(def res (compile source env where))
(if (= (type res) :function)
(res)
(do

75
src/compiler/compile.c
View File

@@ -30,38 +30,32 @@
#undef DST_V_DEF_COPYMEM
#undef DST_V_DEF_FLATTENMEM
void dstc_ast_push(DstCompiler *c, Dst x) {
static void dstc_ast_push(DstCompiler *c, const Dst *tup) {
DstSourceMapping mapping;
if (c->result.status == DST_COMPILE_ERROR) {
return;
}
if (c->parser) {
int found = dst_parser_lookup(c->parser, x, &mapping);
if (!found) {
/* Duplicate last value */
if (dst_v_count(c->ast_stack)) {
mapping.start = dst_tuple_sm_start(tup);
mapping.end = dst_tuple_sm_end(tup);
if (mapping.start < 0 || mapping.end < 0) {
/* Reuse previous mapping */
mapping = dst_v_last(c->ast_stack);
} else {
mapping.start = -1;
mapping.end = -1;
}
}
} else {
mapping.start = -1;
mapping.end = -1;
}
dst_v_push(c->ast_stack, mapping);
c->current_mapping = mapping;
}
void dstc_ast_pop(DstCompiler *c) {
static void dstc_ast_pop(DstCompiler *c) {
if (c->result.status == DST_COMPILE_ERROR) {
return;
}
dst_v_pop(c->ast_stack);
}
DstSourceMapping dstc_ast(DstCompiler *c) {
return dst_v_last(c->ast_stack);
if (dst_v_count(c->ast_stack)) {
c->current_mapping = dst_v_last(c->ast_stack);
} else {
c->current_mapping.start = -1;
c->current_mapping.end = -1;
}
}
DstFopts dstc_fopts_default(DstCompiler *c) {
@@ -368,7 +362,7 @@ DstSlot dstc_resolve(
/* Emit a raw instruction with source mapping. */
void dstc_emit(DstCompiler *c, uint32_t instr) {
dst_v_push(c->buffer, instr);
dst_v_push(c->mapbuffer, dstc_ast(c));
dst_v_push(c->mapbuffer, c->current_mapping);
}
/* Add a constant to the current scope. Return the index of the constant. */
@@ -801,7 +795,6 @@ DstSlot dstc_value(DstFopts opts, Dst x) {
DstCompiler *c = opts.compiler;
int macrorecur = 0;
opts.compiler->recursion_guard--;
dstc_ast_push(c, x);
recur:
if (dstc_iserr(&opts)) {
return dstc_cslot(dst_wrap_nil());
@@ -827,6 +820,8 @@ recur:
DstSlot head;
DstFopts subopts = dstc_fopts_default(c);
const Dst *tup = dst_unwrap_tuple(x);
if (!macrorecur)
dstc_ast_push(c, tup);
/* Empty tuple is tuple literal */
if (dst_tuple_length(tup) == 0) {
compiled = 1;
@@ -872,6 +867,8 @@ recur:
ret = dstc_call(opts, dstc_toslots(c, tup + 1, dst_tuple_length(tup) - 1), head);
}
}
/* Pop source mapping for tuple */
dstc_ast_pop(c);
}
break;
case DST_ARRAY:
@@ -898,8 +895,6 @@ recur:
ret = opts.hint;
}
opts.compiler->recursion_guard++;
/* Only pop on good path in case of error. */
dstc_ast_pop(c);
return ret;
}
@@ -946,9 +941,7 @@ DstFuncDef *dstc_pop_funcdef(DstCompiler *c) {
}
/* Get source from parser */
if (c->parser && (c->parser->flags & DST_PARSEFLAG_SOURCEMAP)) {
def->source = c->parser->source;
}
def->source = c->source;
def->arity = 0;
def->flags = 0;
@@ -963,20 +956,22 @@ DstFuncDef *dstc_pop_funcdef(DstCompiler *c) {
}
/* Initialize a compiler */
static void dstc_init(DstCompiler *c, DstTable *env, DstParser *p) {
static void dstc_init(DstCompiler *c, DstTable *env, const uint8_t *where) {
c->scopes = NULL;
c->buffer = NULL;
c->mapbuffer = NULL;
c->recursion_guard = DST_RECURSION_GUARD;
c->env = env;
c->parser = p;
c->source = where;
c->ast_stack = NULL;
c->current_mapping.start = -1;
c->current_mapping.end = -1;
/* Init result */
c->result.error = NULL;
c->result.status = DST_COMPILE_OK;
c->result.funcdef = NULL;
c->result.error_start = -1;
c->result.error_end = -1;
c->result.error_mapping.start = -1;
c->result.error_mapping.end = -1;
}
/* Deinitialize a compiler struct */
@@ -990,12 +985,11 @@ static void dstc_deinit(DstCompiler *c) {
}
/* Compile a form. */
DstCompileResult dst_compile(Dst source, DstTable *env, int flags, DstParser *p) {
DstCompileResult dst_compile(Dst source, DstTable *env, const uint8_t *where) {
DstCompiler c;
DstFopts fopts;
(void) flags;
dstc_init(&c, env, p);
dstc_init(&c, env, where);
/* Push a function scope */
dstc_scope(&c, DST_SCOPE_FUNCTION | DST_SCOPE_TOP);
@@ -1013,9 +1007,7 @@ DstCompileResult dst_compile(Dst source, DstTable *env, int flags, DstParser *p)
def->name = dst_cstring("_thunk");
c.result.funcdef = def;
} else {
DstSourceMapping m = dstc_ast(&c);
c.result.error_start = m.start;
c.result.error_end = m.end;
c.result.error_mapping = c.current_mapping;
}
dstc_deinit(&c);
@@ -1031,19 +1023,18 @@ int dst_compile_cfun(DstArgs args) {
DST_MINARITY(args, 2);
DST_MAXARITY(args, 3);
DST_ARG_TABLE(env, args, 1);
DstParser *p = NULL;
const uint8_t *source = NULL;
if (args.n == 3) {
p = dst_check_parser(args.v[2]);
if (!p) DST_THROW(args, "expected parser for second argument");
DST_ARG_STRING(source, args, 2);
}
res = dst_compile(args.v[0], env, 0, p);
res = dst_compile(args.v[0], env, source);
if (res.status == DST_COMPILE_OK) {
DST_RETURN_FUNCTION(args, dst_thunk(res.funcdef));
} else {
t = dst_table(2);
dst_table_put(t, dst_csymbolv(":error"), dst_wrap_string(res.error));
dst_table_put(t, dst_csymbolv(":error-start"), dst_wrap_integer(res.error_start));
dst_table_put(t, dst_csymbolv(":error-end"), dst_wrap_integer(res.error_end));
dst_table_put(t, dst_csymbolv(":error-start"), dst_wrap_integer(res.error_mapping.start));
dst_table_put(t, dst_csymbolv(":error-end"), dst_wrap_integer(res.error_mapping.end));
DST_RETURN_TABLE(args, t);
}
}

10
src/compiler/compile.h
View File

@@ -104,12 +104,13 @@ struct DstCompiler {
/* Keep track of where we are in the source */
DstSourceMapping *ast_stack;
DstSourceMapping current_mapping;
/* Hold the environment */
DstTable *env;
/* Hold a reference to a parser if we need it */
DstParser *parser;
/* Name of source to attach to generated functions */
const uint8_t *source;
DstCompileResult result;
};
@@ -145,11 +146,6 @@ struct DstSpecial {
/****************************************************/
/* Manipulate the ast stack for source mapping reasons. */
void dstc_ast_push(DstCompiler *c, Dst x);
void dstc_ast_pop(DstCompiler *c);
DstSourceMapping dstc_ast(DstCompiler *c);
/* Get a cfunction optimizer. Return NULL if none exists. */
const DstCFunOptimizer *dstc_cfunopt(DstCFunction cfun);

9
src/compiler/run.c
View File

@@ -31,16 +31,16 @@ int dst_dobytes(DstTable *env, const uint8_t *bytes, int32_t len, const char *so
int32_t index = 0;
int dudeol = 0;
int done = 0;
const uint8_t *source = sourcePath ? dst_cstring(sourcePath) : NULL;
const uint8_t *where = sourcePath ? dst_cstring(sourcePath) : NULL;
if (where) dst_gcroot(dst_wrap_string(where));
dst_parser_init(&parser);
dst_parser_init(&parser, sourcePath ? DST_PARSEFLAG_SOURCEMAP : 0);
parser.source = source;
while (!errflags && !done) {
switch (dst_parser_status(&parser)) {
case DST_PARSE_FULL:
{
Dst form = dst_parser_produce(&parser);
DstCompileResult cres = dst_compile(form, env, 0, &parser);
DstCompileResult cres = dst_compile(form, env, where);
if (cres.status == DST_COMPILE_OK) {
DstFunction *f = dst_thunk(cres.funcdef);
DstFiber *fiber = dst_fiber(f, 64);
@@ -83,6 +83,7 @@ int dst_dobytes(DstTable *env, const uint8_t *bytes, int32_t len, const char *so
}
}
dst_parser_deinit(&parser);
if (where) dst_gcunroot(dst_wrap_string(where));
return errflags;
}

99
src/core/parse.c
View File

@@ -22,36 +22,6 @@
#include <dst/dst.h>
/* Add a value to the parsemap */
static void pm_put(DstParser *p, Dst key, int32_t start, int32_t end) {
int32_t nextid = p->pm_count;
if (!dst_checktype(key, DST_TUPLE))
return;
dst_tuple_id(dst_unwrap_tuple(key)) = nextid;
p->pm_count++;
if (nextid >= p->pm_capacity) {
int32_t newCapacity = 2 * nextid + 2;
DstSourceMapping *newPms = realloc(p->pms, sizeof(DstSourceMapping) * newCapacity);
if (!newPms) {
DST_OUT_OF_MEMORY;
}
p->pms = newPms;
}
p->pms[nextid].start = start;
p->pms[nextid].end = end;
}
/* Get a value from the parse map. The returned pointer
* should be read and discarded immediately. */
static DstSourceMapping *pm_get(DstParser *p, Dst ast) {
if (!dst_checktype(ast, DST_TUPLE))
return NULL;
int32_t id = dst_tuple_id(dst_unwrap_tuple(ast));
if (id < 0 || id >= p->pm_count)
return NULL;
return p->pms + id;
}
/* Quote a value */
static Dst quote(Dst x) {
Dst *t = dst_tuple_begin(2);
@@ -197,15 +167,19 @@ static void popstate(DstParser *p, Dst val) {
len = newtop->qcount;
/* Quote the returned value qcount times */
for (i = 0; i < len; i++) {
if (p->flags & DST_PARSEFLAG_SOURCEMAP)
pm_put(p, val, (int32_t) top.start, (int32_t) p->index);
if (dst_checktype(val, DST_TUPLE)) {
dst_tuple_sm_start(dst_unwrap_tuple(val)) = (int32_t) top.start;
dst_tuple_sm_end(dst_unwrap_tuple(val)) = (int32_t) p->index;
}
val = quote(val);
}
newtop->qcount = 0;
/* Ast wrap */
if (p->flags & DST_PARSEFLAG_SOURCEMAP)
pm_put(p, val, (int32_t) top.start, (int32_t) p->index);
if (dst_checktype(val, DST_TUPLE)) {
dst_tuple_sm_start(dst_unwrap_tuple(val)) = (int32_t) top.start;
dst_tuple_sm_end(dst_unwrap_tuple(val)) = (int32_t) p->index;
}
newtop->argn++;
push_arg(p, val);
@@ -590,16 +564,7 @@ Dst dst_parser_produce(DstParser *parser) {
return ret;
}
int dst_parser_lookup(DstParser *parser, Dst key, DstSourceMapping *out) {
DstSourceMapping *results = pm_get(parser, key);
if (results) {
*out = *results;
return 1;
}
return 0;
}
void dst_parser_init(DstParser *parser, int flags) {
void dst_parser_init(DstParser *parser) {
parser->args = NULL;
parser->states = NULL;
parser->buf = NULL;
@@ -612,12 +577,6 @@ void dst_parser_init(DstParser *parser, int flags) {
parser->error = NULL;
parser->index = 0;
parser->lookback = -1;
parser->flags = flags;
parser->source = NULL;
parser->pms = NULL;
parser->pm_count = 0;
parser->pm_capacity = 0;
pushstate(parser, root, PFLAG_CONTAINER);
}
@@ -626,7 +585,6 @@ void dst_parser_deinit(DstParser *parser) {
free(parser->args);
free(parser->buf);
free(parser->states);
free(parser->pms);
}
/* C functions */
@@ -635,9 +593,6 @@ static int parsermark(void *p, size_t size) {
size_t i;
DstParser *parser = (DstParser *)p;
(void) size;
if (parser->source) {
dst_mark(dst_wrap_string(parser->source));
}
for (i = 0; i < parser->argcount; i++) {
dst_mark(parser->args[i]);
}
@@ -668,13 +623,9 @@ DstParser *dst_check_parser(Dst x) {
/* C Function parser */
static int cfun_parser(DstArgs args) {
int flags = 0;
DST_MAXARITY(args, 1);
if (args.n == 1) {
DST_ARG_INTEGER(flags, args, 0);
}
DST_FIXARITY(args, 0);
DstParser *p = dst_abstract(&dst_parse_parsertype, sizeof(DstParser));
dst_parser_init(p, flags);
dst_parser_init(p);
DST_RETURN_ABSTRACT(args, p);
}
@@ -802,32 +753,6 @@ static int cfun_state(DstArgs args) {
DST_RETURN_STRING(args, str);
}
static int cfun_lookup(DstArgs args) {
DstParser *p;
DST_FIXARITY(args, 2);
DST_CHECKABSTRACT(args, 0, &dst_parse_parsertype);
p = (DstParser *) dst_unwrap_abstract(args.v[0]);
DstSourceMapping *results = pm_get(p, args.v[1]);
if (results) {
Dst t[2];
t[0] = dst_wrap_integer(results->start);
t[1] = dst_wrap_integer(results->end);
DST_RETURN_TUPLE(args, dst_tuple_n(t, 2));
}
DST_RETURN_NIL(args);
}
static int cfun_setsource(DstArgs args) {
DstParser *p;
const uint8_t *source;
DST_FIXARITY(args, 2);
DST_CHECKABSTRACT(args, 0, &dst_parse_parsertype);
p = (DstParser *) dst_unwrap_abstract(args.v[0]);
DST_ARG_STRING(source, args, 1);
p->source = source;
DST_RETURN(args, args.v[0]);
}
static const DstReg cfuns[] = {
{"parser.new", cfun_parser},
{"parser.produce", cfun_produce},
@@ -837,8 +762,6 @@ static const DstReg cfuns[] = {
{"parser.status", cfun_status},
{"parser.flush", cfun_flush},
{"parser.state", cfun_state},
{"parser.lookup", cfun_lookup},
{"parser.set-source", cfun_setsource},
{NULL, NULL}
};

3
src/core/tuple.c
View File

@@ -33,7 +33,8 @@ Dst *dst_tuple_begin(int32_t length) {
char *data = dst_gcalloc(DST_MEMORY_TUPLE, 4 * sizeof(int32_t) + length * sizeof(Dst));
Dst *tuple = (Dst *)(data + (4 * sizeof(int32_t)));
dst_tuple_length(tuple) = length;
dst_tuple_id(tuple) = -1;
dst_tuple_sm_start(tuple) = -1;
dst_tuple_sm_end(tuple) = -1;
return tuple;
}

6
src/include/dst/dst.h
View File

@@ -37,14 +37,13 @@ extern "C" {
#include "dsttypes.h"
/* Parsing */
void dst_parser_init(DstParser *parser, int flags);
void dst_parser_init(DstParser *parser);
void dst_parser_deinit(DstParser *parser);
int dst_parser_consume(DstParser *parser, uint8_t c);
enum DstParserStatus dst_parser_status(DstParser *parser);
Dst dst_parser_produce(DstParser *parser);
const char *dst_parser_error(DstParser *parser);
void dst_parser_flush(DstParser *parser);
int dst_parser_lookup(DstParser *parser, Dst key, DstSourceMapping *out);
DstParser *dst_check_parser(Dst x);
/* Number scanning */
@@ -81,7 +80,8 @@ int dst_buffer_push_u64(DstBuffer *buffer, uint64_t x);
#define dst_tuple_raw(t) ((int32_t *)(t) - 4)
#define dst_tuple_length(t) (dst_tuple_raw(t)[0])
#define dst_tuple_hash(t) ((dst_tuple_raw(t)[1]))
#define dst_tuple_id(t) ((dst_tuple_raw(t)[2]))
#define dst_tuple_sm_start(t) ((dst_tuple_raw(t)[2]))
#define dst_tuple_sm_end(t) ((dst_tuple_raw(t)[3]))
Dst *dst_tuple_begin(int32_t length);
const Dst *dst_tuple_end(Dst *tuple);
const Dst *dst_tuple_n(Dst *values, int32_t n);

5
src/include/dst/dstcompile.h
View File

@@ -39,10 +39,9 @@ struct DstCompileResult {
enum DstCompileStatus status;
DstFuncDef *funcdef;
const uint8_t *error;
int32_t error_start;
int32_t error_end;
DstSourceMapping error_mapping;
};
DstCompileResult dst_compile(Dst source, DstTable *env, int flags, DstParser *parser);
DstCompileResult dst_compile(Dst source, DstTable *env, const uint8_t *where);
int dst_compile_cfun(DstArgs args);
int dst_lib_compile(DstArgs args);

5
src/include/dst/dsttypes.h
View File

@@ -523,10 +523,6 @@ struct DstParser {
const char *error;
DstParseState *states;
uint8_t *buf;
const uint8_t *source; /* optional source path/string */
DstSourceMapping *pms;
int32_t pm_capacity;
int32_t pm_count;
size_t argcount;
size_t argcap;
size_t statecount;
@@ -535,7 +531,6 @@ struct DstParser {
size_t bufcap;
size_t index;
int lookback;
int flags;
};
/* Defines an abstract type */