diff --git a/Makefile b/Makefile index 6143a664..173f91fe 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ else CLIBS:=$(CLIBS) -lrt endif -$(shell mkdir -p build/core build/mainclient build/webclient) +$(shell mkdir -p build/core build/mainclient build/webclient build/boot) # Source headers JANET_HEADERS=$(sort $(wildcard src/include/janet/*.h)) @@ -60,14 +60,33 @@ JANET_WEBCLIENT_SOURCES=$(sort $(wildcard src/webclient/*.c)) all: $(JANET_TARGET) $(JANET_LIBRARY) +################################################################## +##### The bootstrap interpreter that compiles the core image ##### +################################################################## + +JANET_BOOT_OBJECTS=$(patsubst src/%.c,build/%.boot.o,$(JANET_CORE_SOURCES) src/boot/boot.c) \ + build/core.gen.o \ + build/boot.gen.o + +build/%.boot.o: src/%.c + $(CC) $(CFLAGS) -DJANET_BOOTSTRAP -o $@ -c $< + +build/janet_boot: $(JANET_BOOT_OBJECTS) + $(CC) $(CFLAGS) -DJANET_BOOTSTRAP -o $@ $^ $(CLIBS) + +# Now the reason we bootstrap in the first place +build/core_image.c: build/janet_boot + build/janet_boot + ########################################################## ##### The main interpreter program and shared object ##### ########################################################## -JANET_CORE_OBJECTS=$(patsubst src/%.c,build/%.o,$(JANET_CORE_SOURCES)) build/core.gen.o +JANET_CORE_OBJECTS=$(patsubst src/%.c,build/%.o,$(JANET_CORE_SOURCES)) build/core_image.o JANET_MAINCLIENT_OBJECTS=$(patsubst src/%.c,build/%.o,$(JANET_MAINCLIENT_SOURCES)) build/init.gen.o -%.gen.o: %.gen.c +# Compile the core image generated by the bootstrap build +build/core_image.o: build/core_image.c $(JANET_HEADERS) $(JANET_LOCAL_HEADERS) $(CC) $(CFLAGS) -o $@ -c $< build/%.o: src/%.c $(JANET_HEADERS) $(JANET_LOCAL_HEADERS) @@ -109,6 +128,9 @@ emscripten: $(JANET_EMTARGET) ##### Generated C files ##### ############################# +%.gen.o: %.gen.c + $(CC) $(CFLAGS) -o $@ -c $< + build/xxd: tools/xxd.c $(CC) $< -o $@ @@ -118,12 +140,14 @@ build/init.gen.c: src/mainclient/init.janet build/xxd build/xxd $< $@ janet_gen_init build/webinit.gen.c: src/webclient/webinit.janet build/xxd build/xxd $< $@ janet_gen_webinit +build/boot.gen.c: src/boot/boot.janet build/xxd + build/xxd $< $@ janet_gen_boot ######################## ##### Amalgamation ##### ######################## -amalg: build/janet.c build/janet.h +amalg: build/janet.c build/janet.h build/core_image.c build/janet.c: $(JANET_LOCAL_HEADERS) $(JANET_CORE_SOURCES) tools/amalg.janet $(JANET_TARGET) $(JANET_TARGET) tools/amalg.janet > $@ diff --git a/src/boot/boot.c b/src/boot/boot.c new file mode 100644 index 00000000..019d84bb --- /dev/null +++ b/src/boot/boot.c @@ -0,0 +1,43 @@ +/* +* Copyright (c) 2019 Calvin Rose +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +* IN THE SOFTWARE. +*/ + +#include + +extern const unsigned char *janet_gen_boot; +extern int32_t janet_gen_boot_size; + +int main() { + int status; + JanetTable *env; + + /* Set up VM */ + janet_init(); + env = janet_core_env(); + + /* Run bootstrap script to generate core image */ + status = janet_dobytes(env, janet_gen_boot, janet_gen_boot_size, "boot.janet", NULL); + + /* Deinitialize vm */ + janet_deinit(); + + return status; +} diff --git a/src/boot/boot.janet b/src/boot/boot.janet new file mode 100644 index 00000000..4cfae8f2 --- /dev/null +++ b/src/boot/boot.janet @@ -0,0 +1,40 @@ +# Copyright (C) Calvin Rose 2019 + +# The bootstrap script is used to produce the source file for +# embedding the core image. + +# Tool to dump a marshalled version of the janet core to stdout. The +# image should eventually allow janet to be started from a pre-compiled +# image rather than recompiled every time from the embedded source. More +# work will go into shrinking the image (it isn't currently that large but +# could be smaller), creating the mechanism to load the image, and modifying +# the build process to compile janet with a built image rather than +# embedded source. + +# Get image. This image contains as much of the core library and documentation that +# can be written to an image (no cfunctions, no abstracts (stdout, stdin, stderr)), +# everything else goes. Cfunctions and abstracts will be referenced from a registry +# table which will be generated on janet startup. +(do + (def image (let [env-pairs (pairs (env-lookup *env*)) + essential-pairs (filter (fn [[k v]] (or (cfunction? v) (abstract? v))) env-pairs) + lookup (table ;(mapcat identity essential-pairs)) + reverse-lookup (invert lookup)] + (marshal *env* reverse-lookup))) + + # Create C source file that contains images a uint8_t buffer. This + # can be compiled and linked statically into the main janet library + # and example client. + (def chunks (seq [b :in image] (string b))) + (def image-file (file/open "build/core_image.c" :w)) + (file/write image-file + "#include \n" + "static const unsigned char janet_core_image_bytes[] = {") + (loop [line :in (partition 16 chunks)] + (def str (string ;(interpose ", " line))) + (file/write image-file str ",\n")) + (file/write image-file + "0};\n\n" + "const unsigned char *janet_core_image = janet_core_image_bytes;\n" + "size_t janet_core_image_size = sizeof(janet_core_image_bytes);\n") + (file/close image-file)) diff --git a/src/core/array.c b/src/core/array.c index 4bd7a8c7..44f0a3f5 100644 --- a/src/core/array.c +++ b/src/core/array.c @@ -267,5 +267,5 @@ static const JanetReg array_cfuns[] = { /* Load the array module */ void janet_lib_array(JanetTable *env) { - janet_cfuns(env, NULL, array_cfuns); + janet_core_cfuns(env, NULL, array_cfuns); } diff --git a/src/core/asm.c b/src/core/asm.c index 75696abd..b0672e30 100644 --- a/src/core/asm.c +++ b/src/core/asm.c @@ -951,7 +951,7 @@ static const JanetReg asm_cfuns[] = { /* Load the library */ void janet_lib_asm(JanetTable *env) { - janet_cfuns(env, NULL, asm_cfuns); + janet_core_cfuns(env, NULL, asm_cfuns); } #endif diff --git a/src/core/buffer.c b/src/core/buffer.c index 6e035720..25a2e61c 100644 --- a/src/core/buffer.c +++ b/src/core/buffer.c @@ -387,5 +387,5 @@ static const JanetReg buffer_cfuns[] = { }; void janet_lib_buffer(JanetTable *env) { - janet_cfuns(env, NULL, buffer_cfuns); + janet_core_cfuns(env, NULL, buffer_cfuns); } diff --git a/src/core/compile.c b/src/core/compile.c index bbaae4e9..7f5daa9b 100644 --- a/src/core/compile.c +++ b/src/core/compile.c @@ -738,5 +738,5 @@ static const JanetReg compile_cfuns[] = { }; void janet_lib_compile(JanetTable *env) { - janet_cfuns(env, NULL, compile_cfuns); + janet_core_cfuns(env, NULL, compile_cfuns); } diff --git a/src/core/core.janet b/src/core/core.janet index b4a6bcd0..c4897c87 100644 --- a/src/core/core.janet +++ b/src/core/core.janet @@ -1067,6 +1067,20 @@ value, one key will be ignored." (++ i)) ret) +(defn partition + "Partition an indexed data structure into tuples + of size n. Returns a new array." + [n ind] + (var i 0) (var nextn n) + (def len (length ind)) + (def ret (array/new (math/ceil (/ len n)))) + (while (<= nextn len) + (array/push ret (tuple/slice ind i nextn)) + (set i nextn) + (+= nextn n)) + (if (not= i len) (array/push ret (tuple/slice ind i))) + ret) + ### ### ### Pattern Matching diff --git a/src/core/corelib.c b/src/core/corelib.c index 6cfaef36..d1f5ebac 100644 --- a/src/core/corelib.c +++ b/src/core/corelib.c @@ -28,8 +28,13 @@ #endif /* Generated bytes */ +#ifdef JANET_BOOTSTRAP extern const unsigned char *janet_gen_core; extern int32_t janet_gen_core_size; +#else +extern const unsigned char *janet_core_image; +extern size_t janet_core_image_size; +#endif /* Use LoadLibrary on windows or dlopen on posix to load dynamic libaries * with native code. */ @@ -371,7 +376,7 @@ static const JanetReg corelib_cfuns[] = { {NULL, NULL, NULL} }; -#ifndef JANET_NO_BOOTSTRAP +#ifdef JANET_BOOTSTRAP /* Utility for inline assembly */ static void janet_quick_asm( @@ -595,12 +600,9 @@ static const uint32_t bnot_asm[] = { JanetTable *janet_core_env(void) { JanetTable *env = janet_table(0); - Janet ret = janet_wrap_table(env); + janet_core_cfuns(env, NULL, corelib_cfuns); - /* Load main functions */ - janet_cfuns(env, NULL, corelib_cfuns); - -#ifndef JANET_NO_BOOTSTRAP +#ifdef JANET_BOOTSTRAP janet_quick_asm(env, JANET_FUN_YIELD, "debug", 0, 1, debug_asm, sizeof(debug_asm), JDOC("(debug)\n\n" "Throws a debug signal that can be caught by a parent fiber and used to inspect " @@ -731,11 +733,11 @@ JanetTable *janet_core_env(void) { JDOC("The build identifier of the running janet program.")); /* Allow references to the environment */ - janet_def(env, "_env", ret, JDOC("The environment table for the current scope.")); -#endif + janet_def(env, "_env", janet_wrap_table(env), JDOC("The environment table for the current scope.")); /* Set as gc root */ janet_gcroot(janet_wrap_table(env)); +#endif /* Load auxiliary envs */ janet_lib_io(env); @@ -756,9 +758,26 @@ JanetTable *janet_core_env(void) { janet_lib_asm(env); #endif -#ifndef JANET_NO_BOOTSTRAP +#ifdef JANET_BOOTSTRAP /* Run bootstrap source */ janet_dobytes(env, janet_gen_core, janet_gen_core_size, "core.janet", NULL); +#else + + /* Unmarshal from core image */ + Janet marsh_out; + int status = janet_unmarshal( + janet_core_image, + janet_core_image_size, + 0, + &marsh_out, + env, + NULL); + if (status) { + printf("error unmarshaling core image\n"); + exit(1); + } + janet_gcroot(marsh_out); + env = janet_unwrap_table(marsh_out); #endif return env; diff --git a/src/core/debug.c b/src/core/debug.c index 5a69229d..e2dd0aff 100644 --- a/src/core/debug.c +++ b/src/core/debug.c @@ -376,5 +376,5 @@ static const JanetReg debug_cfuns[] = { /* Module entry point */ void janet_lib_debug(JanetTable *env) { - janet_cfuns(env, NULL, debug_cfuns); + janet_core_cfuns(env, NULL, debug_cfuns); } diff --git a/src/core/fiber.c b/src/core/fiber.c index 9be9ce92..c5e16f0b 100644 --- a/src/core/fiber.c +++ b/src/core/fiber.c @@ -433,5 +433,5 @@ static const JanetReg fiber_cfuns[] = { /* Module entry point */ void janet_lib_fiber(JanetTable *env) { - janet_cfuns(env, NULL, fiber_cfuns); + janet_core_cfuns(env, NULL, fiber_cfuns); } diff --git a/src/core/io.c b/src/core/io.c index 4882ac17..7051a90a 100644 --- a/src/core/io.c +++ b/src/core/io.c @@ -394,18 +394,18 @@ static const JanetReg io_cfuns[] = { /* Module entry point */ void janet_lib_io(JanetTable *env) { - janet_cfuns(env, NULL, io_cfuns); + janet_core_cfuns(env, NULL, io_cfuns); /* stdout */ - janet_def(env, "stdout", + janet_core_def(env, "stdout", makef(stdout, IO_APPEND | IO_NOT_CLOSEABLE | IO_SERIALIZABLE), JDOC("The standard output file.")); /* stderr */ - janet_def(env, "stderr", + janet_core_def(env, "stderr", makef(stderr, IO_APPEND | IO_NOT_CLOSEABLE | IO_SERIALIZABLE), JDOC("The standard error file.")); /* stdin */ - janet_def(env, "stdin", + janet_core_def(env, "stdin", makef(stdin, IO_READ | IO_NOT_CLOSEABLE | IO_SERIALIZABLE), JDOC("The standard input file.")); } diff --git a/src/core/marsh.c b/src/core/marsh.c index 39e659ff..614af254 100644 --- a/src/core/marsh.c +++ b/src/core/marsh.c @@ -234,10 +234,12 @@ static void marshal_one_def(MarshalState *st, JanetFuncDef *def, int flags) { /* marshal source maps if needed */ if (def->flags & JANET_FUNCDEF_FLAG_HASSOURCEMAP) { + int32_t current = 0; for (int32_t i = 0; i < def->bytecode_length; i++) { JanetSourceMapping map = def->sourcemap[i]; - pushint(st, map.start); - pushint(st, map.end); + pushint(st, map.start - current); + pushint(st, map.end - map.start); + current = map.end; } } } @@ -748,13 +750,16 @@ static const uint8_t *unmarshal_one_def( /* Unmarshal source maps if needed */ if (def->flags & JANET_FUNCDEF_FLAG_HASSOURCEMAP) { + int32_t current = 0; def->sourcemap = malloc(sizeof(JanetSourceMapping) * bytecode_length); if (!def->sourcemap) { JANET_OUT_OF_MEMORY; } for (int32_t i = 0; i < bytecode_length; i++) { - def->sourcemap[i].start = readint(st, &data); - def->sourcemap[i].end = readint(st, &data); + current += readint(st, &data); + def->sourcemap[i].start = current; + current += readint(st, &data); + def->sourcemap[i].end = current; } } else { def->sourcemap = NULL; @@ -1175,5 +1180,5 @@ static const JanetReg marsh_cfuns[] = { /* Module entry point */ void janet_lib_marsh(JanetTable *env) { - janet_cfuns(env, NULL, marsh_cfuns); + janet_core_cfuns(env, NULL, marsh_cfuns); } diff --git a/src/core/math.c b/src/core/math.c index b49f9fbf..4dd0fd5b 100644 --- a/src/core/math.c +++ b/src/core/math.c @@ -181,8 +181,8 @@ static const JanetReg math_cfuns[] = { /* Module entry point */ void janet_lib_math(JanetTable *env) { - janet_cfuns(env, NULL, math_cfuns); -#ifndef JANET_NO_BOOTSTRAP + janet_core_cfuns(env, NULL, math_cfuns); +#ifdef JANET_BOOTSTRAP janet_def(env, "math/pi", janet_wrap_number(3.1415926535897931), JDOC("The value pi.")); janet_def(env, "math/e", janet_wrap_number(2.7182818284590451), diff --git a/src/core/os.c b/src/core/os.c index f85886b3..43b7add8 100644 --- a/src/core/os.c +++ b/src/core/os.c @@ -378,5 +378,5 @@ static const JanetReg os_cfuns[] = { /* Module entry point */ void janet_lib_os(JanetTable *env) { - janet_cfuns(env, NULL, os_cfuns); + janet_core_cfuns(env, NULL, os_cfuns); } diff --git a/src/core/parse.c b/src/core/parse.c index 09330a0a..a389f862 100644 --- a/src/core/parse.c +++ b/src/core/parse.c @@ -876,5 +876,5 @@ static const JanetReg parse_cfuns[] = { /* Load the library */ void janet_lib_parse(JanetTable *env) { - janet_cfuns(env, NULL, parse_cfuns); + janet_core_cfuns(env, NULL, parse_cfuns); } diff --git a/src/core/peg.c b/src/core/peg.c index 82c66ed9..46f21985 100644 --- a/src/core/peg.c +++ b/src/core/peg.c @@ -1103,5 +1103,5 @@ static const JanetReg peg_cfuns[] = { /* Load the peg module */ void janet_lib_peg(JanetTable *env) { - janet_cfuns(env, NULL, peg_cfuns); + janet_core_cfuns(env, NULL, peg_cfuns); } diff --git a/src/core/specials.c b/src/core/specials.c index a82d3236..4b1ce3c0 100644 --- a/src/core/specials.c +++ b/src/core/specials.c @@ -221,7 +221,6 @@ static JanetSlot janetc_varset(JanetFopts opts, int32_t argn, const Janet *argv) return rvalue; } else { /* Error */ - janet_inspect(argv[0]); janetc_cerror(opts.compiler, "expected symbol or tuple for l-value to set"); return janetc_cslot(janet_wrap_nil()); } diff --git a/src/core/string.c b/src/core/string.c index 452db7e1..97f3c120 100644 --- a/src/core/string.c +++ b/src/core/string.c @@ -619,5 +619,5 @@ static const JanetReg string_cfuns[] = { /* Module entry point */ void janet_lib_string(JanetTable *env) { - janet_cfuns(env, NULL, string_cfuns); + janet_core_cfuns(env, NULL, string_cfuns); } diff --git a/src/core/table.c b/src/core/table.c index 23bd44dc..c03bfa2b 100644 --- a/src/core/table.c +++ b/src/core/table.c @@ -273,5 +273,5 @@ static const JanetReg table_cfuns[] = { /* Load the table module */ void janet_lib_table(JanetTable *env) { - janet_cfuns(env, NULL, table_cfuns); + janet_core_cfuns(env, NULL, table_cfuns); } diff --git a/src/core/tuple.c b/src/core/tuple.c index 2d7f1090..9d824c79 100644 --- a/src/core/tuple.c +++ b/src/core/tuple.c @@ -146,5 +146,5 @@ static const JanetReg tuple_cfuns[] = { /* Load the tuple module */ void janet_lib_tuple(JanetTable *env) { - janet_cfuns(env, NULL, tuple_cfuns); + janet_core_cfuns(env, NULL, tuple_cfuns); } diff --git a/src/core/util.c b/src/core/util.c index 4264917e..afa7ee83 100644 --- a/src/core/util.c +++ b/src/core/util.c @@ -284,6 +284,24 @@ void janet_cfuns(JanetTable *env, const char *regprefix, const JanetReg *cfuns) } } +#ifndef JANET_BOOTSTRAP +void janet_core_def(JanetTable *env, const char *name, Janet x, const void *p) { + (void) p; + janet_table_put(env, janet_csymbolv(name), x); +} + +void janet_core_cfuns(JanetTable *env, const char *regprefix, const JanetReg *cfuns) { + (void) regprefix; + while (cfuns->name) { + Janet name = janet_csymbolv(cfuns->name); + Janet fun = janet_wrap_cfunction(cfuns->cfun); + janet_core_def(env, cfuns->name, fun, cfuns->documentation); + janet_table_put(janet_vm_registry, fun, name); + cfuns++; + } +} +#endif + /* Resolve a symbol in the environment */ JanetBindingType janet_resolve(JanetTable *env, const uint8_t *sym, Janet *out) { Janet ref; diff --git a/src/core/util.h b/src/core/util.h index c10586d7..aa6acff3 100644 --- a/src/core/util.h +++ b/src/core/util.h @@ -28,8 +28,9 @@ #endif /* Omit docstrings in some builds */ -#ifdef JANET_NO_BOOTSTRAP +#ifndef JANET_BOOTSTRAP #define JDOC(x) NULL +#define JANET_NO_BOOTSTRAP #else #define JDOC(x) x #endif @@ -52,6 +53,16 @@ const void *janet_strbinsearch( size_t itemsize, const uint8_t *key); +/* Inside the janet core, defining globals is different + * at bootstrap time and normal runtime */ +#ifdef JANET_BOOTSTRAP +#define janet_core_def janet_def +#define janet_core_cfuns janet_cfuns +#else +void janet_core_def(JanetTable *env, const char *name, Janet x, const void *p); +void janet_core_cfuns(JanetTable *env, const char *regprefix, const JanetReg *cfuns); +#endif + /* Initialize builtin libraries */ void janet_lib_io(JanetTable *env); void janet_lib_math(JanetTable *env); diff --git a/src/include/janet/janet.h b/src/include/janet/janet.h index e4a0f385..887be395 100644 --- a/src/include/janet/janet.h +++ b/src/include/janet/janet.h @@ -1116,7 +1116,6 @@ JANET_API Janet janet_getindex(Janet ds, int32_t index); JANET_API int32_t janet_length(Janet x); JANET_API void janet_put(Janet ds, Janet key, Janet value); JANET_API void janet_putindex(Janet ds, int32_t index, Janet value); -JANET_API void janet_inspect(Janet x); /* VM functions */ JANET_API int janet_init(void); diff --git a/tools/amalg.janet b/tools/amalg.janet index 7dcf3f5b..40d23d78 100644 --- a/tools/amalg.janet +++ b/tools/amalg.janet @@ -71,5 +71,6 @@ (each h headers (dofile h)) (each s sources (dofile s)) -# Relies on this file being built +# Relies on these files being built (dofile "build/core.gen.c") +(dofile "build/core_image.c") diff --git a/tools/marshal_core.janet b/tools/marshal_core.janet deleted file mode 100644 index a5415f05..00000000 --- a/tools/marshal_core.janet +++ /dev/null @@ -1,21 +0,0 @@ -# Tool to dump a marshalled version of the janet core to stdout. The -# image should eventually allow janet to be started from a pre-compiled -# image rather than recompiled every time from the embedded source. More -# work will go into shrinking the image (it isn't currently that large but -# could be smaller), creating the mechanism to load the image, and modifying -# the build process to compile janet with a built image rather than -# embedded source. - -# Get image. This image contains as much of the core library and documentation that -# can be written to an image (no cfunctions, no abstracts (stdout, stdin, stderr)), -# everything else goes. Cfunctions and abstracts will be referenced from a register -# table which will be generated on janet startup. -(def image (let [env-pairs (pairs (env-lookup *env*)) - essential-pairs (filter (fn [[k v]] (or (cfunction? v) (abstract? v))) env-pairs) - lookup (table ;(mapcat identity essential-pairs)) - reverse-lookup (invert lookup)] - (marshal (table/getproto *env*) reverse-lookup))) - -# Write image -(file/write stdout image) -(file/flush stdout)