Boot core library from image rather than source

This should speed up start time and reduce malloc/free
usage to about 15% of what is what previously for startup.
The current cost is slightly larger binary as the representaion
of the image is currently less compact than source code.

Makefile

@@ -47,7 +47,7 @@ else
 	CLIBS:=$(CLIBS) -lrt
 endif
 
-$(shell mkdir -p build/core build/mainclient build/webclient)
+$(shell mkdir -p build/core build/mainclient build/webclient build/boot)
 
 # Source headers
 JANET_HEADERS=$(sort $(wildcard src/include/janet/*.h))
@@ -60,14 +60,33 @@ JANET_WEBCLIENT_SOURCES=$(sort $(wildcard src/webclient/*.c))
 
 all: $(JANET_TARGET) $(JANET_LIBRARY)
 
+##################################################################
+##### The bootstrap interpreter that compiles the core image #####
+##################################################################
+
+JANET_BOOT_OBJECTS=$(patsubst src/%.c,build/%.boot.o,$(JANET_CORE_SOURCES) src/boot/boot.c) \
+	build/core.gen.o \
+	build/boot.gen.o
+
+build/%.boot.o: src/%.c
+	$(CC) $(CFLAGS) -DJANET_BOOTSTRAP -o $@ -c $<
+
+build/janet_boot: $(JANET_BOOT_OBJECTS)
+	$(CC) $(CFLAGS) -DJANET_BOOTSTRAP -o $@ $^ $(CLIBS)
+
+# Now the reason we bootstrap in the first place
+build/core_image.c: build/janet_boot
+	build/janet_boot
+
 ##########################################################
 ##### The main interpreter program and shared object #####
 ##########################################################
 
-JANET_CORE_OBJECTS=$(patsubst src/%.c,build/%.o,$(JANET_CORE_SOURCES)) build/core.gen.o
+JANET_CORE_OBJECTS=$(patsubst src/%.c,build/%.o,$(JANET_CORE_SOURCES)) build/core_image.o
 JANET_MAINCLIENT_OBJECTS=$(patsubst src/%.c,build/%.o,$(JANET_MAINCLIENT_SOURCES)) build/init.gen.o
 
-%.gen.o: %.gen.c
+# Compile the core image generated by the bootstrap build
+build/core_image.o: build/core_image.c $(JANET_HEADERS) $(JANET_LOCAL_HEADERS)
 	$(CC) $(CFLAGS) -o $@ -c $<
 
 build/%.o: src/%.c $(JANET_HEADERS) $(JANET_LOCAL_HEADERS)
@@ -109,6 +128,9 @@ emscripten: $(JANET_EMTARGET)
 ##### Generated C files #####
 #############################
 
+%.gen.o: %.gen.c
+	$(CC) $(CFLAGS) -o $@ -c $<
+
 build/xxd: tools/xxd.c
 	$(CC) $< -o $@
 
@@ -118,12 +140,14 @@ build/init.gen.c: src/mainclient/init.janet build/xxd
 	build/xxd $< $@ janet_gen_init
 build/webinit.gen.c: src/webclient/webinit.janet build/xxd
 	build/xxd $< $@ janet_gen_webinit
+build/boot.gen.c: src/boot/boot.janet build/xxd
+	build/xxd $< $@ janet_gen_boot
 
 ########################
 ##### Amalgamation #####
 ########################
 
-amalg: build/janet.c build/janet.h
+amalg: build/janet.c build/janet.h build/core_image.c
 
 build/janet.c: $(JANET_LOCAL_HEADERS) $(JANET_CORE_SOURCES) tools/amalg.janet $(JANET_TARGET)
 	$(JANET_TARGET) tools/amalg.janet > $@

src/boot/boot.c

@@ -0,0 +1,43 @@
+/*
+* Copyright (c) 2019 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#include <janet/janet.h>
+
+extern const unsigned char *janet_gen_boot;
+extern int32_t janet_gen_boot_size;
+
+int main() {
+    int status;
+    JanetTable *env;
+
+    /* Set up VM */
+    janet_init();
+    env = janet_core_env();
+
+    /* Run bootstrap script to generate core image */
+    status = janet_dobytes(env, janet_gen_boot, janet_gen_boot_size, "boot.janet", NULL);
+
+    /* Deinitialize vm */
+    janet_deinit();
+
+    return status;
+}

src/boot/boot.janet

@@ -0,0 +1,40 @@
+# Copyright (C) Calvin Rose 2019
+
+# The bootstrap script is used to produce the source file for
+# embedding the core image.
+
+# Tool to dump a marshalled version of the janet core to stdout. The
+# image should eventually allow janet to be started from a pre-compiled
+# image rather than recompiled every time from the embedded source. More
+# work will go into shrinking the image (it isn't currently that large but
+# could be smaller), creating the mechanism to load the image, and modifying
+# the build process to compile janet with a built image rather than
+# embedded source.
+
+# Get image. This image contains as much of the core library and documentation that
+# can be written to an image (no cfunctions, no abstracts (stdout, stdin, stderr)),
+# everything else goes. Cfunctions and abstracts will be referenced from a registry
+# table which will be generated on janet startup.
+(do
+  (def image (let [env-pairs (pairs (env-lookup *env*))
+                   essential-pairs (filter (fn [[k v]] (or (cfunction? v) (abstract? v))) env-pairs)
+                   lookup (table ;(mapcat identity essential-pairs))
+                   reverse-lookup (invert lookup)]
+               (marshal *env* reverse-lookup)))
+
+  # Create C source file that contains images a uint8_t buffer. This
+  # can be compiled and linked statically into the main janet library
+  # and example client.
+  (def chunks (seq [b :in image] (string b)))
+  (def image-file (file/open "build/core_image.c" :w))
+  (file/write image-file
+              "#include <janet/janet.h>\n"
+              "static const unsigned char janet_core_image_bytes[] = {")
+  (loop [line :in (partition 16 chunks)]
+    (def str (string ;(interpose ", " line)))
+    (file/write image-file str ",\n"))
+  (file/write image-file
+              "0};\n\n"
+              "const unsigned char *janet_core_image = janet_core_image_bytes;\n"
+              "size_t janet_core_image_size = sizeof(janet_core_image_bytes);\n")
+  (file/close image-file))

src/core/array.c

@@ -267,5 +267,5 @@ static const JanetReg array_cfuns[] = {
 
 /* Load the array module */
 void janet_lib_array(JanetTable *env) {
-    janet_cfuns(env, NULL, array_cfuns);
+    janet_core_cfuns(env, NULL, array_cfuns);
 }

src/core/asm.c

@@ -951,7 +951,7 @@ static const JanetReg asm_cfuns[] = {
 
 /* Load the library */
 void janet_lib_asm(JanetTable *env) {
-    janet_cfuns(env, NULL, asm_cfuns);
+    janet_core_cfuns(env, NULL, asm_cfuns);
 }
 
 #endif

src/core/buffer.c

@@ -387,5 +387,5 @@ static const JanetReg buffer_cfuns[] = {
 };
 
 void janet_lib_buffer(JanetTable *env) {
-    janet_cfuns(env, NULL, buffer_cfuns);
+    janet_core_cfuns(env, NULL, buffer_cfuns);
 }

src/core/compile.c

@@ -738,5 +738,5 @@ static const JanetReg compile_cfuns[] = {
 };
 
 void janet_lib_compile(JanetTable *env) {
-    janet_cfuns(env, NULL, compile_cfuns);
+    janet_core_cfuns(env, NULL, compile_cfuns);
 }

src/core/core.janet

@@ -1067,6 +1067,20 @@ value, one key will be ignored."
     (++ i))
   ret)
 
+(defn partition
+  "Partition an indexed data structure into tuples
+  of size n. Returns a new array."
+  [n ind]
+  (var i 0) (var nextn n)
+  (def len (length ind))
+  (def ret (array/new (math/ceil (/ len n))))
+  (while (<= nextn len)
+    (array/push ret (tuple/slice ind i nextn))
+    (set i nextn)
+    (+= nextn n))
+  (if (not= i len) (array/push ret (tuple/slice ind i)))
+  ret)
+
 ###
 ###
 ### Pattern Matching

src/core/corelib.c

@@ -28,8 +28,13 @@
 #endif
 
 /* Generated bytes */
+#ifdef JANET_BOOTSTRAP
 extern const unsigned char *janet_gen_core;
 extern int32_t janet_gen_core_size;
+#else
+extern const unsigned char *janet_core_image;
+extern size_t janet_core_image_size;
+#endif
 
 /* Use LoadLibrary on windows or dlopen on posix to load dynamic libaries
  * with native code. */
@@ -371,7 +376,7 @@ static const JanetReg corelib_cfuns[] = {
     {NULL, NULL, NULL}
 };
 
-#ifndef JANET_NO_BOOTSTRAP
+#ifdef JANET_BOOTSTRAP
 
 /* Utility for inline assembly */
 static void janet_quick_asm(
@@ -595,12 +600,9 @@ static const uint32_t bnot_asm[] = {
 
 JanetTable *janet_core_env(void) {
     JanetTable *env = janet_table(0);
-    Janet ret = janet_wrap_table(env);
+    janet_core_cfuns(env, NULL, corelib_cfuns);
 
-    /* Load main functions */
-    janet_cfuns(env, NULL, corelib_cfuns);
-
-#ifndef JANET_NO_BOOTSTRAP
+#ifdef JANET_BOOTSTRAP
     janet_quick_asm(env, JANET_FUN_YIELD, "debug", 0, 1, debug_asm, sizeof(debug_asm),
             JDOC("(debug)\n\n"
                 "Throws a debug signal that can be caught by a parent fiber and used to inspect "
@@ -731,11 +733,11 @@ JanetTable *janet_core_env(void) {
             JDOC("The build identifier of the running janet program."));
 
     /* Allow references to the environment */
-    janet_def(env, "_env", ret, JDOC("The environment table for the current scope."));
-#endif
+    janet_def(env, "_env", janet_wrap_table(env), JDOC("The environment table for the current scope."));
 
     /* Set as gc root */
     janet_gcroot(janet_wrap_table(env));
+#endif
 
     /* Load auxiliary envs */
     janet_lib_io(env);
@@ -756,9 +758,26 @@ JanetTable *janet_core_env(void) {
     janet_lib_asm(env);
 #endif
 
-#ifndef JANET_NO_BOOTSTRAP
+#ifdef JANET_BOOTSTRAP
     /* Run bootstrap source */
     janet_dobytes(env, janet_gen_core, janet_gen_core_size, "core.janet", NULL);
+#else
+
+    /* Unmarshal from core image */
+    Janet marsh_out;
+    int status = janet_unmarshal(
+            janet_core_image,
+            janet_core_image_size,
+            0,
+            &marsh_out,
+            env,
+            NULL);
+    if (status) {
+        printf("error unmarshaling core image\n");
+        exit(1);
+    }
+    janet_gcroot(marsh_out);
+    env = janet_unwrap_table(marsh_out);
 #endif
 
     return env;

src/core/debug.c

@@ -376,5 +376,5 @@ static const JanetReg debug_cfuns[] = {
 
 /* Module entry point */
 void janet_lib_debug(JanetTable *env) {
-    janet_cfuns(env, NULL, debug_cfuns);
+    janet_core_cfuns(env, NULL, debug_cfuns);
 }

src/core/fiber.c

@@ -433,5 +433,5 @@ static const JanetReg fiber_cfuns[] = {
 
 /* Module entry point */
 void janet_lib_fiber(JanetTable *env) {
-    janet_cfuns(env, NULL, fiber_cfuns);
+    janet_core_cfuns(env, NULL, fiber_cfuns);
 }

src/core/io.c

@@ -394,18 +394,18 @@ static const JanetReg io_cfuns[] = {
 
 /* Module entry point */
 void janet_lib_io(JanetTable *env) {
-    janet_cfuns(env, NULL, io_cfuns);
+    janet_core_cfuns(env, NULL, io_cfuns);
 
     /* stdout */
-    janet_def(env, "stdout",
+    janet_core_def(env, "stdout",
             makef(stdout, IO_APPEND | IO_NOT_CLOSEABLE | IO_SERIALIZABLE),
             JDOC("The standard output file."));
     /* stderr */
-    janet_def(env, "stderr",
+    janet_core_def(env, "stderr",
             makef(stderr, IO_APPEND | IO_NOT_CLOSEABLE | IO_SERIALIZABLE),
             JDOC("The standard error file."));
     /* stdin */
-    janet_def(env, "stdin",
+    janet_core_def(env, "stdin",
             makef(stdin, IO_READ | IO_NOT_CLOSEABLE | IO_SERIALIZABLE),
             JDOC("The standard input file."));
 }

src/core/marsh.c

@@ -234,10 +234,12 @@ static void marshal_one_def(MarshalState *st, JanetFuncDef *def, int flags) {
 
     /* marshal source maps if needed */
     if (def->flags & JANET_FUNCDEF_FLAG_HASSOURCEMAP) {
+        int32_t current = 0;
         for (int32_t i = 0; i < def->bytecode_length; i++) {
             JanetSourceMapping map = def->sourcemap[i];
-            pushint(st, map.start);
-            pushint(st, map.end);
+            pushint(st, map.start - current);
+            pushint(st, map.end - map.start);
+            current = map.end;
         }
     }
 }
@@ -748,13 +750,16 @@ static const uint8_t *unmarshal_one_def(
 
         /* Unmarshal source maps if needed */
         if (def->flags & JANET_FUNCDEF_FLAG_HASSOURCEMAP) {
+            int32_t current = 0;
             def->sourcemap = malloc(sizeof(JanetSourceMapping) * bytecode_length);
             if (!def->sourcemap) {
                 JANET_OUT_OF_MEMORY;
             }
             for (int32_t i = 0; i < bytecode_length; i++) {
-                def->sourcemap[i].start = readint(st, &data);
-                def->sourcemap[i].end = readint(st, &data);
+                current += readint(st, &data);
+                def->sourcemap[i].start = current;
+                current += readint(st, &data);
+                def->sourcemap[i].end = current;
             }
         } else {
             def->sourcemap = NULL;
@@ -1175,5 +1180,5 @@ static const JanetReg marsh_cfuns[] = {
 
 /* Module entry point */
 void janet_lib_marsh(JanetTable *env) {
-    janet_cfuns(env, NULL, marsh_cfuns);
+    janet_core_cfuns(env, NULL, marsh_cfuns);
 }

src/core/math.c

@@ -181,8 +181,8 @@ static const JanetReg math_cfuns[] = {
 
 /* Module entry point */
 void janet_lib_math(JanetTable *env) {
-    janet_cfuns(env, NULL, math_cfuns);
-#ifndef JANET_NO_BOOTSTRAP
+    janet_core_cfuns(env, NULL, math_cfuns);
+#ifdef JANET_BOOTSTRAP
     janet_def(env, "math/pi", janet_wrap_number(3.1415926535897931),
             JDOC("The value pi."));
     janet_def(env, "math/e", janet_wrap_number(2.7182818284590451),

src/core/os.c

@@ -378,5 +378,5 @@ static const JanetReg os_cfuns[] = {
 
 /* Module entry point */
 void janet_lib_os(JanetTable *env) {
-    janet_cfuns(env, NULL, os_cfuns);
+    janet_core_cfuns(env, NULL, os_cfuns);
 }

src/core/parse.c

@@ -876,5 +876,5 @@ static const JanetReg parse_cfuns[] = {
 
 /* Load the library */
 void janet_lib_parse(JanetTable *env) {
-    janet_cfuns(env, NULL, parse_cfuns);
+    janet_core_cfuns(env, NULL, parse_cfuns);
 }

src/core/peg.c

@@ -1103,5 +1103,5 @@ static const JanetReg peg_cfuns[] = {
 
 /* Load the peg module */
 void janet_lib_peg(JanetTable *env) {
-    janet_cfuns(env, NULL, peg_cfuns);
+    janet_core_cfuns(env, NULL, peg_cfuns);
 }

src/core/specials.c

@@ -221,7 +221,6 @@ static JanetSlot janetc_varset(JanetFopts opts, int32_t argn, const Janet *argv)
         return rvalue;
     } else {
         /* Error */
-        janet_inspect(argv[0]);
         janetc_cerror(opts.compiler, "expected symbol or tuple for l-value to set");
         return janetc_cslot(janet_wrap_nil());
     }

src/core/string.c

@@ -619,5 +619,5 @@ static const JanetReg string_cfuns[] = {
 
 /* Module entry point */
 void janet_lib_string(JanetTable *env) {
-    janet_cfuns(env, NULL, string_cfuns);
+    janet_core_cfuns(env, NULL, string_cfuns);
 }

src/core/table.c

@@ -273,5 +273,5 @@ static const JanetReg table_cfuns[] = {
 
 /* Load the table module */
 void janet_lib_table(JanetTable *env) {
-    janet_cfuns(env, NULL, table_cfuns);
+    janet_core_cfuns(env, NULL, table_cfuns);
 }

src/core/tuple.c

@@ -146,5 +146,5 @@ static const JanetReg tuple_cfuns[] = {
 
 /* Load the tuple module */
 void janet_lib_tuple(JanetTable *env) {
-    janet_cfuns(env, NULL, tuple_cfuns);
+    janet_core_cfuns(env, NULL, tuple_cfuns);
 }

src/core/util.c

@@ -284,6 +284,24 @@ void janet_cfuns(JanetTable *env, const char *regprefix, const JanetReg *cfuns)
     }
 }
 
+#ifndef JANET_BOOTSTRAP
+void janet_core_def(JanetTable *env, const char *name, Janet x, const void *p) {
+    (void) p;
+    janet_table_put(env, janet_csymbolv(name), x);
+}
+
+void janet_core_cfuns(JanetTable *env, const char *regprefix, const JanetReg *cfuns) {
+    (void) regprefix;
+    while (cfuns->name) {
+        Janet name = janet_csymbolv(cfuns->name);
+        Janet fun = janet_wrap_cfunction(cfuns->cfun);
+        janet_core_def(env, cfuns->name, fun, cfuns->documentation);
+        janet_table_put(janet_vm_registry, fun, name);
+        cfuns++;
+    }
+}
+#endif
+
 /* Resolve a symbol in the environment */
 JanetBindingType janet_resolve(JanetTable *env, const uint8_t *sym, Janet *out) {
     Janet ref;

src/core/util.h

@@ -28,8 +28,9 @@
 #endif
 
 /* Omit docstrings in some builds */
-#ifdef JANET_NO_BOOTSTRAP
+#ifndef JANET_BOOTSTRAP
 #define JDOC(x) NULL
+#define JANET_NO_BOOTSTRAP
 #else
 #define JDOC(x) x
 #endif
@@ -52,6 +53,16 @@ const void *janet_strbinsearch(
         size_t itemsize,
         const uint8_t *key);
 
+/* Inside the janet core, defining globals is different
+ * at bootstrap time and normal runtime */
+#ifdef JANET_BOOTSTRAP
+#define janet_core_def janet_def
+#define janet_core_cfuns janet_cfuns
+#else
+void janet_core_def(JanetTable *env, const char *name, Janet x, const void *p);
+void janet_core_cfuns(JanetTable *env, const char *regprefix, const JanetReg *cfuns);
+#endif
+
 /* Initialize builtin libraries */
 void janet_lib_io(JanetTable *env);
 void janet_lib_math(JanetTable *env);

src/include/janet/janet.h

@@ -1116,7 +1116,6 @@ JANET_API Janet janet_getindex(Janet ds, int32_t index);
 JANET_API int32_t janet_length(Janet x);
 JANET_API void janet_put(Janet ds, Janet key, Janet value);
 JANET_API void janet_putindex(Janet ds, int32_t index, Janet value);
-JANET_API void janet_inspect(Janet x);
 
 /* VM functions */
 JANET_API int janet_init(void);

tools/amalg.janet

@@ -71,5 +71,6 @@
 (each h headers (dofile h))
 (each s sources (dofile s))
 
-# Relies on this file being built
+# Relies on these files being built
 (dofile "build/core.gen.c")
+(dofile "build/core_image.c")

tools/marshal_core.janet

@@ -1,21 +0,0 @@
-# Tool to dump a marshalled version of the janet core to stdout. The
-# image should eventually allow janet to be started from a pre-compiled
-# image rather than recompiled every time from the embedded source. More
-# work will go into shrinking the image (it isn't currently that large but
-# could be smaller), creating the mechanism to load the image, and modifying
-# the build process to compile janet with a built image rather than
-# embedded source.
-
-# Get image. This image contains as much of the core library and documentation that
-# can be written to an image (no cfunctions, no abstracts (stdout, stdin, stderr)),
-# everything else goes. Cfunctions and abstracts will be referenced from a register
-# table which will be generated on janet startup.
-(def image (let [env-pairs (pairs (env-lookup *env*))
-                 essential-pairs (filter (fn [[k v]] (or (cfunction? v) (abstract? v))) env-pairs)
-                 lookup (table ;(mapcat identity essential-pairs))
-                 reverse-lookup (invert lookup)]
-             (marshal (table/getproto *env*) reverse-lookup)))
-
-# Write image
-(file/write stdout image)
-(file/flush stdout)