Add utf-8 compatibility in parser. Symbols can

be valid utf-8 strings

2

@@ -26,7 +26,7 @@
 void dst_compile_slotpool_init(DstSlotPool *pool) {
     pool->s = NULL;
     pool->count = 0;
-    pool->free = 0;
+    pool->max;
     pool->cap = 0;
 }
 
@@ -34,35 +34,34 @@ void dst_compile_slotpool_deinit(DstSlotPool *pool) {
     free(pool->s);
     pool->s = NULL;
     pool->cap = 0;
+    pool->max = 0;
     pool->count = 0;
-    pool->free = 0;
 }
 
 void dst_compile_slotpool_extend(DstSlotPool *pool, int32_t extra) {
     int32_t i;
-    int32_t newcount = pool->count + extra;
+    int32_t newcap = pool->cap + extra;
-    if (newcount > pool->cap) {
+    if (newcap > pool->cap) {
-        int32_t newcap = 2 * newcount;
+        newcap *= 2;
         pool->s = realloc(pool->s, newcap * sizeof(DstSlot));
         if (NULL == pool->s) {
             DST_OUT_OF_MEMORY;
         }
-        pool->cap = newcap;
     }
+    pool->cap = newcap;
     /* Mark all new slots as free */
-    for (i = pool->count; i < newcount; i++) {
+    for (i = pool->count; i < newcap; i++) {
         pool->s[i].flags = 0;
     }
-    pool->count = newcount;
 }
 
 DstSlot *dst_compile_slotpool_alloc(DstSlotPool *pool) {
     int32_t oldcount = pool->count;
     int32_t newcount = oldcount == 0xF0 ? 0x101 : oldcount + 1;
     int32_t index = newcount - 1;
-    while (pool->free < pool->count) {
+    while (pool->count < pool->cap) {
-        if (!(pool->s[pool->free].flags & DST_SLOT_NOTEMPTY)) {
+        if (!(pool->s[pool->count].flags & DST_SLOT_NOTEMPTY)) {
-            return pool->s + pool->free; 
+            return pool->s + pool->count; 
         }
         pool->free++;
     }

Makefile

@@ -31,7 +31,7 @@ PREFIX=/usr/local
 DST_TARGET=dst
 DST_XXD=xxd
 DEBUGGER=lldb
-DST_INTERNAL_HEADERS=$(addprefix core/,symcache.h opcodes.h strtod.h compile.h)
+DST_INTERNAL_HEADERS=$(addprefix core/,symcache.h opcodes.h strtod.h compile.h gc.h)
 DST_HEADERS=$(addprefix include/dst/,dst.h)
 
 #############################
@@ -59,7 +59,7 @@ $(DST_XXD): libs/xxd.c
 ###################################
 
 DST_CORE_SOURCES=$(addprefix core/,\
-				 array.c asm.c buffer.c compile.c compile_slotpool.c \
+				 array.c asm.c buffer.c compile.c\
 				 fiber.c func.c gc.c parse.c string.c strtod.c\
 				 struct.c symcache.c syscalls.c table.c tuple.c userdata.c util.c\
 				 value.c vm.c wrap.c)

core/array.c

@@ -21,6 +21,7 @@
 */
 
 #include <dst/dst.h>
+#include "gc.h"
 
 /* Iniializes an array */
 DstArray *dst_array_init(DstArray *array, int32_t capacity) {
@@ -43,7 +44,7 @@ void dst_array_deinit(DstArray *array) {
 
 /* Creates a new array */
 DstArray *dst_array(int32_t capacity) {
-    DstArray *array = dst_alloc(DST_MEMORY_ARRAY, sizeof(DstArray));
+    DstArray *array = dst_gcalloc(DST_MEMORY_ARRAY, sizeof(DstArray));
     return dst_array_init(array, capacity);
 }
 

core/asm.c

@@ -24,6 +24,7 @@
 
 #include <dst/dst.h>
 #include "opcodes.h"
+#include "gc.h"
 
 /* Bytecode op argument types */
 
@@ -511,7 +512,7 @@ static DstAssembleResult dst_asm1(DstAssembler *parent, DstAssembleOptions opts)
     DstValue x;
 
     /* Initialize funcdef */
-    def = dst_alloc(DST_MEMORY_FUNCDEF, sizeof(DstFuncDef));
+    def = dst_gcalloc(DST_MEMORY_FUNCDEF, sizeof(DstFuncDef));
     def->environments = NULL;
     def->constants = NULL;
     def->bytecode = NULL;
@@ -756,7 +757,7 @@ DstFunction *dst_asm_func(DstAssembleResult result) {
     if (result.status != DST_ASSEMBLE_OK) {
         return NULL;
     }
-    DstFunction *func = dst_alloc(DST_MEMORY_FUNCTION, sizeof(DstFunction));
+    DstFunction *func = dst_gcalloc(DST_MEMORY_FUNCTION, sizeof(DstFunction));
     func->def = result.funcdef;
     func->envs = NULL;
     return func;
@@ -906,5 +907,8 @@ DstValue dst_disasm(DstFuncDef *def) {
         dst_table_put(ret, dst_csymbolv("environments"), dst_wrap_array(envs));
     }
 
+    /* Add slotcount */
+    dst_table_put(ret, dst_csymbolv("slotcount"), dst_wrap_integer(def->slotcount));
+
     return dst_wrap_struct(dst_table_to_struct(ret));
 }

core/buffer.c

@@ -21,6 +21,7 @@
 */
 
 #include <dst/dst.h>
+#include "gc.h"
 
 /* Initialize a buffer */
 DstBuffer *dst_buffer_init(DstBuffer *buffer, int32_t capacity) {
@@ -44,7 +45,7 @@ void dst_buffer_deinit(DstBuffer *buffer) {
 
 /* Initialize a buffer */
 DstBuffer *dst_buffer(int32_t capacity) {
-    DstBuffer *buffer = dst_alloc(DST_MEMORY_BUFFER, sizeof(DstBuffer));
+    DstBuffer *buffer = dst_gcalloc(DST_MEMORY_BUFFER, sizeof(DstBuffer));
     return dst_buffer_init(buffer, capacity);
 }
 

core/compile.h

@@ -39,7 +39,8 @@ typedef struct DstCFunctionOptimizer DstCFunctionOptimizer;
 #define DST_SLOT_CONSTANT 0x10000
 #define DST_SLOT_NAMED 0x20000
 #define DST_SLOT_MUTABLE 0x40000
-#define DST_SLOT_NOTEMPTY 0x80000
+#define DST_SLOT_REF 0x80000
+/* Needed for handling single element arrays as global vars. */
 
 #define DST_SLOTTYPE_ANY 0xFFFF
 
@@ -62,33 +63,35 @@ struct DstSlot {
  * var
  * varset
  * do
+ * apply (overloaded with normal function)
  */
 
 #define DST_SCOPE_FUNCTION 1
 #define DST_SCOPE_LASTSLOT 2
 #define DST_SCOPE_FIRSTSLOT 4
 #define DST_SCOPE_ENV 8
-
+#define DST_SCOPE_TOP 16
-/* Hold a bunch of slots together */
-typedef struct DstSlotPool DstSlotPool;
-struct DstSlotPool {
-    DstSlot *s;
-    int32_t count;
-    int32_t cap;
-    int32_t free;
-};
 
 /* A lexical scope during compilation */
 struct DstScope {
-    int32_t level;
-    DstArray constants; /* Constants for the funcdef */
-    DstTable constantrev; /* Map constants -> constant inidices */
-    DstTable symbols; /* Map symbols -> Slot pointers */
 
-    /* Hold all slots in use. Data structures that store
+    /* Constants for this funcdef */
-     * slots should link them to this datatstructure */
+    int32_t ccount;
-    DstSlotPool slots;
+    int32_t ccap;
-    DstSlotPool unorderedslots;
+    DstValue *consts;
+
+    /* Map of symbols to slots. Use a simple linear scan for symbols. */
+    int32_t symcap;
+    int32_t symcount;
+    struct {
+        const uint8_t *sym;
+        DstSlot slot;
+    } *syms;
+
+    /* Bit vector with allocated slot indices. Used to allocate new slots */
+    uint32_t *slots;
+    int32_t scap;
+    int32_t smax;
 
     /* Referenced closure environents. The values at each index correspond
      * to which index to get the environment from in the parent. The enironment
@@ -116,11 +119,14 @@ struct DstCompiler {
     uint32_t *buffer;
     int32_t *mapbuffer;
 
+    /* Hold the environment */
+    DstValue env;
+
     DstCompileResults results;
 };
 
 #define DST_FOPTS_TAIL 0x10000
-#define DST_FOPTS_FORCESLOT 0x20000
+#define DST_FOPTS_HINT 0x20000
 
 /* Compiler state */
 struct DstFormOptions {
@@ -128,7 +134,7 @@ struct DstFormOptions {
     DstValue x;
     const DstValue *sourcemap;
     uint32_t flags; /* bit set of accepted primitive types */
-    int32_t hint;
+    DstSlot hint;
 };
 
 /* A grouping of optimizations on a cfunction given certain conditions
@@ -149,25 +155,18 @@ extern DstCFunctionOptimizer dst_compiler_optimizers[255];
 /* An array of special forms */
 extern DstSpecial dst_compiler_specials[16];
 
-void dst_compile_slotpool_init(DstSlotPool *pool);
-void dst_compile_slotpool_deinit(DstSlotPool *pool);
-DstSlot *dst_compile_slotpool_alloc(DstSlotPool *pool);
-void dst_compile_slotpool_extend(DstSlotPool *pool, int32_t extra);
-void dst_compile_slotpool_free(DstSlotPool *pool, DstSlot *s);
-void dst_compile_slotpool_freeindex(DstSlotPool *pool, int32_t index);
-
 /* Dispatch to correct form compiler */
-DstSlot *dst_compile_value(DstFormOptions opts);
+DstSlot dst_compile_value(DstFormOptions opts);
 
 /* Compile special forms */
-DstSlot *dst_compile_do(DstFormOptions opts, int32_t argn, const DstValue *argv);
+DstSlot dst_compile_do(DstFormOptions opts, int32_t argn, const DstValue *argv);
-DstSlot *dst_compile_fn(DstFormOptions opts, int32_t argn, const DstValue *argv);
+DstSlot dst_compile_fn(DstFormOptions opts, int32_t argn, const DstValue *argv);
-DstSlot *dst_compile_cond(DstFormOptions opts, int32_t argn, const DstValue *argv);
+DstSlot dst_compile_cond(DstFormOptions opts, int32_t argn, const DstValue *argv);
-DstSlot *dst_compile_while(DstFormOptions opts, int32_t argn, const DstValue *argv);
+DstSlot dst_compile_while(DstFormOptions opts, int32_t argn, const DstValue *argv);
-DstSlot *dst_compile_quote(DstFormOptions opts, int32_t argn, const DstValue *argv);
+DstSlot dst_compile_quote(DstFormOptions opts, int32_t argn, const DstValue *argv);
-DstSlot *dst_compile_def(DstFormOptions opts, int32_t argn, const DstValue *argv);
+DstSlot dst_compile_def(DstFormOptions opts, int32_t argn, const DstValue *argv);
-DstSlot *dst_compile_var(DstFormOptions opts, int32_t argn, const DstValue *argv);
+DstSlot dst_compile_var(DstFormOptions opts, int32_t argn, const DstValue *argv);
-DstSlot *dst_compile_varset(DstFormOptions opts, int32_t argn, const DstValue *argv);
+DstSlot dst_compile_varset(DstFormOptions opts, int32_t argn, const DstValue *argv);
 
 /****************************************************/
 
@@ -183,14 +182,13 @@ DstFormOptions dst_compile_getopts_value(DstFormOptions opts, DstValue key);
 void dst_compile_scope(DstCompiler *c, int newfn);
 void dst_compile_popscope(DstCompiler *c);
 
-DstSlot *dst_compile_constantslot(DstCompiler *c, DstValue x);
+DstSlot dst_compile_constantslot(DstValue x);
-void dst_compile_freeslot(DstCompiler *c, DstSlot *slot);
+void dst_compile_freeslot(DstCompiler *c, DstSlot slot);
 
 /* Search for a symbol */
-DstSlot *dst_compile_resolve(DstCompiler *c, const DstValue *sourcemap, const uint8_t *sym);
+DstSlot dst_compile_resolve(DstCompiler *c, const DstValue *sourcemap, const uint8_t *sym);
 
 /* Emit instructions. */
-
 void dst_compile_emit(DstCompiler *c, const DstValue *sourcemap, uint32_t instr);
 
 #endif

core/fiber.c

@@ -21,10 +21,11 @@
 */
 
 #include <dst/dst.h>
+#include "gc.h"
 
 /* Initialize a new fiber */
 DstFiber *dst_fiber(int32_t capacity) {
-    DstFiber *fiber = dst_alloc(DST_MEMORY_FIBER, sizeof(DstFiber));
+    DstFiber *fiber = dst_gcalloc(DST_MEMORY_FIBER, sizeof(DstFiber));
     fiber->capacity = capacity;
     if (capacity) {
         DstValue *data = malloc(sizeof(DstValue) * capacity);

core/gc.c

@@ -22,12 +22,18 @@
 
 #include <dst/dst.h>
 #include "symcache.h"
+#include "gc.h"
 
 /* GC State */
 void *dst_vm_blocks;
 uint32_t dst_vm_memory_interval;
 uint32_t dst_vm_next_collection;
 
+/* Roots */
+DstValue *dst_vm_roots;
+uint32_t dst_vm_root_count;
+uint32_t dst_vm_root_capacity;
+
 /* Helpers for marking the various gc types */
 static void dst_mark_funcenv(DstFuncEnv *env);
 static void dst_mark_funcdef(DstFuncDef *def);
@@ -58,46 +64,6 @@ void dst_mark(DstValue x) {
     }
 }
 
-/* Pin a value. This prevents a value from being garbage collected.
- * Needed if the valueis not accesible to the garbage collector, but
- * still in use by the program. For example, a c function that
- * creates a table, and then runs the garbage collector without
- * ever saving the table anywhere (except on the c stack, which
- * the gc cannot inspect). */
-void dst_pin(DstValue x) {
-    switch (dst_type(x)) {
-        default: break;
-        case DST_STRING:
-        case DST_SYMBOL: dst_pin_string(dst_unwrap_string(x)); break;
-        case DST_FUNCTION: dst_pin_function(dst_unwrap_function(x)); break;
-        case DST_ARRAY: dst_pin_array(dst_unwrap_array(x)); break;
-        case DST_TABLE: dst_pin_table(dst_unwrap_table(x)); break;
-        case DST_STRUCT: dst_pin_struct(dst_unwrap_struct(x)); break;
-        case DST_TUPLE: dst_pin_tuple(dst_unwrap_tuple(x)); break;
-        case DST_BUFFER: dst_pin_buffer(dst_unwrap_buffer(x)); break;
-        case DST_FIBER: dst_pin_fiber(dst_unwrap_fiber(x)); break;
-        case DST_USERDATA: dst_pin_userdata(dst_unwrap_pointer(x)); break;
-    }
-}
-
-/* Unpin a value. This enables the GC to collect the value's
- * memory again. */
-void dst_unpin(DstValue x) {
-    switch (dst_type(x)) {
-        default: break;
-        case DST_STRING:
-        case DST_SYMBOL: dst_unpin_string(dst_unwrap_string(x)); break;
-        case DST_FUNCTION: dst_unpin_function(dst_unwrap_function(x)); break;
-        case DST_ARRAY: dst_unpin_array(dst_unwrap_array(x)); break;
-        case DST_TABLE: dst_unpin_table(dst_unwrap_table(x)); break;
-        case DST_STRUCT: dst_unpin_struct(dst_unwrap_struct(x)); break;
-        case DST_TUPLE: dst_unpin_tuple(dst_unwrap_tuple(x)); break;
-        case DST_BUFFER: dst_unpin_buffer(dst_unwrap_buffer(x)); break;
-        case DST_FIBER: dst_unpin_fiber(dst_unwrap_fiber(x)); break;
-        case DST_USERDATA: dst_unpin_userdata(dst_unwrap_pointer(x)); break;
-    }
-}
-
 static void dst_mark_string(const uint8_t *str) {
     dst_gc_mark(dst_string_raw(str));
 }
@@ -298,7 +264,7 @@ void dst_sweep() {
 }
 
 /* Allocate some memory that is tracked for garbage collection */
-void *dst_alloc(DstMemoryType type, size_t size) {
+void *dst_gcalloc(DstMemoryType type, size_t size) {
     DstGCMemoryHeader *mdata;
     size_t total = size + sizeof(DstGCMemoryHeader);
 
@@ -326,12 +292,47 @@ void *dst_alloc(DstMemoryType type, size_t size) {
 
 /* Run garbage collection */
 void dst_collect() {
+    uint32_t i;
     if (dst_vm_fiber)
         dst_mark_fiber(dst_vm_fiber);
+    for (i = 0; i < dst_vm_root_count; i++)
+        dst_mark(dst_vm_roots[i]);
     dst_sweep();
     dst_vm_next_collection = 0;
 }
 
+/* Add a root value to the GC. This prevents the GC from removing a value
+ * and all of its children. If gcroot is called on a value n times, unroot
+ * must also be called n times to remove it as a gc root. */
+void dst_gcroot(DstValue root) {
+    uint32_t newcount = dst_vm_root_count + 1;
+    if (newcount > dst_vm_root_capacity) {
+        uint32_t newcap = 2 * newcount;
+        dst_vm_roots = realloc(dst_vm_roots, sizeof(DstValue) * newcap);
+        if (NULL == dst_vm_roots) {
+            DST_OUT_OF_MEMORY;
+        }
+        dst_vm_root_capacity = newcap;
+    }
+    dst_vm_roots[dst_vm_root_count] = root;
+    dst_vm_root_count = newcount;
+}
+
+/* Remove a root value from the GC. This allows the gc to potentially reclaim
+ * a value and all its children. */
+int dst_gcunroot(DstValue root) {
+    DstValue *vtop = dst_vm_roots + dst_vm_root_count;
+    DstValue *v = dst_vm_roots;
+    /* Search from top to bottom as access is most likely LIFO */
+    for (v = dst_vm_roots; v < vtop; v++) {
+        if (dst_equals(root, *v)) {
+            *v = dst_vm_roots[--dst_vm_root_count];
+            return 1;
+        }
+    }
+    return 0;
+}
+
 /* Free all allocated memory */
 void dst_clear_memory() {
     DstGCMemoryHeader *current = dst_vm_blocks;

core/gc.h

@@ -0,0 +1,72 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#ifndef DST_GC_H
+#define DST_GC_H
+
+#include <dst/dst.h>
+
+/* The metadata header associated with an allocated block of memory */
+#define dst_gc_header(mem) ((DstGCMemoryHeader *)(mem) - 1)
+
+#define DST_MEM_TYPEBITS 0xFF
+#define DST_MEM_REACHABLE 0x100
+#define DST_MEM_DISABLED 0x200
+
+#define dst_gc_settype(m, t) ((dst_gc_header(m)->flags |= (0xFF & (t))))
+#define dst_gc_type(m) (dst_gc_header(m)->flags & 0xFF)
+
+#define dst_gc_mark(m) (dst_gc_header(m)->flags |= DST_MEM_REACHABLE)
+#define dst_gc_unmark(m) (dst_gc_header(m)->flags &= ~DST_MEM_COLOR)
+#define dst_gc_reachable(m) (dst_gc_header(m)->flags & DST_MEM_REACHABLE)
+
+
+/* Memory header struct. Node of a linked list of memory blocks. */
+typedef struct DstGCMemoryHeader DstGCMemoryHeader;
+struct DstGCMemoryHeader {
+    DstGCMemoryHeader *next;
+    uint32_t flags;
+};
+
+/* Memory types for the GC. Different from DstType to include funcenv and funcdef. */
+typedef enum DstMemoryType DstMemoryType;
+enum DstMemoryType {
+    DST_MEMORY_NONE,
+    DST_MEMORY_STRING,
+    DST_MEMORY_SYMBOL,
+    DST_MEMORY_ARRAY,
+    DST_MEMORY_TUPLE,
+    DST_MEMORY_TABLE,
+    DST_MEMORY_STRUCT,
+    DST_MEMORY_FIBER,
+    DST_MEMORY_BUFFER,
+    DST_MEMORY_FUNCTION,
+    DST_MEMORY_USERDATA,
+    DST_MEMORY_FUNCENV,
+    DST_MEMORY_FUNCDEF
+};
+
+/* To allocate collectable memory, one must calk dst_alloc, initialize the memory,
+ * and then call when dst_enablegc when it is initailize and reachable by the gc (on the DST stack) */
+void *dst_gcalloc(DstMemoryType type, size_t size);
+
+#endif

core/parse.c

@@ -51,20 +51,94 @@ static int is_whitespace(uint8_t c) {
         || c == ',';
 }
 
-/* Check if a character is a valid symbol character */
+/* Code gen
-/* TODO - allow utf8 - shouldn't be difficult, err on side
+
- * of inclusivity */
+printf("static uint32_t symchars[8] = {\n\t");
-static int is_symbol_char(uint8_t c) {
+for (int i = 0; i < 256; i += 32) {
+    uint32_t block = 0;
+    for (int j = 0; j < 32; j++) {
+        block |= is_symbol_char_gen(i + j) << j;
+    }
+    printf("0x%08x%s", block, (i == (256 - 32)) ? "" : ", ");
+}
+printf("\n};\n");
+
+static int is_symbol_char_gen(uint8_t c) {
     if (c >= 'a' && c <= 'z') return 1;
     if (c >= 'A' && c <= 'Z') return 1;
-    if (c >= '0' && c <= ':') return 1;
+    if (c >= '0' && c <= '9') return 1;
-    if (c >= '<' && c <= '@') return 1;
+    return (c == '!' ||
-    if (c >= '*' && c <= '/') return 1;
+        c == '$' ||
-    if (c >= '$' && c <= '&') return 1;
+        c == '&' ||
-    if (c == '_') return 1;
+        c == '*' ||
-    if (c == '^') return 1;
+        c == '+' ||
-    if (c == '!') return 1;
+        c == '-' ||
+        c == '.' ||
+        c == '/' ||
+        c == ':' ||
+        c == '<' ||
+        c == '=' ||
+        c == '>' ||
+        c == '@' ||
+        c == '\\' ||
+        c == '^' ||
+        c == '_' ||
+        c == '~' ||
+        c == '|');
+}
+
+The table contains 256 bits, where each bit is 1
+if the corresponding ascci code is a symbol char, and 0
+if not. */
+static uint32_t symchars[256] = {
+	0x00000000, 0x77ffec52, 0xd7ffffff, 0x57fffffe,
+	0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
+};
+
+/* Check if a character is a valid symbol character */
+/* TODO - allow utf8 - shouldn't be difficult, err on side
+ * of inclusivity
+ * symbol chars are A-Z, a-z, 0-9, or one of !$&*+-./:<=>@\^_~| */
+static int is_symbol_char(uint8_t c) {
+    return symchars[c >> 5] & (1 << (c & 0x1F));
+}
+
+/* Validate some utf8. Useful for identifiers. Only validates
+ * the encoding, does not check for valid codepoints (they
+ * are less well defined than the encoding). */
+static int valid_utf8(const uint8_t *str, int32_t len) {
+    int32_t i = 0;
+    int32_t j;
+    while (i < len) {
+        int32_t nexti;
+        uint8_t c = str[i];
+
+        /* Check the number of bytes in code point */
+        if (c < 0x80) nexti = i + 1;
+        else if ((c >> 5) == 0x06) nexti = i + 2;
+        else if ((c >> 4) == 0x0E) nexti = i + 3;
+        else if ((c >> 3) == 0x1E) nexti = i + 4;
+        /* Don't allow 5 or 6 byte code points */
+        else return 0;
+
+        /* No overflow */
+        if (nexti > len)
             return 0;
+
+        /* Ensure trailing bytes are well formed (10XX XXXX) */
+        for (j = i + 1; j < nexti; j++) {
+            if ((str[j] >> 6) != 2)
+                return 0;
+        }
+
+        /* Check for overlong encodings */ 
+        if ((nexti == i + 2) && str[i] < 0xC2) return 0;
+        if ((str[i] == 0xE0) && str[i + 1] < 0xA0) return 0;
+        if ((str[i] == 0xF0) && str[i + 1] < 0x90) return 0;
+
+        i = nexti;
+    }
+    return 1;
 }
 
 /* Get hex digit from a letter */
@@ -172,6 +246,8 @@ static const uint8_t *parse_recur(
                 if (*src >= '0' && *src <= '9') {
                     goto sym_nodigits;
                 } else {
+                    if (!valid_utf8(src, tokenend - src))
+                        goto invalid_utf8;
                     ret = dst_symbolv(src, tokenend - src);
                 }
             }
@@ -398,6 +474,11 @@ static const uint8_t *parse_recur(
     args->status = DST_PARSE_ERROR;
     return src;
 
+    invalid_utf8:
+    args->errmsg = "identifier is not valid utf-8";
+    args->status = DST_PARSE_ERROR;
+    return src;
+
     too_much_recur:
     args->errmsg = "recursed too deeply in parsing";
     args->status = DST_PARSE_ERROR;

core/stl.c

@@ -0,0 +1,866 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#include <dst/dst.h>
+
+#define MAKE_BINOP(name, op)\
+static DstValue dst_stl_binop_##name(DstValue lhs, DstValue rhs) {\
+    if (dst_checktype(lhs, DST_INTEGER))\
+        if (dst_checktype(rhs, DST_INTEGER))\
+            return dst_wrap_integer(dst_unwrap_integer(lhs) op dst_unwrap_integer(rhs));\
+        else if (dst_checktype(rhs, DST_REAL))\
+            return dst_wrap_real(dst_unwrap_integer(lhs) op dst_unwrap_real(lhs));\
+        else\
+            return dst_wrap_nil();\
+    else if (dst_checktype(lhs, DST_REAL))\
+        if (dst_checktype(rhs, DST_INTEGER))\
+            return dst_wrap_real(dst_unwrap_real(lhs) op dst_unwrap_integer(rhs));\
+        else if (dst_checktype(rhs, DST_REAL))\
+            return dst_wrap_real(dst_unwrap_real(lhs) op dst_unwrap_real(rhs));\
+        else\
+            return dst_wrap_nil();\
+    else\
+        return dst_wrap_nil();\
+}
+
+#define SIMPLE_ACCUM_FUNCTION(name, op)\
+MAKE_BINOP(name, op)\
+int dst_stl_##name(DstValue *argv, int32_t argn) {\
+    DstValue lhs, rhs;\
+    int32_t j, count;\
+    lhs = argv[0];\
+    rhs = argv[1];\
+    for (j = 1; j < argn; ++j) {\
+        lhs = dst_stl_binop_##name(lhs, rhs);\
+    }\
+    if (dst_checktype(lhs, DST_NIL))\
+        dst_c_throwc(vm, "expected number");\
+    dst_vm_fiber->ret = lhs;
+    return 0;\
+}
+
+SIMPLE_ACCUM_FUNCTION(add, +)
+SIMPLE_ACCUM_FUNCTION(mul, *)
+SIMPLE_ACCUM_FUNCTION(sub, -)
+
+/* Detect division by zero */
+MAKE_BINOP(div, /)
+int dst_stl_div(Dst *vm) {
+    DstValue lhs, rhs;
+    uint32_t j, count;
+    count = dst_args(vm);
+    lhs = dst_arg(vm, 0);
+    for (j = 1; j < count; ++j) {
+        rhs = dst_arg(vm, j);
+        if (lhs.type == DST_INTEGER && rhs.type == DST_INTEGER && rhs.as.integer == 0)
+            dst_c_throwc(vm, "cannot integer divide by 0");
+        lhs = dst_stl_binop_div(lhs, rhs);
+    }
+    if (lhs.type == DST_NIL)
+        dst_c_throwc(vm, "expected number");
+    dst_c_return(vm, lhs);
+}
+
+#undef SIMPLE_ACCUM_FUNCTION
+
+#define BITWISE_FUNCTION(name, op) \
+int dst_stl_##name(Dst *vm) {\
+    DstValue ret;\
+    uint32_t i, count;\
+    count = dst_args(vm);\
+    ret = dst_arg(vm, 0);\
+    if (ret.type != DST_INTEGER) {\
+        dst_c_throwc(vm, "expected integer");\
+    }\
+    if (count < 2) {\
+        dst_c_return(vm, ret);\
+    }\
+    for (i = 1; i < count; ++i) {\
+        DstValue next = dst_arg(vm, i);\
+        if (next.type != DST_INTEGER) {\
+            dst_c_throwc(vm, "expected integer");\
+        }\
+        ret.as.integer = ret.as.integer op next.as.integer;\
+    }\
+    dst_c_return(vm, ret);\
+}
+
+BITWISE_FUNCTION(band, &)
+BITWISE_FUNCTION(bor, |)
+BITWISE_FUNCTION(bxor, ^)
+BITWISE_FUNCTION(blshift, <<)
+BITWISE_FUNCTION(brshift, >>)
+
+#undef BITWISE_FUNCTION
+
+int dst_stl_bnot(Dst *vm) {
+    DstValue in = dst_arg(vm, 0);
+    uint32_t count = dst_args(vm);
+    if (count != 1 || in.type != DST_INTEGER) {
+        dst_c_throwc(vm, "expected 1 integer argument");
+    }
+    in.as.integer = ~in.as.integer;
+    dst_c_return(vm, in);
+}
+
+#define COMPARE_FUNCTION(name, check)\
+int dst_stl_##name(Dst *vm) {\
+    DstValue ret;\
+    uint32_t i, count;\
+    count = dst_args(vm);\
+    ret.as.boolean = 1;\
+    ret.type = DST_BOOLEAN;\
+    if (count < 2) {\
+        dst_c_return(vm, ret);\
+    }\
+    for (i = 1; i < count; ++i) {\
+        DstValue lhs = dst_arg(vm, i - 1);\
+        DstValue rhs = dst_arg(vm, i);\
+        if (!(check)) {\
+            ret.as.boolean = 0;\
+            break;\
+        }\
+    }\
+    dst_c_return(vm, ret);\
+}
+
+COMPARE_FUNCTION(lessthan, dst_compare(lhs, rhs) < 0)
+COMPARE_FUNCTION(greaterthan, dst_compare(lhs, rhs) > 0)
+COMPARE_FUNCTION(equal, dst_equals(lhs, rhs))
+COMPARE_FUNCTION(notequal, !dst_equals(lhs, rhs))
+COMPARE_FUNCTION(lessthaneq, dst_compare(lhs, rhs) <= 0)
+COMPARE_FUNCTION(greaterthaneq, dst_compare(lhs, rhs) >= 0)
+
+#undef COMPARE_FUNCTION
+
+/* Boolean not */
+int dst_stl_not(Dst *vm) {
+    dst_c_return(vm, boolean(!dst_truthy(dst_arg(vm, 0))));
+}
+
+/****/
+/* Core */
+/****/
+
+/* Empty a mutable datastructure */
+int dst_stl_clear(Dst *vm) {
+    DstValue x = dst_arg(vm, 0);
+    switch (x.type) {
+    default:
+        dst_c_throwc(vm, "cannot clear");
+    case DST_ARRAY:
+        x.as.array->count = 0;
+        break;
+    case DST_BUFFER:
+        x.as.buffer->count = 0;
+        break;
+    case DST_TABLE:
+        dst_table_clear(x.as.table);
+        break;
+    }
+    dst_c_return(vm, x);
+}
+
+/* Get length of object */
+int dst_stl_length(Dst *vm) {
+    dst_set_integer(vm, 0, dst_length(vm, 0));
+    dst_return(vm, 0);
+    return 0;
+}
+
+/* Get hash of a value */
+int dst_stl_hash(Dst *vm) {
+    dst_set_integer(vm, 0, dst_hash(vm, 0););
+    dst_return(vm, 0);
+    return 0;
+}
+
+/* Convert to integer */
+int dst_stl_to_int(Dst *vm) {
+    DstValue x = dst_arg(vm, 0);
+    if (x.type == DST_INTEGER) dst_c_return(vm, x);
+    if (x.type == DST_REAL)
+        dst_c_return(vm, integer((DstInteger) x.as.real));
+    else
+       dst_c_throwc(vm, "expected number");
+}
+
+/* Convert to integer */
+int dst_stl_to_real(Dst *vm) {
+    DstValue x = dst_arg(vm, 0);
+    if (x.type == DST_REAL) dst_c_return(vm, x);
+    if (x.type == DST_INTEGER)
+        dst_c_return(vm, dst_wrap_real((DstReal) x.as.integer));
+    else
+       dst_c_throwc(vm, "expected number");
+}
+
+/* Get a slice of a sequence */
+int dst_stl_slice(Dst *vm) {
+    uint32_t count = dst_args(vm);
+    int32_t from, to;
+    DstValue x;
+    const DstValue *data;
+    const uint8_t *cdata;
+    uint32_t length;
+    uint32_t newlength;
+    DstInteger num;
+
+    /* Get data */
+    x = dst_arg(vm, 0);
+    if (!dst_seq_view(x, &data, &length) &&
+            !dst_chararray_view(x, &cdata, &length))  {
+        dst_c_throwc(vm, "expected array/tuple/buffer/symbol/string");
+    }
+
+    /* Get from index */
+    if (count < 2) {
+        from = 0;
+    } else {
+        if (!dst_check_integer(vm, 1, &num))
+            dst_c_throwc(vm, DST_EXPECTED_INTEGER);
+        from = dst_startrange(num, length);
+    }
+
+    /* Get to index */
+    if (count < 3) {
+        to = length;
+    } else {
+        if (!dst_check_integer(vm, 2, &num))
+            dst_c_throwc(vm, DST_EXPECTED_INTEGER);
+        to = dst_endrange(num, length);
+    }
+
+    /* Check from bad bounds */
+    if (from < 0 || to < 0 || to < from)
+        dst_c_throwc(vm, "index out of bounds");
+
+    /* Build slice */
+    newlength = to - from;
+    if (x.type == DST_TUPLE) {
+        DstValue *tup = dst_tuple_begin(vm, newlength);
+        dst_memcpy(tup, data + from, newlength * sizeof(DstValue));
+        dst_c_return(vm, dst_wrap_tuple(dst_tuple_end(vm, tup)));
+    } else if (x.type == DST_ARRAY) {
+        DstArray *arr = dst_array(vm, newlength);
+        arr->count = newlength;
+        dst_memcpy(arr->data, data + from, newlength * sizeof(DstValue));
+        dst_c_return(vm, dst_wrap_array(arr));
+    } else if (x.type == DST_STRING) {
+        dst_c_return(vm, dst_wrap_string(dst_string_b(vm, x.as.string + from, newlength)));
+    } else if (x.type == DST_SYMBOL) {
+        dst_c_return(vm, dst_wrap_symbol(dst_string_b(vm, x.as.string + from, newlength)));
+    } else { /* buffer */
+        DstBuffer *b = dst_buffer(vm, newlength);
+        dst_memcpy(b->data, x.as.buffer->data, newlength);
+        b->count = newlength;
+        dst_c_return(vm, dst_wrap_buffer(b));
+    }
+}
+
+/* Create array */
+int dst_stl_array(Dst *vm) {
+    uint32_t i;
+    uint32_t count = dst_args(vm);
+    DstArray *array = dst_array(vm, count);
+    for (i = 0; i < count; ++i)
+        array->data[i] = dst_arg(vm, i);
+    dst_c_return(vm, dst_wrap_array(array));
+}
+
+/* Create tuple */
+int dst_stl_tuple(Dst *vm) {
+    uint32_t i;
+    uint32_t count = dst_args(vm);
+    DstValue *tuple= dst_tuple_begin(vm, count);
+    for (i = 0; i < count; ++i)
+        tuple[i] = dst_arg(vm, i);
+    dst_c_return(vm, dst_wrap_tuple(dst_tuple_end(vm, tuple)));
+}
+
+/* Create object */
+int dst_stl_table(Dst *vm) {
+    uint32_t i;
+    uint32_t count = dst_args(vm);
+    DstTable *table;
+    if (count % 2 != 0)
+        dst_c_throwc(vm, "expected even number of arguments");
+    table = dst_table(vm, 4 * count);
+    for (i = 0; i < count; i += 2)
+        dst_table_put(vm, table, dst_arg(vm, i), dst_arg(vm, i + 1));
+    dst_c_return(vm, dst_wrap_table(table));
+}
+
+/* Create struct */
+int dst_stl_struct(Dst *vm) {
+    uint32_t i;
+    uint32_t count = dst_args(vm);
+    DstValue *st;
+    if (count % 2 != 0)
+        dst_c_throwc(vm, "expected even number of arguments");
+    st = dst_struct_begin(vm, count / 2);
+    for (i = 0; i < count; i += 2)
+        dst_struct_put(st, dst_arg(vm, i), dst_arg(vm, i + 1));
+    dst_c_return(vm, dst_wrap_struct(dst_struct_end(vm, st)));
+}
+
+/* Create a buffer */
+int dst_stl_buffer(Dst *vm) {
+    uint32_t i, count;
+    const uint8_t *dat;
+    uint32_t slen;
+    DstBuffer *buf = dst_buffer(vm, 10);
+    count = dst_args(vm);
+    for (i = 0; i < count; ++i) {
+        if (dst_chararray_view(dst_arg(vm, i), &dat, &slen))
+            dst_buffer_append(vm, buf, dat, slen);
+        else
+            dst_c_throwc(vm, DST_EXPECTED_STRING);
+    }
+    dst_c_return(vm, dst_wrap_buffer(buf));
+}
+
+/* Create a string */
+int dst_stl_string(Dst *vm) {
+    uint32_t j;
+    uint32_t count = dst_args(vm);
+    uint32_t length = 0;
+    uint32_t index = 0;
+    uint8_t *str;
+    const uint8_t *dat;
+    uint32_t slen;
+    /* Find length and assert string arguments */
+    for (j = 0; j < count; ++j) {
+        if (!dst_chararray_view(dst_arg(vm, j), &dat, &slen)) {
+            DstValue newarg;
+            dat = dst_to_string(vm, dst_arg(vm, j));
+            slen = dst_string_length(dat);
+            newarg.type = DST_STRING;
+            newarg.as.string = dat;
+            dst_set_arg(vm, j, newarg);
+        }
+        length += slen;
+    }
+    /* Make string */
+    str = dst_string_begin(vm, length);
+    for (j = 0; j < count; ++j) {
+        dst_chararray_view(dst_arg(vm, j), &dat, &slen);
+        dst_memcpy(str + index, dat, slen);
+        index += slen;
+    }
+    dst_c_return(vm, dst_wrap_string(dst_string_end(vm, str)));
+}
+
+/* Create a symbol */
+int dst_stl_symbol(Dst *vm) {
+    int ret = dst_stl_string(vm);
+    if (ret == DST_RETURN_OK) {
+        vm->ret.type = DST_SYMBOL;
+    }
+    return ret;
+}
+
+/* Create a thread */
+int dst_stl_thread(Dst *vm) {
+    DstThread *t;
+    DstValue callee = dst_arg(vm, 0);
+    DstValue parent = dst_arg(vm, 1);
+    DstValue errorParent = dst_arg(vm, 2);
+    t = dst_thread(vm, callee, 10);
+    if (callee.type != DST_FUNCTION && callee.type != DST_CFUNCTION)
+        dst_c_throwc(vm, "expected function in thread constructor");
+    if (parent.type == DST_THREAD) {
+        t->parent = parent.as.thread;
+    } else if (parent.type != DST_NIL) {
+        dst_c_throwc(vm, "expected thread/nil as parent");
+    } else {
+        t->parent = vm->thread;
+    }
+    dst_c_return(vm, dst_wrap_thread(t));
+}
+
+/* Get current thread */
+int dst_stl_current(Dst *vm) {
+    dst_c_return(vm, dst_wrap_thread(vm->thread));
+}
+
+/* Get parent of a thread */
+/* TODO - consider implications of this function
+ * for sandboxing */
+int dst_stl_parent(Dst *vm) {
+    DstThread *t;
+    if (!dst_check_thread(vm, 0, &t))
+        dst_c_throwc(vm, "expected thread");
+    if (t->parent == NULL)
+        dst_c_return(vm, dst_wrap_nil());
+    dst_c_return(vm, dst_wrap_thread(t->parent));
+}
+
+/* Get the status of a thread */
+int dst_stl_status(Dst *vm) {
+    DstThread *t;
+    const char *cstr;
+    if (!dst_check_thread(vm, 0, &t))
+        dst_c_throwc(vm, "expected thread");
+    switch (t->status) {
+        case DST_THREAD_PENDING:
+            cstr = "pending";
+            break;
+        case DST_THREAD_ALIVE:
+            cstr = "alive";
+            break;
+        case DST_THREAD_DEAD:
+            cstr = "dead";
+            break;
+        case DST_THREAD_ERROR:
+            cstr = "error";
+            break;
+    }
+    dst_c_return(vm, dst_string_cv(vm, cstr));
+}
+
+/* Associative get */
+int dst_stl_get(Dst *vm) {
+    DstValue ret;
+    uint32_t count;
+    const char *err;
+    count = dst_args(vm);
+    if (count != 2)
+        dst_c_throwc(vm, "expects 2 arguments");
+    err = dst_get(dst_arg(vm, 0), dst_arg(vm, 1), &ret);
+    if (err != NULL)
+        dst_c_throwc(vm, err);
+    else
+        dst_c_return(vm, ret);
+}
+
+/* Associative set */
+int dst_stl_set(Dst *vm) {
+    uint32_t count;
+    const char *err;
+    count = dst_args(vm);
+    if (count != 3)
+        dst_c_throwc(vm, "expects 3 arguments");
+    err = dst_set(vm, dst_arg(vm, 0), dst_arg(vm, 1), dst_arg(vm, 2));
+    if (err != NULL)
+        dst_c_throwc(vm, err);
+    else
+        dst_c_return(vm, dst_arg(vm, 0));
+}
+
+/* Push to end of array */
+int dst_stl_push(Dst *vm) {
+    DstValue ds = dst_arg(vm, 0);
+    if (ds.type != DST_ARRAY)
+        dst_c_throwc(vm, "expected array");
+    dst_array_push(vm, ds.as.array, dst_arg(vm, 1));
+    dst_c_return(vm, ds);
+}
+
+/* Pop from end of array */
+int dst_stl_pop(Dst *vm) {
+    DstValue ds = dst_arg(vm, 0);
+    if (ds.type != DST_ARRAY)
+        dst_c_throwc(vm, "expected array");
+    dst_c_return(vm, dst_array_pop(ds.as.array));
+}
+
+/* Peek at end of array */
+int dst_stl_peek(Dst *vm) {
+    DstValue ds = dst_arg(vm, 0);
+    if (ds.type != DST_ARRAY)
+        dst_c_throwc(vm, "expected array");
+    dst_c_return(vm, dst_array_peek(ds.as.array));
+}
+
+/* Ensure array capacity */
+int dst_stl_ensure(Dst *vm) {
+    DstValue ds = dst_arg(vm, 0);
+    DstValue cap = dst_arg(vm, 1);
+    if (ds.type != DST_ARRAY)
+        dst_c_throwc(vm, "expected array");
+    if (cap.type != DST_INTEGER)
+        dst_c_throwc(vm, DST_EXPECTED_INTEGER);
+    dst_array_ensure(vm, ds.as.array, (uint32_t) cap.as.integer);
+    dst_c_return(vm, ds);
+}
+
+/* Get next key in struct or table */
+int dst_stl_next(Dst *vm) {
+    DstValue ds = dst_arg(vm, 0);
+    DstValue key = dst_arg(vm, 1);
+    if (ds.type == DST_TABLE) {
+        dst_c_return(vm, dst_table_next(ds.as.table, key));
+    } else if (ds.type == DST_STRUCT) {
+        dst_c_return(vm, dst_struct_next(ds.as.st, key));
+    } else {
+        dst_c_throwc(vm, "expected table or struct");
+    }
+}
+
+/* Print values for inspection */
+int dst_stl_print(Dst *vm) {
+    uint32_t j, count;
+    count = dst_args(vm);
+    for (j = 0; j < count; ++j) {
+        uint32_t i;
+        const uint8_t *string = dst_to_string(vm, dst_arg(vm, j));
+        uint32_t len = dst_string_length(string);
+        for (i = 0; i < len; ++i)
+            fputc(string[i], stdout);
+    }
+    fputc('\n', stdout);
+    return DST_RETURN_OK;
+}
+
+/* Long description */
+int dst_stl_description(Dst *vm) {
+    DstValue x = dst_arg(vm, 0);
+    const uint8_t *buf = dst_description(vm, x);
+    dst_c_return(vm, dst_wrap_string(buf));
+}
+
+/* Short description */
+int dst_stl_short_description(Dst *vm) {
+    DstValue x = dst_arg(vm, 0);
+    const uint8_t *buf = dst_short_description(vm, x);
+    dst_c_return(vm, dst_wrap_string(buf));
+}
+
+/* Exit */
+int dst_stl_exit(Dst *vm) {
+    int ret;
+    DstValue x = dst_arg(vm, 0);
+    ret = x.type == DST_INTEGER ? x.as.integer : (x.type == DST_REAL ? x.as.real : 0);
+    exit(ret);
+    return DST_RETURN_OK;
+}
+
+/* Throw error */
+int dst_stl_error(Dst *vm) {
+    dst_c_throw(vm, dst_arg(vm, 0));
+}
+
+/***/
+/* Function reflection */
+/***/
+
+int dst_stl_funcenv(Dst *vm) {
+    DstFunction *fn;
+    if (!dst_check_function(vm, 0, &fn))
+        dst_c_throwc(vm, "expected function");
+    if (fn->env)
+        dst_c_return(vm, dst_wrap_funcenv(fn->env));
+    else
+        return DST_RETURN_OK;
+}
+
+int dst_stl_funcdef(Dst *vm) {
+    DstFunction *fn;
+    if (!dst_check_function(vm, 0, &fn))
+        dst_c_throwc(vm, "expected function");
+    dst_c_return(vm, dst_wrap_funcdef(fn->def));
+}
+
+int dst_stl_funcparent(Dst *vm) {
+    DstFunction *fn;
+    if (!dst_check_function(vm, 0, &fn))
+        dst_c_throwc(vm, "expected function");
+    if (fn->parent)
+        dst_c_return(vm, dst_wrap_function(fn->parent));
+    else
+        return DST_RETURN_OK;
+}
+
+int dst_stl_def(Dst *vm) {
+    DstValue key = dst_arg(vm, 0);
+    if (dst_args(vm) != 2) {
+        dst_c_throwc(vm, "expected 2 arguments to global-def");
+    }
+    if (key.type != DST_STRING && key.type != DST_SYMBOL) {
+        dst_c_throwc(vm, "expected string/symbol as first argument");
+    }
+    key.type = DST_SYMBOL;
+    dst_env_put(vm, vm->env, key, dst_arg(vm, 1));
+    dst_c_return(vm, dst_arg(vm, 1));
+}
+
+int dst_stl_var(Dst *vm) {
+    DstValue key = dst_arg(vm, 0);
+    if (dst_args(vm) != 2) {
+        dst_c_throwc(vm, "expected 2 arguments to global-var");
+    }
+    if (key.type != DST_STRING && key.type != DST_SYMBOL) {
+        dst_c_throwc(vm, "expected string as first argument");
+    }
+    key.type = DST_SYMBOL;
+    dst_env_putvar(vm, vm->env, key, dst_arg(vm, 1));
+    dst_c_return(vm, dst_arg(vm, 1));
+}
+
+/****/
+/* IO */
+/****/
+
+/* File type definition */
+static DstUserType dst_stl_filetype = {
+    "stl.file",
+    NULL,
+    NULL,
+    NULL,
+    NULL
+};
+
+/* Open a a file and return a userdata wrapper arounf the C file API. */
+int dst_stl_open(Dst *vm) {
+    const uint8_t *fname = dst_to_string(vm, dst_arg(vm, 0));
+    const uint8_t *fmode = dst_to_string(vm, dst_arg(vm, 1));
+    FILE *f;
+    FILE **fp;
+    if (dst_args(vm) < 2 || dst_arg(vm, 0).type != DST_STRING
+            || dst_arg(vm, 1).type != DST_STRING)
+        dst_c_throwc(vm, "expected filename and filemode");
+    f = fopen((const char *)fname, (const char *)fmode);
+    if (!f)
+        dst_c_throwc(vm, "could not open file");
+    fp = dst_userdata(vm, sizeof(FILE *), &dst_stl_filetype);
+    *fp = f;
+    dst_c_return(vm, dst_wrap_userdata(fp));
+}
+
+/* Read an entire file into memory */
+int dst_stl_slurp(Dst *vm) {
+    DstBuffer *b;
+    long fsize;
+    FILE *f;
+    FILE **fp = dst_check_userdata(vm, 0, &dst_stl_filetype);
+    if (fp == NULL) dst_c_throwc(vm, "expected file");
+    if (!dst_check_buffer(vm, 1, &b)) b = dst_buffer(vm, 10);
+    f = *fp;
+    /* Read whole file */
+    fseek(f, 0, SEEK_END);
+    fsize = ftell(f);
+    fseek(f, 0, SEEK_SET);
+    /* Ensure buffer size */
+    dst_buffer_ensure(vm, b, b->count + fsize);
+    fread((char *)(b->data + b->count), fsize, 1, f);
+    b->count += fsize;
+    dst_c_return(vm, dst_wrap_buffer(b));
+}
+
+/* Read a certain number of bytes into memory */
+int dst_stl_read(Dst *vm) {
+    DstBuffer *b;
+    FILE *f;
+    int64_t len;
+    FILE **fp = dst_check_userdata(vm, 0, &dst_stl_filetype);
+    if (fp == NULL) dst_c_throwc(vm, "expected file");
+    if (!(dst_check_integer(vm, 1, &len))) dst_c_throwc(vm, "expected integer");
+    if (!dst_check_buffer(vm, 2, &b)) b = dst_buffer(vm, 10);
+    f = *fp;
+    /* Ensure buffer size */
+    dst_buffer_ensure(vm, b, b->count + len);
+    b->count += fread((char *)(b->data + b->count), len, 1, f) * len;
+    dst_c_return(vm, dst_wrap_buffer(b));
+}
+
+/* Write bytes to a file */
+int dst_stl_write(Dst *vm) {
+    FILE *f;
+    const uint8_t *data;
+    uint32_t len;
+    FILE **fp = dst_check_userdata(vm, 0, &dst_stl_filetype);
+    if (fp == NULL) dst_c_throwc(vm, "expected file");
+    if (!dst_chararray_view(dst_arg(vm, 1), &data, &len)) dst_c_throwc(vm, "expected string|buffer");
+    f = *fp;
+    fwrite(data, len, 1, f);
+    return DST_RETURN_OK;
+}
+
+/* Close a file */
+int dst_stl_close(Dst *vm) {
+    FILE **fp = dst_check_userdata(vm, 0, &dst_stl_filetype);
+    if (fp == NULL) dst_c_throwc(vm, "expected file");
+    fclose(*fp);
+    dst_c_return(vm, dst_wrap_nil());
+}
+
+/****/
+/* Temporary */
+/****/
+
+/* Force garbage collection */
+int dst_stl_gcollect(Dst *vm) {
+	dst_collect(vm);
+	return DST_RETURN_OK;
+}
+
+/***/
+/* Compilation */
+/***/
+
+/* Generate a unique symbol */
+static int dst_stl_gensym(Dst *vm) {
+    DstValue source = dst_arg(vm, 0);
+    const uint8_t *sym = NULL;
+    uint32_t len;
+    const uint8_t *data;
+    if (source.type == DST_NIL) {
+        sym = dst_string_cu(vm, "");
+    } else if (dst_chararray_view(source, &data, &len)) {
+        sym = dst_string_bu(vm, data, len);
+    } else {
+        dst_c_throwc(vm, "exepcted string/buffer/symbol/nil");
+    }
+    dst_c_return(vm, dst_wrap_symbol(sym));
+}
+
+/* Compile a value */
+static int dst_stl_compile(Dst *vm) {
+    DstTable *env = vm->env;
+    if (dst_arg(vm, 1).type == DST_TABLE) {
+        env = dst_arg(vm, 1).as.table;
+    }
+    dst_c_return(vm, dst_compile(vm, env, dst_arg(vm, 0)));
+}
+
+/* Get vm->env */
+static int dst_stl_getenv(Dst *vm) {
+    dst_c_return(vm, dst_wrap_table(vm->env));
+}
+
+/* Set vm->env */
+static int dst_stl_setenv(Dst *vm) {
+    DstValue newEnv = dst_arg(vm, 0);
+    if (newEnv.type != DST_TABLE) {
+        dst_c_throwc(vm, "expected table");
+    }
+    vm->env = newEnv.as.table;
+    return DST_RETURN_OK;
+}
+
+/****/
+/* Bootstraping */
+/****/
+
+static const DstModuleItem std_module[] = {
+    /* Arithmetic */
+    {"+", dst_stl_add},
+    {"*", dst_stl_mul},
+    {"-", dst_stl_sub},
+    {"/", dst_stl_div},
+    /* Comparisons */
+    {"<", dst_stl_lessthan},
+    {">", dst_stl_greaterthan},
+    {"=", dst_stl_equal},
+    {"not=", dst_stl_notequal},
+    {"<=", dst_stl_lessthaneq},
+    {">=", dst_stl_greaterthaneq},
+    /* Bitwise arithmetic */
+    {"band", dst_stl_band},
+    {"bor", dst_stl_bor},
+    {"bxor", dst_stl_bxor},
+    {"blshift", dst_stl_blshift},
+    {"brshift", dst_stl_brshift},
+    {"bnot", dst_stl_bnot},
+    /* IO */
+    {"open", dst_stl_open},
+    {"slurp", dst_stl_slurp},
+    {"read", dst_stl_read},
+    {"write", dst_stl_write},
+    /* Compile */
+    {"gensym", dst_stl_gensym},
+    {"getenv", dst_stl_getenv},
+    {"setenv", dst_stl_setenv},
+    {"compile", dst_stl_compile},
+    /* Other */
+    {"not", dst_stl_not},
+    {"clear", dst_stl_clear},
+    {"length", dst_stl_length},
+    {"hash", dst_stl_hash},
+    {"integer", dst_stl_to_int},
+    {"real", dst_stl_to_real},
+    {"type", dst_stl_type},
+    {"slice", dst_stl_slice},
+    {"array", dst_stl_array},
+    {"tuple", dst_stl_tuple},
+    {"table", dst_stl_table},
+    {"struct", dst_stl_struct},
+    {"buffer", dst_stl_buffer},
+    {"string", dst_stl_string},
+    {"symbol", dst_stl_symbol},
+    {"thread", dst_stl_thread},
+    {"status", dst_stl_status},
+    {"current", dst_stl_current},
+    {"parent", dst_stl_parent},
+    {"print", dst_stl_print},
+    {"description", dst_stl_description},
+    {"short-description", dst_stl_short_description},
+    {"exit!", dst_stl_exit},
+    {"get", dst_stl_get},
+    {"set!", dst_stl_set},
+    {"next", dst_stl_next},
+    {"error", dst_stl_error},
+    {"push!", dst_stl_push},
+    {"pop!", dst_stl_pop},
+    {"peek", dst_stl_peek},
+    {"ensure!", dst_stl_ensure},
+    {"open", dst_stl_open},
+    {"slurp", dst_stl_slurp},
+    {"read", dst_stl_read},
+    {"write", dst_stl_write},
+    {"close", dst_stl_close},
+    {"funcenv", dst_stl_funcenv},
+    {"funcdef", dst_stl_funcdef},
+    {"funcparent", dst_stl_funcparent},
+    {"gcollect", dst_stl_gcollect},
+    {"global-def", dst_stl_def},
+    {"global-var", dst_stl_var},
+    {NULL, NULL}
+};
+
+/* Load stl library into the current environment. Create stl module object
+ * only if it is not yet created. */
+void dst_stl_load(Dst *vm) {
+    DstValue maybeEnv = dst_table_get(vm->modules, dst_string_cvs(vm, "std"));
+    if (maybeEnv.type == DST_TABLE) {
+        /* Module already created, so merge into main vm. */
+        dst_env_merge(vm, vm->env, maybeEnv.as.table);
+    } else {
+        /* Module not yet created */
+        /* Load the normal c functions */
+        dst_module_mutable(vm, "std", std_module);
+        /* Wrap stdin and stdout */
+        FILE **inp = dst_userdata(vm, sizeof(FILE *), &dst_stl_filetype);
+        FILE **outp = dst_userdata(vm, sizeof(FILE *), &dst_stl_filetype);
+        FILE **errp = dst_userdata(vm, sizeof(FILE *), &dst_stl_filetype);
+        *inp = stdin;
+        *outp = stdout;
+        *errp = stderr;
+        dst_module_put(vm, "std", "stdin", dst_wrap_userdata(inp));
+        dst_module_put(vm, "std", "stdout", dst_wrap_userdata(outp));
+        dst_module_put(vm, "std", "stderr", dst_wrap_userdata(outp));
+        /* Now merge */
+        maybeEnv = dst_table_get(vm->modules, dst_string_cvs(vm, "std"));
+        dst_env_merge(vm, vm->env, maybeEnv.as.table);
+    }
+}

core/string.c

@@ -22,10 +22,11 @@
 
 #include <dst/dst.h>
 #include "strtod.h"
+#include "gc.h"
 
 /* Begin building a string */
 uint8_t *dst_string_begin(int32_t length) {
-    char *data = dst_alloc(DST_MEMORY_STRING, 2 * sizeof(int32_t) + length);
+    char *data = dst_gcalloc(DST_MEMORY_STRING, 2 * sizeof(int32_t) + length);
     uint8_t *str = (uint8_t *) (data + 2 * sizeof(int32_t));
     dst_string_length(str) = length;
     return str;
@@ -40,7 +41,7 @@ const uint8_t *dst_string_end(uint8_t *str) {
 /* Load a buffer as a string */
 const uint8_t *dst_string(const uint8_t *buf, int32_t len) {
     int32_t hash = dst_string_calchash(buf, len);
-    char *data = dst_alloc(DST_MEMORY_STRING, 2 * sizeof(int32_t) + len);
+    char *data = dst_gcalloc(DST_MEMORY_STRING, 2 * sizeof(int32_t) + len);
     uint8_t *str = (uint8_t *) (data + 2 * sizeof(int32_t));
     memcpy(str, buf, len);
     dst_string_length(str) = len;

core/struct.c

@@ -21,6 +21,7 @@
 */
 
 #include <dst/dst.h>
+#include "gc.h"
 
 #define dst_struct_maphash(cap, hash) (((uint32_t)(hash) % (cap)) & 0xFFFFFFFE);
 
@@ -34,7 +35,7 @@ DstValue *dst_struct_begin(int32_t count) {
      * is added or s is changed, change the macro dst_struct_capacity in internal.h */
     int32_t capacity = 4 * count;
     size_t s = sizeof(int32_t) * 2 + (capacity * sizeof(DstValue));
-    char *data = dst_alloc(DST_MEMORY_STRUCT, s);
+    char *data = dst_gcalloc(DST_MEMORY_STRUCT, s);
     DstValue *st = (DstValue *) (data + 2 * sizeof(int32_t));
     dst_memempty(st, capacity);
     dst_struct_length(st) = count;

core/symcache.c

@@ -26,6 +26,7 @@
  * whole program. Equality is then just a pointer check. */
 
 #include <dst/dst.h>
+#include "gc.h"
 
 /* Cache state */
 const uint8_t **dst_vm_cache = NULL;
@@ -170,7 +171,7 @@ const uint8_t *dst_symbol(const uint8_t *str, int32_t len) {
     const uint8_t **bucket = dst_symcache_findmem(str, len, hash, &success);
     if (success)
         return *bucket;
-    newstr = dst_alloc(DST_MEMORY_SYMBOL, 2 * sizeof(int32_t) + len)
+    newstr = dst_gcalloc(DST_MEMORY_SYMBOL, 2 * sizeof(int32_t) + len)
         + (2 * sizeof(int32_t));
     dst_string_hash(newstr) = hash;
     dst_string_length(newstr) = len;
@@ -223,7 +224,7 @@ const uint8_t *dst_symbol_gen(const uint8_t *buf, int32_t len) {
      * is enough for resolving collisions. */
     int32_t newlen = len + 8;
     int32_t newbufsize = newlen + 2 * sizeof(int32_t);
-    uint8_t *str = (uint8_t *)(dst_alloc(DST_MEMORY_SYMBOL, newbufsize) + 2 * sizeof(int32_t));
+    uint8_t *str = (uint8_t *)(dst_gcalloc(DST_MEMORY_SYMBOL, newbufsize) + 2 * sizeof(int32_t));
     dst_string_length(str) = newlen;
     memcpy(str, buf, len);
     str[len] = '-';

core/table.c

@@ -21,6 +21,7 @@
 */
 
 #include <dst/dst.h>
+#include "gc.h"
 
 #define dst_table_maphash(cap, hash) (((uint32_t)(hash) % (cap)) & 0xFFFFFFFE)
 
@@ -46,7 +47,7 @@ void dst_table_deinit(DstTable *table) {
 
 /* Create a new table */
 DstTable *dst_table(int32_t capacity) {
-    DstTable *table = dst_alloc(DST_MEMORY_TABLE, sizeof(DstTable));
+    DstTable *table = dst_gcalloc(DST_MEMORY_TABLE, sizeof(DstTable));
     return dst_table_init(table, capacity);
 }
 
@@ -56,6 +57,7 @@ static DstValue *dst_table_find(DstTable *t, DstValue key) {
     int32_t index = dst_table_maphash(t->capacity, dst_hash(key));
     int32_t i, j;
     int32_t start[2], end[2];
+    DstValue *first_bucket = NULL;
     start[0] = index; end[0] = t->capacity;
     start[1] = 0; end[1] = index;
     for (j = 0; j < 2; ++j) {
@@ -64,13 +66,16 @@ static DstValue *dst_table_find(DstTable *t, DstValue key) {
                 if (dst_checktype(t->data[i + 1], DST_NIL)) {
                     /* Empty */
                     return t->data + i;
+                } else if (NULL == first_bucket) {
+                    /* Marked deleted and not seen free bucket yet. */
+                    first_bucket = t->data + i;
                 }
             } else if (dst_equals(t->data[i], key)) {
                 return t->data + i;
             }
         }
     }
-    return NULL;
+    return first_bucket;
 }
 
 /* Resize the dictionary table. */
@@ -176,10 +181,24 @@ const DstValue *dst_table_to_struct(DstTable *t) {
     int32_t i;
     DstValue *st = dst_struct_begin(t->count);
     for (i = 0; i < t->capacity; i += 2) {
-        if (!dst_checktype(t->data[i], DST_NIL))
+        if (!dst_checktype(t->data[i], DST_NIL)) {
             dst_struct_put(st, t->data[i], t->data[i + 1]);
         }
+    }
     return dst_struct_end(st);
 }
 
+/* Merge a struct or another table into a table. */
+void dst_table_merge(DstTable *t,  DstValue other) {
+    int32_t count, cap, i;
+    const DstValue *hmap;
+    if (dst_hashtable_view(other, &hmap, &count, &cap)) {
+        for (i = 0; i < cap; i += 2) {
+            if (!dst_checktype(hmap[i], DST_NIL)) {
+                dst_table_put(t, hmap[i], hmap[i + 1]);
+            }
+        }
+    }
+}
+
 #undef dst_table_maphash

core/tuple.c

@@ -22,12 +22,13 @@
 
 #include <dst/dst.h>
 #include "symcache.h"
+#include "gc.h"
 
 /* Create a new empty tuple of the given size. This will return memory
  * which should be filled with DstValues. The memory will not be collected until
  * dst_tuple_end is called. */
 DstValue *dst_tuple_begin(int32_t length) {
-    char *data = dst_alloc(DST_MEMORY_TUPLE, 2 * sizeof(int32_t) + length * sizeof(DstValue));
+    char *data = dst_gcalloc(DST_MEMORY_TUPLE, 2 * sizeof(int32_t) + length * sizeof(DstValue));
     DstValue *tuple = (DstValue *)(data + (2 * sizeof(int32_t)));
     dst_tuple_length(tuple) = length;
     return tuple;

core/userdata.c

@@ -21,10 +21,11 @@
 */
 
 #include <dst/dst.h>
+#include "gc.h"
 
 /* Create new userdata */
 void *dst_userdata(size_t size, const DstUserType *utype) {
-    char *data = dst_alloc(DST_MEMORY_USERDATA, sizeof(DstUserdataHeader) + size);
+    char *data = dst_gcalloc(DST_MEMORY_USERDATA, sizeof(DstUserdataHeader) + size);
     DstUserdataHeader *header = (DstUserdataHeader *)data;
     void *user = data + sizeof(DstUserdataHeader);
     header->size = size;

core/util.c

@@ -116,3 +116,16 @@ int dst_hashtable_view(DstValue tab, const DstValue **data, int32_t *len, int32_
     return 0;
 }
 
+/* Load c functions into an environment */
+DstValue dst_loadreg(DstReg *regs, size_t count) {
+    size_t i;
+    DstTable *t = dst_table(count);
+    for (i = 0; i < count; i++) {
+        DstValue sym = dst_csymbolv(regs[i].name);
+        DstValue func = dst_wrap_cfunction(regs[i].function);
+        DstTable *subt = dst_table(1);
+        dst_table_put(subt, dst_csymbolv("value"), func);
+        dst_table_put(t, sym, dst_wrap_table(subt));
+    }
+    return dst_wrap_table(t);
+}

core/vm.c

@@ -23,9 +23,10 @@
 #include <dst/dst.h>
 #include "opcodes.h"
 #include "symcache.h"
+#include "gc.h"
 
 /* VM State */
-DstFiber *dst_vm_fiber;
+DstFiber *dst_vm_fiber = NULL;
 
 /* Helper to ensure proper fiber is activated after returning */
 static int dst_update_fiber() {
@@ -417,14 +418,14 @@ int dst_continue() {
             vm_assert((int32_t)oparg(2, 0xFFFF) < func->def->constants_length, "invalid constant");
             vm_assert(dst_checktype(func->def->constants[oparg(2, 0xFFFF)], DST_NIL), "constant must be funcdef");
             fd = (DstFuncDef *)(dst_unwrap_pointer(func->def->constants[(int32_t)oparg(2, 0xFFFF)]));
-            fn = dst_alloc(DST_MEMORY_FUNCTION, sizeof(DstFunction));
+            fn = dst_gcalloc(DST_MEMORY_FUNCTION, sizeof(DstFunction));
             fn->envs = malloc(sizeof(DstFuncEnv *) * fd->environments_length);
             if (NULL == fn->envs) {
                 DST_OUT_OF_MEMORY;
             }
             if (fd->flags & DST_FUNCDEF_FLAG_NEEDSENV) {
                 /* Delayed capture of current stack frame */
-                DstFuncEnv *env = dst_alloc(DST_MEMORY_FUNCENV, sizeof(DstFuncEnv));
+                DstFuncEnv *env = dst_gcalloc(DST_MEMORY_FUNCENV, sizeof(DstFuncEnv));
                 env->offset = dst_vm_fiber->frame;
                 env->as.fiber = dst_vm_fiber;
                 env->length = func->def->slotcount;
@@ -498,7 +499,7 @@ int dst_continue() {
 
         case DOP_TAILCALL:
         {
-            DstValue callee = stack[oparg(2, 0xFFFF)];
+            DstValue callee = stack[oparg(1, 0xFFFFFF)];
             if (dst_checktype(callee, DST_FUNCTION)) {
                 func = dst_unwrap_function(callee);
                 dst_fiber_funcframe_tail(dst_vm_fiber, func);
@@ -684,6 +685,10 @@ int dst_init() {
     dst_symcache_init();
     /* Set thread */
     dst_vm_fiber = NULL;
+    /* Initialize gc roots */
+    dst_vm_roots = NULL;
+    dst_vm_root_count = 0;
+    dst_vm_root_capacity = 0;
     return 0;
 }
 
@@ -692,4 +697,8 @@ void dst_deinit() {
     dst_clear_memory();
     dst_vm_fiber = NULL;
     dst_symcache_deinit();
+    free(dst_vm_roots);
+    dst_vm_roots = NULL;
+    dst_vm_root_count = 0;
+    dst_vm_root_capacity = 0;
 }

dsttest/basic.dst

@@ -1,4 +1,4 @@
 # A really basic for to compile. for testing the compiler. Will extend
 # as compiler is extended.
-123
+(∑ 1 2 3)
 

gsttests/basic.dst

@@ -1,3 +1,23 @@
+# Copyright (c) 2017 Calvin Rose
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
 (print "\nRunning basic tests...\n")
 
 (var num-tests-passed 0)

include/dst/dst.h

@@ -113,6 +113,9 @@
 #define DST_INTEGER_MIN INT32_MIN
 #define DST_INTEGER_MAX INT32_MAX
 
+/* Helper for debugging */
+#define dst_trace(x) dst_puts(dst_formatc("DST TRACE %s, %d: %v\n", __FILE__, __LINE__, x))
+
 /* Prevent some recursive functions from recursing too deeply
  * ands crashing (the parser). Instead, error out. */
 #define DST_RECURSION_GUARD 1000
@@ -133,14 +136,23 @@ typedef struct DstTable DstTable;
 typedef struct DstFiber DstFiber;
 
 /* Other structs */
+typedef struct DstReg DstReg;
 typedef struct DstUserdataHeader DstUserdataHeader;
 typedef struct DstFuncDef DstFuncDef;
 typedef struct DstFuncEnv DstFuncEnv;
 typedef struct DstStackFrame DstStackFrame;
-typedef union DstValueUnion DstValueUnion;
 typedef struct DstUserType DstUserType;
 typedef int (*DstCFunction)(DstValue *argv, int32_t argn);
 
+typedef enum DstAssembleStatus DstAssembleStatus;
+typedef struct DstAssembleResult DstAssembleResult;
+typedef struct DstAssembleOptions DstAssembleOptions;
+typedef enum DstCompileStatus DstCompileStatus;
+typedef struct DstCompileOptions DstCompileOptions;
+typedef struct DstCompileResults DstCompileResults;
+typedef struct DstParseResult DstParseResult;
+typedef enum DstParseStatus DstParseStatus;
+
 /* Names of all of the types */
 extern const char *dst_type_names[16];
 
@@ -372,6 +384,12 @@ DstValue dst_wrap_pointer(void *x);
 /* End of tagged union implementation */
 #endif
 
+/* Used for creating libraries of cfunctions. */
+struct DstReg {
+    const char *name;
+    DstCFunction function;
+};
+
 /* A lightweight green thread in dst. Does not correspond to
  * operating system threads. */
 struct DstFiber {
@@ -452,7 +470,7 @@ struct DstFuncEnv {
         DstValue *values;
     } as;
     int32_t length; /* Size of environment */
-    int32_t offset; /* Stack offset when values still on stack. If offset is 0, then
+    int32_t offset; /* Stack offset when values still on stack. If offset is <= 0, then
         environment is no longer on the stack. */
 };
 
@@ -477,12 +495,64 @@ struct DstUserdataHeader {
     size_t size;
 };
 
-/* The VM state. Rather than a struct that is passed
+/* Assemble structs */
- * around, the vm state is global for simplicity and performance. */
+enum DstAssembleStatus {
+    DST_ASSEMBLE_OK,
+    DST_ASSEMBLE_ERROR
+};
 
-/* TODO - somehow wrap these for windows dynamic linking. Either that,
+struct DstAssembleOptions {
- * or force static linking. see
+    const DstValue *sourcemap;
- * https://stackoverflow.com/questions/19373061/what-happens-to-global-and-static-variables-in-a-shared-library-when-it-is-dynam */
+    DstValue source;
+    uint32_t flags;
+};
+
+struct DstAssembleResult {
+    DstFuncDef *funcdef;
+    const uint8_t *error;
+    int32_t error_start;
+    int32_t error_end;
+    DstAssembleStatus status;
+};
+
+/* Compile structs */
+enum DstCompileStatus {
+    DST_COMPILE_OK,
+    DST_COMPILE_ERROR
+};
+
+struct DstCompileResults {
+    DstCompileStatus status;
+    DstFuncDef *funcdef;
+    const uint8_t *error;
+    int32_t error_start;
+    int32_t error_end;
+};
+
+struct DstCompileOptions {
+    uint32_t flags;
+    const DstValue *sourcemap;
+    DstValue source;
+    DstValue env;
+};
+
+/* Parse structs */
+enum DstParseStatus {
+    DST_PARSE_OK,
+    DST_PARSE_ERROR,
+    DST_PARSE_UNEXPECTED_EOS
+};
+
+struct DstParseResult {
+    DstValue value;
+    const uint8_t *error;
+    const DstValue *map;
+    int32_t bytes_read;
+    DstParseStatus status;
+};
+
+/* The VM state. Rather than a struct that is passed
+ * around, the vm state is global for simplicity. */
 
 /* Garbage collection */
 extern void *dst_vm_blocks;
@@ -498,6 +568,11 @@ extern uint32_t dst_vm_cache_deleted;
 /* Syscall table */
 extern const DstCFunction dst_vm_syscalls[256];
 
+/* GC roots */
+extern DstValue *dst_vm_roots;
+extern uint32_t dst_vm_root_count;
+extern uint32_t dst_vm_root_capacity;
+
 /* GC roots - TODO consider a top level fiber pool (per thread?) */
 extern DstFiber *dst_vm_fiber;
 
@@ -585,6 +660,7 @@ DstValue dst_table_remove(DstTable *t, DstValue key);
 void dst_table_put(DstTable *t, DstValue key, DstValue value);
 DstValue dst_table_next(DstTable *t, DstValue key);
 const DstValue *dst_table_to_struct(DstTable *t);
+void dst_table_merge(DstTable *t,  DstValue other);
 
 /* Fiber */
 DstFiber *dst_fiber(int32_t capacity);
@@ -607,24 +683,6 @@ void dst_fiber_popframe(DstFiber *fiber);
 void dst_function_detach(DstFunction *func);
 
 /* Assembly */
-typedef enum {
-    DST_ASSEMBLE_OK,
-    DST_ASSEMBLE_ERROR
-} DstAssembleStatus;
-typedef struct DstAssembleResult DstAssembleResult;
-typedef struct DstAssembleOptions DstAssembleOptions;
-struct DstAssembleResult {
-    DstFuncDef *funcdef;
-    const uint8_t *error;
-    int32_t error_start;
-    int32_t error_end;
-    DstAssembleStatus status;
-};
-struct DstAssembleOptions {
-    const DstValue *sourcemap;
-    DstValue source;
-    uint32_t flags;
-};
 DstAssembleResult dst_asm(DstAssembleOptions opts);
 DstFunction *dst_asm_func(DstAssembleResult result);
 DstValue dst_disasm(DstFuncDef *def);
@@ -655,21 +713,9 @@ void dst_setindex(DstValue ds, DstValue value, int32_t index);
 extern const char dst_base64[65];
 int32_t dst_array_calchash(const DstValue *array, int32_t len);
 int32_t dst_string_calchash(const uint8_t *str, int32_t len);
+DstValue dst_loadreg(DstReg *regs, size_t count);
 
 /* Parsing */
-typedef enum {
-    DST_PARSE_OK,
-    DST_PARSE_ERROR,
-    DST_PARSE_UNEXPECTED_EOS
-} DstParseStatus;
-typedef struct DstParseResult DstParseResult;
-struct DstParseResult {
-    DstValue value;
-    const uint8_t *error;
-    const DstValue *map;
-    int32_t bytes_read;
-    DstParseStatus status;
-};
 DstParseResult dst_parse(const uint8_t *src, int32_t len);
 DstParseResult dst_parsec(const char *src);
 const DstValue *dst_parse_submap_index(const DstValue *map, int32_t index);
@@ -684,25 +730,6 @@ int dst_run(DstValue callee);
 DstValue dst_transfer(DstFiber *fiber, DstValue x);
 
 /* Compile */
-typedef enum DstCompileStatus {
-    DST_COMPILE_OK,
-    DST_COMPILE_ERROR
-} DstCompileStatus;
-
-/* Results of compilation */
-typedef struct DstCompileResults {
-    DstCompileStatus status;
-    DstFuncDef *funcdef;
-    const uint8_t *error;
-    int32_t error_start;
-    int32_t error_end;
-} DstCompileResults;
-
-typedef struct DstCompileOptions {
-    uint32_t flags;
-    const DstValue *sourcemap;
-    DstValue source;
-} DstCompileOptions;
 
 /* Compile source code into FuncDef. */
 DstCompileResults dst_compile(DstCompileOptions opts);
@@ -710,98 +737,12 @@ DstFunction *dst_compile_func(DstCompileResults results);
 
 /* GC */
 
-/* The metadata header associated with an allocated block of memory */
-#define dst_gc_header(mem) ((DstGCMemoryHeader *)(mem) - 1)
-
-#define DST_MEM_TYPEBITS 0xFF
-#define DST_MEM_REACHABLE 0x100
-#define DST_MEM_DISABLED 0x200
-
-#define dst_gc_settype(m, t) ((dst_gc_header(m)->flags |= (0xFF & (t))))
-#define dst_gc_type(m) (dst_gc_header(m)->flags & 0xFF)
-
-#define dst_gc_mark(m) (dst_gc_header(m)->flags |= DST_MEM_REACHABLE)
-#define dst_gc_unmark(m) (dst_gc_header(m)->flags &= ~DST_MEM_COLOR)
-#define dst_gc_reachable(m) (dst_gc_header(m)->flags & DST_MEM_REACHABLE)
-
-
-/* Memory header struct. Node of a linked list of memory blocks. */
-typedef struct DstGCMemoryHeader DstGCMemoryHeader;
-struct DstGCMemoryHeader {
-    DstGCMemoryHeader *next;
-    uint32_t flags;
-};
-
-/* Memory types for the GC. Different from DstType to include funcenv and funcdef. */
-typedef enum DstMemoryType DstMemoryType;
-enum DstMemoryType {
-    DST_MEMORY_NONE,
-    DST_MEMORY_STRING,
-    DST_MEMORY_SYMBOL,
-    DST_MEMORY_ARRAY,
-    DST_MEMORY_TUPLE,
-    DST_MEMORY_TABLE,
-    DST_MEMORY_STRUCT,
-    DST_MEMORY_FIBER,
-    DST_MEMORY_BUFFER,
-    DST_MEMORY_FUNCTION,
-    DST_MEMORY_USERDATA,
-    DST_MEMORY_FUNCENV,
-    DST_MEMORY_FUNCDEF
-};
-
-/* Preventn GC from freeing some memory. */
-#define dst_disablegc(m) dst_gc_header(m)->flags |= DST_MEM_DISABLED
-
-/* To allocate collectable memory, one must calk dst_alloc, initialize the memory,
- * and then call when dst_enablegc when it is initailize and reachable by the gc (on the DST stack) */
-void *dst_alloc(DstMemoryType type, size_t size);
-#define dst_enablegc(m) dst_gc_header(m)->flags &= ~DST_MEM_DISABLED
-
-/* When doing C interop, it is often needed to disable GC on a value.  This is
- * needed when a garbage collection could occur in the middle of a c function.
- * This could happen, for example, if one calls back into dst inside of a c
- * function. The pin and unpin functions toggle garbage collection on a value
- * when needed. Note that no dst functions will call gc when you don't want it
- * to. GC only happens automatically in the interpreter loop. Pinning values
- * wil NOT recursively pin sub values. 
- *
- * Be careful whennig bypassing garbage collection like this. It can easily
- * lead to memory leaks or other undesirable side effects. */
-void dst_pin(DstValue x);
-void dst_unpin(DstValue x);
-
-/* Specific types can also be pinned and unpinned as well. */
-#define dst_pin_table dst_disablegc
-#define dst_pin_array dst_disablegc
-#define dst_pin_buffer dst_disablegc
-#define dst_pin_function dst_disablegc
-#define dst_pin_fiber dst_disablegc
-#define dst_pin_string(s) dst_disablegc(dst_string_raw(s))
-#define dst_pin_symbol(s) dst_disablegc(dst_string_raw(s))
-#define dst_pin_tuple(s) dst_disablegc(dst_tuple_raw(s))
-#define dst_pin_struct(s) dst_disablegc(dst_struct_raw(s))
-#define dst_pin_userdata(s) dst_disablegc(dst_userdata_header(s))
-
-#define dst_unpin_table dst_enablegc
-#define dst_unpin_array dst_enablegc
-#define dst_unpin_buffer dst_enablegc
-#define dst_unpin_function dst_enablegc
-#define dst_unpin_fiber dst_enablegc
-#define dst_unpin_string(s) dst_enablegc(dst_string_raw(s))
-#define dst_unpin_symbol(s) dst_enablegc(dst_string_raw(s))
-#define dst_unpin_tuple(s) dst_enablegc(dst_tuple_raw(s))
-#define dst_unpin_struct(s) dst_enablegc(dst_struct_raw(s))
-#define dst_unpin_userdata(s) dst_enablegc(dst_userdata_header(s))
-
 void dst_mark(DstValue x);
 void dst_sweep();
-
-/* Collect some memory */
 void dst_collect();
-
-/* Clear all memory. */
 void dst_clear_memory();
+void dst_gcroot(DstValue root);
+int dst_gcunroot(DstValue root);
 
 /* Run garbage collection if needed */
 #define dst_maybe_collect() do {\

unittests/compile_test.c

@@ -1,6 +1,15 @@
 #include "unit.h"
 #include <dst/dst.h>
 
+int testprint(DstValue *argv, int32_t argn) {
+    printf("hello!\n");
+    return 0;
+}
+
+DstReg testreg[] = {
+    {"print", testprint}
+};
+
 int main() {
     DstParseResult pres;
     DstCompileOptions opts;
@@ -33,6 +42,8 @@ int main() {
     opts.flags = 0;
     opts.source = pres.value;
     opts.sourcemap = pres.map;
+    opts.env = dst_loadreg(testreg, sizeof(testreg)/sizeof(DstReg));
+    dst_puts(dst_formatc("initial compile env: %v\n", opts.env));
 
     cres = dst_compile(opts);
     if (cres.status == DST_COMPILE_ERROR) {

unittests/fiber_test.c

@@ -1,10 +1,11 @@
 #include "unit.h"
+#include "../core/gc.h"
 #include <dst/dst.h>
 #include <stdio.h>
 
 /* Create dud funcdef and function */
 static DstFunction *dud_func(uint32_t slotcount, uint32_t arity, int varargs) {
-    DstFuncDef *def = dst_alloc(DST_MEMORY_FUNCDEF, sizeof(DstFuncDef));
+    DstFuncDef *def = dst_gcalloc(DST_MEMORY_FUNCDEF, sizeof(DstFuncDef));
     def->environments_length = 0;
     def->constants_length = 0;
     def->bytecode_length = 0;
@@ -14,7 +15,7 @@ static DstFunction *dud_func(uint32_t slotcount, uint32_t arity, int varargs) {
     def->flags = varargs ? DST_FUNCDEF_FLAG_VARARG : 0;
     def->arity = arity;
     def->slotcount = slotcount;
-    DstFunction *f = dst_alloc(DST_MEMORY_FUNCTION, sizeof(DstFunction));
+    DstFunction *f = dst_gcalloc(DST_MEMORY_FUNCTION, sizeof(DstFunction));
     f->envs = NULL;
     f->def = def;
     return f;
@@ -84,5 +85,7 @@ int main() {
     dst_fiber_funcframe_tail(fiber1, dud_func(20, 0, 0));
     debug_print_fiber(fiber1, 1);
 
+    //dst_deinit();
+
     return 0;
 }