WIP

Makefile

@@ -13,8 +13,8 @@ PREFIX=/usr/local
 DST_TARGET=dst
 DST_XXD=xxd
 # Use gdb. On mac use lldb
-DEBUGGER=gdb
-DST_INTERNAL_HEADERS=$(addprefix core/, internal.h bootstrap.h)
+DEBUGGER=lldb
+DST_INTERNAL_HEADERS=$(addprefix core/, internal.h bootstrap.h cache.h)
 DST_HEADERS=$(addprefix include/dst/, dst.h)
 
 #############################
@@ -38,9 +38,9 @@ $(DST_XXD): libs/xxd.c
 ##### The core vm and runtime #####
 ###################################
 DST_CORE_SOURCES=$(addprefix core/,\
-				 util.c wrap.c\
-				 value.c vm.c ds.c gc.c thread.c serialize.c\
-				 string.c bootstrap_parse.c client.c)
+				 util.c wrap.c buffer.c array.c table.c userdata.c func.c\
+				 value.c vm.c ds.c gc.c thread.c serialize.c tuple.c\
+				 string.c bootstrap_parse.c client.c cache.c struct.c)
 DST_CORE_OBJECTS=$(patsubst %.c,%.o,$(DST_CORE_SOURCES))
 
 $(DST_TARGET): $(DST_CORE_OBJECTS) $(DST_LANG_HEADERS)

core/array.c

@@ -0,0 +1,89 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#include "internal.h"
+#include <dst/dst.h>
+
+/* Iniializes an array. Assumes the pointer to the array is already on the stack. */
+DstArray *dst_array(Dst *vm, uint32_t capacity) {
+    DstArray *array = dst_alloc(vm, DST_MEMORY_ARRAY, sizeof(DstArray));
+    DstValue *data = (DstValue *) malloc(sizeof(DstValue) * capacity);
+    if (NULL == array || NULL == data) {
+        DST_OUT_OF_MEMORY;
+    }
+    array->count = 0;
+    array->capacity = capacity;
+    array->data = data;
+    return array;
+}
+
+/* Ensure the array has enough capacity for elements */
+void dst_array_ensure(Dst *vm, DstArray *array, uint32_t capacity) {
+    DstValue *newData;
+    DstValue *old = array->data;
+    if (capacity <= array->capacity) return;
+    newData = realloc(old, capacity * sizeof(DstValue));
+    if (NULL == newData) {
+        DST_OUT_OF_MEMORY;
+    }
+    array->data = newData;
+    array->capacity = capacity;
+}
+
+/* Set the count of an array. Extend with nil if needed. */
+void dst_array_setcount(Dst *vm, DstArray *array, uint32_t count) {
+    if (count > array->count) {
+        uint32_t i;
+        dst_array_ensure(vm, array, count + 1);
+        for (i = array->count; i < count; ++i)
+            array->data[i].type = DST_NIL;
+    }
+    array->count = count;
+}
+
+/* Push a value to the top of the array */
+void dst_array_push(Dst *vm, DstArray *array, DstValue x) {
+    uint32_t newcount = array->count + 1;
+    if (newcount >= array->capacity) }
+        dst_array_ensure(vm, array, newcount * 2);
+    }
+    array->data[array->count] = x;
+    array->count = newcount;
+}
+
+/* Pop a value from the top of the array */
+DstValue dst_array_pop(DstArray *array) {
+    if (array->count) {
+        return array->data[--array->count];
+    } else {
+        return dst_wrap_nil();
+    }
+}
+
+/* Look at the last value in the array */
+DstValue dst_array_peek(DstArray *array) {
+    if (array->count) {
+        return array->data[array->count - 1];
+    } else {
+        return dst_wrap_nil();
+    }
+}

core/buffer.c

@@ -0,0 +1,132 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#include "internal.h"
+#include <dst/dst.h>
+
+/* Initialize a buffer */
+DstBuffer *dst_buffer(Dst *vm, uint32_t capacity) {
+    DstBuffer *buffer = dst_alloc(vm, DST_MEMORY_BUFFER, sizeof(DstBuffer));
+    uint8_t *data = malloc(sizeof(uint8_t) * capacity);
+    if (NULL == data) {
+        DST_OUT_OF_MEMORY;
+    }
+    buffer->count = 0;
+    buffer->capacity = capacity;
+    return buffer;
+}
+
+/* Ensure that the buffer has enough internal capacity */
+void dst_buffer_ensure(Dst *vm, DstBuffer *buffer, uint32_t capacity) {
+    uint8_t *newData;
+    uint8_t *old = buffer->data;
+    if (capacity <= buffer->capacity) return;
+    newData = realloc(old, capacity * sizeof(uint8_t));
+    if (NULL == newData) {
+        DST_OUT_OF_MEMORY;
+    }
+    buffer->data = newData;
+    buffer->capacity = capacity;
+}
+
+/* Adds capacity for enough extra bytes to the buffer. Ensures that the
+ * next n bytes pushed to the buffer will not cause a reallocation */
+void dst_buffer_extra(Dst *vm, DstBuffer *buffer, uint32_t n) {
+    uint32_t newCount = buffer->count + n;
+    if (newCount > buffer->capacity) {
+        uint32_t newCapacity = newCount * 2;
+        uint8_t *newData = realloc(buffer->data, newCapacity * sizeof(uint8_t));
+        if (NULL == newData) {
+            DST_OUT_OF_MEMORY;
+        }
+        buffer->data = newData;
+        buffer->capacity = newCapacity;
+    }
+}
+
+/* Push multiple bytes into the buffer */
+void dst_buffer_push_bytes(Dst *vm, DstBuffer *buffer, const uint8_t *string, uint32_t length) {
+    uint32_t newSize = buffer->count + length;
+    if (newSize > buffer->capacity) {
+        dst_buffer_ensure(vm, buffer, 2 * newSize);
+    }
+    dst_memcpy(buffer->data + buffer->count, string, length);
+    buffer->count = newSize;
+}
+
+/* Push a cstring to buffer */
+void dst_buffer_push_cstring(Dst *vm, DstBuffer *buffer, const char *cstring) {
+    uint32_t len = 0;
+    while (cstring[len]) ++len;
+    dst_buffer_push_bytes(vm, buffer, (const uint8_t *) cstring, len);
+}
+
+/* Push a single byte to the buffer */
+void dst_buffer_push_u8(Dst *vm, DstBuffer *buffer, uint8_t byte) {
+    uint32_t newSize = buffer->count + 1;
+    if (newSize > buffer->capacity) {
+        dst_buffer_ensure(vm, buffer, 2 * newSize);
+    }
+    buffer->data[buffer->count] = byte;
+    buffer->count = newSize;
+}
+
+/* Push a 16 bit unsigned integer to the buffer */
+void dst_buffer_push_u16(Dst *vm, DstBuffer *buffer, uint16_t x) {
+    uint32_t newSize = buffer->count + 2;
+    if (newSize > buffer->capacity) {
+        dst_buffer_ensure(vm, buffer, 2 * newSize);
+    }
+    buffer->data[buffer->count] = x & 0xFF;
+    buffer->data[buffer->count + 1] = (x >> 8) & 0xFF;
+    buffer->count = newSize;
+}
+
+/* Push a 32 bit unsigned integer to the buffer */
+void dst_buffer_push_u32(Dst *vm, DstBuffer *buffer, uint32_t x) {
+    uint32_t newSize = buffer->count + 4;
+    if (newSize > buffer->capacity) {
+        dst_buffer_ensure(vm, buffer, 2 * newSize);
+    }
+    buffer->data[buffer->count] = x & 0xFF;
+    buffer->data[buffer->count + 1] = (x >> 8) & 0xFF;
+    buffer->data[buffer->count + 2] = (x >> 16) & 0xFF;
+    buffer->data[buffer->count + 3] = (x >> 24) & 0xFF;
+    buffer->count = newSize;
+}
+
+/* Push a 64 bit unsigned integer to the buffer */
+void dst_buffer_push_u64(Dst *vm, DstBuffer *buffer, uint64_t x) {
+    uint32_t newSize = buffer->count + 8;
+    if (newSize > buffer->capacity) {
+        dst_buffer_ensure(vm, buffer, 2 * newSize);
+    }
+    buffer->data[buffer->count] = x & 0xFF;
+    buffer->data[buffer->count + 1] = (x >> 8) & 0xFF;
+    buffer->data[buffer->count + 2] = (x >> 16) & 0xFF;
+    buffer->data[buffer->count + 3] = (x >> 24) & 0xFF;
+    buffer->data[buffer->count + 4] = (x >> 32) & 0xFF;
+    buffer->data[buffer->count + 5] = (x >> 40) & 0xFF;
+    buffer->data[buffer->count + 6] = (x >> 48) & 0xFF;
+    buffer->data[buffer->count + 7] = (x >> 56) & 0xFF;
+    buffer->count = newSize;
+}

core/cache.c

@@ -0,0 +1,240 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#include "internal.h"
+#include "cache.h"
+
+/* All immutable values are cached in a global hash table. When an immutable
+ * value is created, this hashtable is checked to see if the value exists. If it
+ * does, return the cached copy instead. This trades creation time and memory for
+ * fast equality, which is especially useful for symbols and strings. This may not
+ * be useful for structs and tuples, in which case it may be removed. However, in cases
+ * where ther are many copies of the same tuple in the program, this approach may
+ * save memory. Values are removed from the cache when they are garbage collected.
+ */
+
+/* Check if two not necesarrily finalized immutable values
+ * are equal. Does caching logic */
+static int dst_cache_equal(DstValue x, DstValue y) {
+    uint32_t i, len;
+    if (x.type != y.type) return 0;
+    switch (x.type) {
+    /* Don't bother implementing equality checks for all types. We only care
+     * about immutable data structures */
+    default:
+        return 0;
+    case DST_STRING:
+        if (dst_string_hash(x.as.string) != dst_string_hash(y.as.string)) return 0;
+        if (dst_string_length(x.as.string) != dst_string_length(y.as.string)) return 0;
+        len = dst_string_length(x.as.string);
+        for (i = 0; i < len; ++i)
+            if (x.as.string[i] != y.as.string[i])
+                return 0;
+        return 1;
+    case DST_STRUCT:
+        if (dst_struct_hash(x.as.st) != dst_struct_hash(y.as.st)) return 0;
+        if (dst_struct_length(x.as.st) != dst_struct_length(y.as.st)) return 0;
+        len = dst_struct_capacity(x.as.st);
+        for (i = 0; i < len; ++i)
+            if (!dst_value_equals(x.as.st[i], y.as.st[i]))
+                return 0;
+        return 1;
+    case DST_TUPLE:
+        if (dst_tuple_hash(x.as.tuple) != dst_tuple_hash(y.as.tuple)) return 0;
+        if (dst_tuple_length(x.as.tuple) != dst_tuple_length(y.as.tuple)) return 0;
+        len = dst_tuple_length(x.as.tuple);
+        for (i = 0; i < len; ++i)
+            if (!dst_value_equals(x.as.tuple[i], y.as.tuple[i]))
+                return 0;
+        return 1;
+    }
+}
+
+/* Check if a value x is equal to a string. Special version of
+ * dst_cache_equal */
+static int dst_cache_strequal(DstValue x, const uint8_t *str, uint32_t len, uint32_t hash) {
+    uint32_t i;
+    if (x.type != DST_STRING) return 0;
+    if (dst_string_hash(x.as.string) != hash) return 0;
+    if (dst_string_length(x.as.string) != len) return 0;
+    for (i = 0; i < len; ++i)
+        if (x.as.string[i] != str[i])
+            return 0;
+    return 1;
+}
+
+/* Find an item in the cache and return its location.
+ * If the item is not found, return the location
+ * where one would put it. */
+static DstValue *dst_cache_find(Dst *vm, DstValue key, int *success) {
+    uint32_t bounds[4];
+    uint32_t i, j, index;
+    uint32_t hash = dst_value_hash(key);
+    DstValue *firstEmpty = NULL;
+    index = hash % vm->cache_capacity;
+    bounds[0] = index;
+    bounds[1] = vm->cache_capacity;
+    bounds[2] = 0;
+    bounds[3] = index;
+    for (j = 0; j < 4; j += 2)
+        for (i = bounds[j]; i < bounds[j+1]; ++i) {
+            DstValue test = vm->cache[i];
+            /* Check empty spots */
+            if (test.type == DST_NIL) {
+                if (firstEmpty == NULL)
+                    firstEmpty = vm->cache + i;
+                goto notfound;
+            }
+            /* Check for marked deleted - use booleans as deleted */
+            if (test.type == DST_BOOLEAN) {
+                if (firstEmpty == NULL)
+                    firstEmpty = vm->cache + i;
+                continue;
+            }
+            if (dst_cache_equal(test, key)) {
+                /* Replace first deleted */
+                *success = 1;
+                if (firstEmpty != NULL) {
+                    *firstEmpty = test;
+                    vm->cache[i].type = DST_BOOLEAN;
+                    return firstEmpty;
+                }
+                return vm->cache + i;
+            }
+        }
+    notfound:
+    *success = 0;
+    return firstEmpty;
+}
+
+/* Find an item in the cache and return its location.
+ * If the item is not found, return the location
+ * where one would put it. Special case of dst_cache_find */
+DstValue *dst_cache_strfind(Dst *vm,
+        const uint8_t *str,
+        uint32_t len,
+        uint32_t hash,
+        int *success) {
+    uint32_t bounds[4];
+    uint32_t i, j, index;
+    DstValue *firstEmpty = NULL;
+    index = hash % vm->cache_capacity;
+    bounds[0] = index;
+    bounds[1] = vm->cache_capacity;
+    bounds[2] = 0;
+    bounds[3] = index;
+    for (j = 0; j < 4; j += 2)
+        for (i = bounds[j]; i < bounds[j+1]; ++i) {
+            DstValue test = vm->cache[i];
+            /* Check empty spots */
+            if (test.type == DST_NIL) {
+                if (firstEmpty == NULL)
+                    firstEmpty = vm->cache + i;
+                goto notfound;
+            }
+            /* Check for marked deleted - use booleans as deleted */
+            if (test.type == DST_BOOLEAN) {
+                if (firstEmpty == NULL)
+                    firstEmpty = vm->cache + i;
+                continue;
+            }
+            if (dst_cache_strequal(test, str, len, hash)) {
+                /* Replace first deleted */
+                *success = 1;
+                if (firstEmpty != NULL) {
+                    *firstEmpty = test;
+                    vm->cache[i].type = DST_BOOLEAN;
+                    return firstEmpty;
+                }
+                return vm->cache + i;
+            }
+        }
+    notfound:
+    *success = 0;
+    return firstEmpty;
+}
+
+/* Resize the cache. */
+static void dst_cache_resize(Dst *vm, uint32_t newCapacity) {
+    uint32_t i, oldCapacity;
+    DstValue *oldCache = vm->cache;
+    DstValue *newCache = calloc(1, newCapacity * sizeof(DstValue));
+    if (newCache == NULL) {
+        DST_OUT_OF_MEMORY;
+    }
+    oldCapacity = vm->cache_capacity;
+    vm->cache = newCache;
+    vm->cache_capacity = newCapacity;
+    vm->cache_deleted = 0;
+    /* Add all of the old cache entries back */
+    for (i = 0; i < oldCapacity; ++i) {
+        int status;
+        DstValue *bucket;
+        DstValue x = oldCache[i];
+        if (x.type != DST_NIL && x.type != DST_BOOLEAN) {
+            bucket = dst_cache_find(vm, x, &status);
+            if (status || bucket == NULL) {
+                /* there was a problem with the algorithm. */
+                break;
+            }
+            *bucket = x;
+        }
+    }
+    /* Free the old cache */
+    free(oldCache);
+}
+
+/* Add a value to the cache given we know it is not
+ * already in the cache and we have a bucket. */
+DstValue dst_cache_add_bucket(Dst *vm, DstValue x, DstValue *bucket) {
+    if ((vm->cache_count + vm->cache_deleted) * 2 > vm->cache_capacity) {
+        int status;
+        dst_cache_resize(vm, vm->cache_count * 4);
+        bucket = dst_cache_find(vm, x, &status);
+    }
+    /* Add x to the cache */
+    vm->cache_count++;
+    *bucket = x;
+    return x;
+}
+
+/* Add a value to the cache */
+DstValue dst_cache_add(Dst *vm, DstValue x) {
+    int status = 0;
+    DstValue *bucket = dst_cache_find(vm, x, &status);
+    if (!status) {
+        return dst_cache_add_bucket(vm, x, bucket);
+    } else {
+        return *bucket;
+    }
+}
+
+/* Remove a value from the cache */
+void dst_cache_remove(Dst *vm, DstValue x) {
+    int status = 0;
+    DstValue *bucket = dst_cache_find(vm, x, &status);
+    if (status) {
+        vm->cache_count--;
+        vm->cache_deleted++;
+        bucket->type = DST_BOOLEAN;
+    }
+}

core/cache.h

@@ -20,19 +20,20 @@
 * IN THE SOFTWARE.
 */
 
-#ifndef DST_BOOTSTRAP_H_defined
-#define DST_BOOTSTRAP_H_defined
+#ifndef DST_CACHE_H_defined
+#define DST_CACHE_H_defined
 
-#define PARSE_OK 0
-#define PARSE_ERROR 1
-#define PARSE_UNEXPECTED_EOS 2
+#include <dst/dst.h>
+#include "internal.h"
 
-int dst_parseb(Dst *vm, uint32_t dest, const uint8_t *src, const uint8_t **newsrc, uint32_t len);
+DstValue dst_cache_add(Dst *vm, DstValue x);
+DstValue *dst_cache_strfind(Dst *vm,
+        const uint8_t *str,
+        uint32_t len,
+        uint32_t hash,
+        int *success);
+DstValue dst_cache_add_bucket(Dst *vm, DstValue x, DstValue *bucket);
 
-/* Parse a c string */
-int dst_parsec(Dst *vm, uint32_t dest, const char *src);
+void dst_cache_remove(Dst *vm, DstValue x);
 
-/* Parse a DST char seq (Buffer, String, Symbol) */
-int dst_parse(Dst *vm, uint32_t dest, uint32_t src);
-
-#endif /* DST_BOOTSTRAP_H_defined */
+#endif

core/client.c

@@ -25,57 +25,40 @@
 #include <dst/dst.h>
 #include "bootstrap.h"
 
-
-/* Simple read line functionality */
-static char *dst_getline() {
-    char *line = malloc(100);
-    char *linep = line;
-    size_t lenmax = 100;
-    size_t len = lenmax;
-    int c;
-    if (line == NULL)
-        return NULL;
-    for (;;) {
-        c = fgetc(stdin);
-        if (c == EOF)
-            break;
-        if (--len == 0) {
-            len = lenmax;
-            char *linen = realloc(linep, lenmax *= 2);
-            if (linen == NULL) {
-                free(linep);
-                return NULL;
+void teststr(Dst *vm, const char *src) {
+    uint32_t len = 0;
+    const uint8_t *bytes;
+    const char *ns = NULL;
+    int status = dst_parsec(vm, src, &ns);
+    if (status) {
+        printf("Parse failed: ");
+        bytes = dst_bytes(vm, -1, &len);
+        for (uint32_t i = 0; i < len; i++)
+            putc(bytes[i], stdout);
+        putc('\n', stdout);
+        printf("%s\n", src);
+        for (const char *scan = src + 1; scan < ns; ++scan)
+            putc(' ', stdout);
+        putc('^', stdout);
+        putc('\n', stdout);
+        return;
     }
-            line = linen + (line - linep);
-            linep = linen;
-        }
-        if ((*line++ = c) == '\n')
-            break;
-    }
-    *line = '\0';
-    return linep;
+    dst_description(vm);
+    bytes = dst_bytes(vm, -1, &len);
+    for (uint32_t i = 0; i < len; i++)
+        putc(bytes[i], stdout);
+    putc('\n', stdout);
 }
 
-int main(int argc, const char **argv) {
+int main() {
 
     Dst *vm = dst_init();
-    for (;;) {
-        char *line = dst_getline();
-        if (line) {
-            uint32_t len = 0;
-            int status = dst_parsec(vm, 0, line);
-            if (status == PARSE_OK) {
-                dst_description(vm, 0, 0);
-            }
-            const uint8_t *b = dst_bytes(vm, 0, &len);
-            for (uint32_t i = 0; i < len; ++i) {
-                putc(b[i]);
-            }
-            putc('\n');
-        } else {
-            break;
-        }
-    }
+
+    teststr(vm, "[+ 1 2 3 \"porkpie\" ]");
+    teststr(vm, "(+ 1 2 \t asdajs 1035.89 3)");
+    teststr(vm, "[+ 1 2 :bokr]");
+    teststr(vm, "{+ 1 2 3}");
+
     dst_deinit(vm);
 
     return 0;

core/compile.c

@@ -373,13 +373,13 @@ static uint16_t compiler_add_literal(DstCompiler *c, DstScope *scope, DstValue x
     uint16_t literalIndex = 0;
     if (checkDup.type != DST_NIL) {
         /* An equal literal is already registered in the current scope */
-        return (uint16_t) checkDup.data.integer;
+        return (uint16_t) checkDup.as.integer;
     } else {
         /* Add our literal for tracking */
         DstValue valIndex;
         valIndex.type = DST_INTEGER;
         literalIndex = scope->literalsArray->count;
-        valIndex.data.integer = literalIndex;
+        valIndex.as.integer = literalIndex;
         dst_table_put(c->vm, scope->literals, x, valIndex);
         dst_array_push(c->vm, scope->literalsArray, x);
     }
@@ -394,7 +394,7 @@ static uint16_t compiler_declare_symbol(DstCompiler *c, DstScope *scope, DstValu
         c_error(c, "expected symbol");
     target = compiler_get_local(c, scope);
     x.type = DST_INTEGER;
-    x.data.integer = target + (flags << 16);
+    x.as.integer = target + (flags << 16);
     dst_table_put(c->vm, scope->locals, sym, x);
     return target;
 }
@@ -409,8 +409,8 @@ static int symbol_resolve(DstCompiler *c, DstValue x, uint16_t *level, uint16_t
         check = dst_table_get(scope->locals, x);
         if (check.type != DST_NIL) {
             *level = currentLevel - scope->level;
-            *index = (uint16_t) (check.data.integer & 0xFFFF);
-            if (flags) *flags = check.data.integer >> 16;
+            *index = (uint16_t) (check.as.integer & 0xFFFF);
+            if (flags) *flags = check.as.integer >> 16;
             return 1;
         }
         scope = scope->parent;
@@ -422,7 +422,7 @@ static int symbol_resolve(DstCompiler *c, DstValue x, uint16_t *level, uint16_t
         DstTable *metas = dst_env_meta(c->vm, c->env);
         DstValue maybeMeta = dst_table_get(metas, x);
         if (maybeMeta.type == DST_TABLE) {
-            DstValue isMutable = dst_table_get(maybeMeta.data.table, dst_string_cv(c->vm, "mutable"));
+            DstValue isMutable = dst_table_get(maybeMeta.as.table, dst_string_cv(c->vm, "mutable"));
             if (dst_truthy(isMutable)) {
                 if (flags) *flags = DST_LOCAL_FLAG_MUTABLE;
                 *out = check;
@@ -464,25 +464,25 @@ static Slot compile_nonref_type(DstCompiler *c, FormOptions opts, DstValue x) {
         dst_buffer_push_u16(c->vm, buffer, DST_OP_NIL);
         dst_buffer_push_u16(c->vm, buffer, ret.index);
     } else if (x.type == DST_BOOLEAN) {
-        dst_buffer_push_u16(c->vm, buffer, x.data.boolean ? DST_OP_TRU : DST_OP_FLS);
+        dst_buffer_push_u16(c->vm, buffer, x.as.boolean ? DST_OP_TRU : DST_OP_FLS);
         dst_buffer_push_u16(c->vm, buffer, ret.index);
     } else if (x.type == DST_REAL) {
         dst_buffer_push_u16(c->vm, buffer, DST_OP_F64);
         dst_buffer_push_u16(c->vm, buffer, ret.index);
-        dst_buffer_push_real(c->vm, buffer, x.data.real);
+        dst_buffer_push_real(c->vm, buffer, x.as.real);
     } else if (x.type == DST_INTEGER) {
-        if (x.data.integer <= 32767 && x.data.integer >= -32768) {
+        if (x.as.integer <= 32767 && x.as.integer >= -32768) {
             dst_buffer_push_u16(c->vm, buffer, DST_OP_I16);
             dst_buffer_push_u16(c->vm, buffer, ret.index);
-            dst_buffer_push_i16(c->vm, buffer, x.data.integer);
-        } else if (x.data.integer <= 2147483647 && x.data.integer >= -2147483648) {
+            dst_buffer_push_i16(c->vm, buffer, x.as.integer);
+        } else if (x.as.integer <= 2147483647 && x.as.integer >= -2147483648) {
             dst_buffer_push_u16(c->vm, buffer, DST_OP_I32);
             dst_buffer_push_u16(c->vm, buffer, ret.index);
-            dst_buffer_push_i32(c->vm, buffer, x.data.integer);
+            dst_buffer_push_i32(c->vm, buffer, x.as.integer);
         } else {
             dst_buffer_push_u16(c->vm, buffer, DST_OP_I64);
             dst_buffer_push_u16(c->vm, buffer, ret.index);
-            dst_buffer_push_i64(c->vm, buffer, x.data.integer);
+            dst_buffer_push_i64(c->vm, buffer, x.as.integer);
         }
     } else {
         c_error(c, "expected boolean, nil, or number type");
@@ -805,14 +805,14 @@ static Slot compile_function(DstCompiler *c, FormOptions opts, const DstValue *f
     /* Define the function parameters */
     if (form[current].type != DST_ARRAY)
         c_error(c, "expected function arguments array");
-    params = form[current++].data.array;
+    params = form[current++].as.array;
     arity = params->count;
     for (i = 0; i < params->count; ++i) {
         DstValue param = params->data[i];
         if (param.type != DST_SYMBOL)
             c_error(c, "function parameters should be symbols");
         /* Check for varargs */
-        if (equal_cstr(param.data.string, "&")) {
+        if (equal_cstr(param.as.string, "&")) {
             if (i != params->count - 1) {
                 c_error(c, "& is reserved for vararg argument in function");
             }
@@ -838,7 +838,7 @@ static Slot compile_function(DstCompiler *c, FormOptions opts, const DstValue *f
         DstFuncDef *def = compiler_gen_funcdef(c, buffer->count - sizeBefore, arity, varargs);
         /* Add this FuncDef as a literal in the outer scope */
         newVal.type = DST_FUNCDEF;
-        newVal.data.def = def;
+        newVal.as.def = def;
         literalIndex = compiler_add_literal(c, scope, newVal);
         dst_buffer_push_u16(c->vm, buffer, DST_OP_CLN);
         dst_buffer_push_u16(c->vm, buffer, ret.index);
@@ -1070,7 +1070,7 @@ static SpecialFormHelper get_special(const DstValue *form) {
     const uint8_t *name;
     if (dst_tuple_length(form) < 1 || form[0].type != DST_SYMBOL)
         return NULL;
-    name = form[0].data.string;
+    name = form[0].as.string;
     /* If we have a symbol with a zero length name, we have other
      * problems. */
     if (dst_string_length(name) == 0)
@@ -1292,13 +1292,13 @@ static Slot compile_value(DstCompiler *c, FormOptions opts, DstValue x) {
             ret = compile_symbol(c, opts, x);
             break;
         case DST_TUPLE:
-            ret = compile_form(c, opts, x.data.tuple);
+            ret = compile_form(c, opts, x.as.tuple);
             break;
         case DST_ARRAY:
-            ret = compile_array(c, opts, x.data.array);
+            ret = compile_array(c, opts, x.as.array);
             break;
         case DST_TABLE:
-            ret = compile_table(c, opts, x.data.table);
+            ret = compile_table(c, opts, x.as.table);
             break;
         default:
             ret = compile_literal(c, opts, x);

core/env.c

@@ -1,134 +0,0 @@
-/*
-* Copyright (c) 2017 Calvin Rose
-*
-* Permission is hereby granted, free of charge, to any person obtaining a copy
-* of this software and associated documentation files (the "Software"), to
-* deal in the Software without restriction, including without limitation the
-* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-* sell copies of the Software, and to permit persons to whom the Software is
-* furnished to do so, subject to the following conditions:
-*
-* The above copyright notice and this permission notice shall be included in
-* all copies or substantial portions of the Software.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-* IN THE SOFTWARE.
-*/
-
-#include "internal.h"
-
-static DstTable *dst_env_keytab(Dst *vm, DstTable *env, const char *keyword) {
-    DstTable *tab;
-    DstValue key = dst_string_cv(vm, keyword);
-    DstValue maybeTab = dst_table_get(env, key);
-    if (maybeTab.type != DST_TABLE) {
-        tab = dst_table(vm, 10);
-        dst_table_put(vm, env, key, dst_wrap_table(tab));
-    } else {
-        tab = maybeTab.data.table;
-    }
-    return tab;
-}
-
-DstTable *dst_env_nils(Dst *vm, DstTable *env) {
-    return dst_env_keytab(vm, env, "nils");
-}
-
-DstTable *dst_env_meta(Dst *vm, DstTable *env) {
-    return dst_env_keytab(vm, env, "meta");
-}
-
-/* Add many global variables and bind to nil */
-static void mergenils(Dst *vm, DstTable *destEnv, DstTable *nils) {
-    const DstValue *data = nils->data;
-    uint32_t len = nils->capacity;
-    uint32_t i;
-    DstTable *destNils = dst_env_nils(vm, destEnv);
-    for (i = 0; i < len; i += 2) {
-        if (data[i].type == DST_SYMBOL) {
-            dst_table_put(vm, destEnv, data[i], dst_wrap_nil());
-            dst_table_put(vm, destNils, data[i], dst_wrap_boolean(1));
-        }
-    }
-}
-
-/* Add many global variable metadata */
-static void mergemeta(Dst *vm, DstTable *destEnv, DstTable *meta) {
-    const DstValue *data = meta->data;
-    uint32_t len = meta->capacity;
-    uint32_t i;
-    DstTable *destMeta = dst_env_meta(vm, destEnv);
-    for (i = 0; i < len; i += 2) {
-        if (data[i].type == DST_SYMBOL) {
-            dst_table_put(vm, destMeta, data[i], data[i + 1]);
-        }
-    }
-}
-
-/* Simple strequal between dst string ans c string, no 0s in b allowed */
-static int streq(const char *str, const uint8_t *b) {
-    uint32_t len = dst_string_length(b);
-    uint32_t i;
-    const uint8_t *ustr = (const uint8_t *)str;
-    for (i = 0; i < len; ++i) {
-        if (ustr[i] != b[i])
-            return 0;
-    }
-    return 1;
-}
-
-/* Add many global variables */
-void dst_env_merge(Dst *vm, DstTable *destEnv, DstTable *srcEnv) {
-    const DstValue *data = srcEnv->data;
-    uint32_t len = srcEnv->capacity;
-    uint32_t i;
-    for (i = 0; i < len; i += 2) {
-        if (data[i].type == DST_SYMBOL) {
-            dst_table_put(vm, destEnv, data[i], data[i + 1]);
-        } else if (data[i].type == DST_STRING) {
-            const uint8_t *k = data[i].data.string;
-            if (streq("nils", k)) {
-                if (data[i + 1].type == DST_TABLE)
-                    mergenils(vm, destEnv, data[i + 1].data.table);
-            } else if (streq("meta", k)) {
-               if (data[i + 1].type == DST_TABLE)
-                mergemeta(vm, destEnv, data[i + 1].data.table);
-            }
-        }
-    }
-}
-
-void dst_env_put(Dst *vm, DstTable *env, DstValue key, DstValue value) {
-    DstTable *meta = dst_env_meta(vm, env);
-    dst_table_put(vm, meta, key, dst_wrap_nil());
-    dst_table_put(vm, env, key, value);
-    if (value.type == DST_NIL) {
-        dst_table_put(vm, dst_env_nils(vm, env), key, dst_wrap_boolean(1));
-    }
-}
-
-void dst_env_putc(Dst *vm, DstTable *env, const char *key, DstValue value) {
-    DstValue keyv = dst_string_cvs(vm, key);
-    dst_env_put(vm, env, keyv, value);
-}
-
-void dst_env_putvar(Dst *vm, DstTable *env, DstValue key, DstValue value) {
-    DstTable *meta = dst_env_meta(vm, env);
-    DstTable *newmeta = dst_table(vm, 4);
-    DstArray *ref = dst_array(vm, 1);
-    ref->count = 1;
-    ref->data[0] = value;
-    dst_table_put(vm, env, key, dst_wrap_array(ref));
-    dst_table_put(vm, newmeta, dst_string_cv(vm, "mutable"), dst_wrap_boolean(1));
-    dst_table_put(vm, meta, key, dst_wrap_table(newmeta));
-}
-
-void dst_env_putvarc(Dst *vm, DstTable *env, const char *key, DstValue value) {
-    DstValue keyv = dst_string_cvs(vm, key);
-    dst_env_putvar(vm, env, keyv, value);
-}

core/func.c

@@ -0,0 +1,544 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#include <setjmp.h>
+
+#include "internal.h"
+#include "wrap.h"
+#include "gc.h"
+
+/* Bytecode op argument types */
+
+/* s - a slot */
+/* c - a constant */
+/* i - a small integer */
+/* t - a type (have a simple type for non unions) */
+/* l - a label */
+
+typedef enum DstOpArgType DstOpArgType;
+enum DstOpArgType {
+    DST_OAT_SLOT,
+    DST_OAT_ENVIRONMENT,
+    DST_OAT_CONSTANT,
+    DST_OAT_INTEGER,
+    DST_OAT_TYPE,
+    DST_OAT_SIMPLETYPE,
+    DST_OAT_LABEL
+}
+
+/* Convert a slot to to an integer for bytecode */
+
+/* Types of instructions */
+/* _0arg - op.---.--.-- (return-nil, noop, vararg arguments) 
+ * _s - op.src.--.-- (push1)
+ * _l - op.XX.XX.XX (jump)
+ * _ss - op.dest.XX.XX (move, swap)
+ * _sl - op.check.XX.XX (jump-if)
+ * _st - op.check.TT.TT (typecheck)
+ * _si - op.dest.XX.XX (load-integer)
+ * _sss - op.dest.op1.op2 (add, subtract, arithmetic, comparison)
+ * _ses - op.dest.up.which (load-upvalue, save-upvalue)
+ * _sc - op.dest.CC.CC (load-constant, closure)
+ */
+
+/* Various types of instructions */
+typedef enum DstInstructionType DstInstructionType;
+enum DstInstructionType {
+    DIT_0, /* No args */
+    DIT_S, /* One slot */
+    DIT_L, /* One label */
+    DIT_SS, /* Two slots */
+    DIT_SL,
+    DIT_ST,
+    DIT_SI,
+    DIT_SSS,
+    DIT_SES,
+    DIT_SC
+};
+
+/* Definition for an instruction in the assembler */
+typedef struct DstInstructionDef DstInstructionDef;
+struct DstInstructionDef {
+    const char *name;
+    DstInstructionType type;
+    uint8_t opcode;
+};
+
+/* Hold all state needed during assembly */
+typedef struct DstAssembler DstAssembler;
+struct DstAssembler {
+    DstAssembler *parent;
+    Dst *vm;
+    DstFuncDef *def;
+    DstValue name;
+    jmp_buf onError;
+
+    DstTable *labels; /* symbol -> bytecode index */
+    DstTable *constants; /* symbol -> constant index */
+    DstTable *slots; /* symbol -> slot index */
+    DstTable *envs; /* symbol -> environment index */
+    uint32_t *bytecode; /* Where to put bytecode */
+    uint32_t bytecode_capacity; /* Set once */
+    uint32_t bytecode_count;
+}
+
+/* The DST value types in order. These types can be used as
+ * mnemonics instead of a bit pattern for type checking */
+static const char *types[] = {
+    "nil",
+    "real",
+    "integer",
+    "boolean",
+    "string",
+    "symbol",
+    "array",
+    "tuple",
+    "table",
+    "struct",
+    "thread",
+    "buffer",
+    "function",
+    "cfunction",
+    "userdata"
+};
+
+/* Dst opcode descriptions in lexographic order. This
+ * allows a binary search over the elements to find the
+ * correct opcode given a name. This works in reasonable
+ * time is easier to setup statically than a hash table or
+ * prefix tree. */
+static const char *dst_ops[] = {
+    {"add", DIT_SSS, 0x01},
+    {"bitand", DIT_SSS, 0x02},
+    {"bitor", DIT_SSS, 0x03},
+    {"bitxor", DIT_SSS, 0x04},
+    {"call", DIT_SS, 0x05},
+    {"closure", DIT_SC, 0x06},
+    {"divide", DIT_SSS, 0x07},
+    {"jump", DIT_L, 0x08},
+    {"jump-if", DIT_SL, 0x09},
+    {"load-constant", DIT_SC, 0x0A},
+    {"load-false", DIT_S, 0x0B},
+    {"load-integer", DIT_SI, 0x0C},
+    {"load-nil", DIT_S, 0x0D},
+    {"load-true", DIT_S, 0x0E},
+    {"load-upvalue", DIT_SES, 0x0F},
+    {"move", DIT_SS, 0x10},
+    {"modulo", DIT_SSS, 0x11},
+    {"multiply", DIT_SSS, 0x12},
+    {"noop", DIT_0, 0x13},
+    {"push", DIT_VARG, 0x14},
+    {"push1", DIT_S, 0x15},
+    {"push2", DIT_SS, 0x16},
+    {"push3", DIT_SSS, 0x17},
+    {"push-array", DIT_S, 0x18},
+    {"return", DIT_S, 0x19},
+    {"return-nil", DIT_0, 0x1A},
+    {"save-upvalue", DIT_SES, 0x1B},
+    {"shift-left", DIT_SSS, 0x1C},
+    {"shift-right", DIT_SSS, 0x1D},
+    {"shift-right-signed", DIT_SSS, 0x1E},
+    {"subtract", DIT_SSS, 0x1F},
+    {"swap", DIT_SS, 0x20},
+    {"syscall", DIT_SI, 0x21},
+    {"tail-call", DIT_S, 0x22},
+    {"transfer", DIT_SSS, 0x23},
+    {"typecheck", DIT_ST, 0x24},
+};
+
+/* Compare a DST string to a native 0 terminated c string. Used in the 
+ * binary search for the instruction definition. */
+static int dst_strcompare(const uint8_t *str, const char *other) {
+    uint32_t len = dst_string_length(str);
+    int index;
+    for (index = 0; index < len; index++) {
+        uint8_t c = str[index];
+        uint8_t k = ((const uint8_t *)other)[index];
+        if (c < k) return -1;
+        if (c > k) return 1;
+        if (k == '\0') break;
+    }
+    return (other[index] == '\0') ? 0 : -1;
+}
+
+/* Find an instruction definition given its name */
+static DstInstructionDef *dst_findi(const uint8_t *key) {
+    DstInstructionDef *low = dst_ops;
+    DstInstructionDef *hi = dst_ops + (sizeof(dst_ops) / sizeof(DstInstructionDef));
+    while (low < hi) {
+        DstInstructionDef *mid = low + ((hi - low) / 2);
+        int comp = dst_strcompare(key, mid->name);
+        if (comp < 0) {
+            hi = mid;
+        } else if (comp > 0) {
+            low = mid + 1;
+        } else {
+            return mid;
+        }
+    }
+    return NULL;
+}
+
+/* Check a dst string against a bunch of test_strings. Return the 
+ * index of the matching test_string, or -1 if not found. */
+static int strsearch(const uint8_t *str, const char **test_strings) {
+    uint32_t len = dst_string_length(str);
+    int index;
+    for (index = 0; ; index++) {
+        uint32_t i;
+        const char *testword = test_strings[index];
+        if (NULL == testword)
+            break;
+        for (i = 0; i < len; i++) {
+            if (testword[i] != str[i])
+                goto nextword;
+        }
+        return index;
+        :nextword
+    }
+    return -1;
+}
+
+/* Takes some dst assembly and gets the required capacity 
+ * for the output bytecode. Does not do any memory allocation, */
+static uint32_t estimate_capacity(const DstValue *assembly, uint32_t n) {
+    uint32_t i;
+    uint32_t cap = 0;
+    for (i = 0; i < n; i++) {
+        /* Ignore non tuple types, they are labels */
+        if (assembly[i].type != DST_TUPLE) continue;
+        cap++;
+    }
+    return cap;
+}
+
+/* Throw some kind of assembly error */
+static void dst_asm_error(DstAssembler *a, const char *message) {
+    printf("%s\n", message);
+    exit(1);
+}
+
+/* Parse an argument to an assembly instruction, and return the result as an
+ * integer. This integer will need to be trimmed and bound checked. */
+static int64_t doarg_1(DstAssembler *a, DstOpArgType argtype, DstValue x) {
+    DstTable *names;
+    switch (argtype) {
+        case DST_OAT_SLOT:
+            c = a->slots;
+            break;
+        case DST_OAT_ENVIRONMENT:
+            c = e->envs;
+            break;
+        case DST_OAT_CONSTANT:
+            c = a->constants;
+            break;
+        case DST_OAT_INTEGER:
+            c = NULL;
+            break;
+        case DST_OAT_TYPE:
+        case DST_OAT_SIMPLETYPE:
+            c = NULL;
+            break;
+        case DST_OAT_LABEL:
+            c = a->labels;
+            break;
+    }
+    switch (x.type) {
+        default:
+            break;
+        case DST_INTEGER:
+            return x.as.integer;
+        case DST_TUPLE:
+        {
+            if (argtype == DST_OAT_TYPE) {
+                int64_t result = 0;
+                uint32_t i = 0;
+                for (i = 0; i < dst_tuple_length(x.as.tuple); i++) {
+                    result |= dst_asm_argument(a, DST_OAT_SIMPLETYPE, x.as.tuple[i]);
+                }
+                return result;
+            }
+            break;
+        }
+        case DST_SYMBOL:
+        {
+            if (NULL != names) {
+                DstValue result = dst_table_get(names, x);
+                if (result.type == DST_INTEGER) {
+                    if (argtype == DST_OAT_LABEL)
+                        return result.as.integer - a->bytecode_count;
+                    return result.as.integer; 
+                } else {
+                    dst_asm_error(a, "unknown name");
+                }
+            } else if (argtype == DST_OAT_TYPE || argtype == DST_OAT_SIMPLETYPE) {
+                int index = strsearch(x.as.string, types);
+                if (index != -1) {
+                    return (int64_t) index;
+                } else {
+                    dst_asm_error(a, "unknown type");
+                }
+            }
+            break;
+        }
+    }
+    dst_asm_error(a, "unexpected type parsing instruction argument");
+}
+
+/* Trim a bytecode operand to 1, 2, or 3 bytes. Error out if
+ * the given argument doesn't fit in the required number of bytes. */
+static uint32_t doarg_2(int nbytes, int hassign, int64_t arg) {
+    /* Calculate the min and max values that can be stored given
+     * nbytes, and whether or not the storage is signed */
+    int64_t min = (-hassign) << ((nbytes << 3) - 1);
+    int64_t max = ~((-1) << ((nbytes << 3) - hassign));
+    if (arg < min)
+        dst_asm_error(a, "instruction argument is too small");
+    if (arg > max)
+        dst_asm_error(a, "instruction argument is too large");
+    return (uint32_t) (arg & 0xFFFFFFFF);
+}
+
+/* Parse a single argument to an instruction. Trims it as well as
+ * try to convert arguments to bit patterns */
+static uint32_t doarg(DstAssembler *a, DstOpArgType argtype, int nth, int nbytes, int hassign, DstValue x) {
+    int64_t arg1 = doarg_1(a, argtype, x);
+    return doarg_2(nbytes, hassign, arg1) << (nth << 3);
+}
+
+/* Provide parsing methods for the different kinds of arguments */
+static uint32_t read_instruction(DstAssembler *a, const DstInstructionDef *idef, const DstValue *argt) {
+    uint32_t instr = idef->opcode;
+    switch (idef->type) {
+        case DIT_0:
+        {
+            if (dst_tuple_lenth(argt) != 1)
+                dst_asm_error(a, "expected 0 arguments: (op)");
+            break;
+        }
+        case DIT_S:
+        {
+            if (dst_tuple_lenth(argt) != 2)
+                dst_asm_error(a, "expected 1 argument: (op, slot)");
+            instr |= doarg(a, DST_OAT_SLOT, 3, 3, 0, argt[1]);
+            break;
+        }
+        case DIT_L:
+        {
+            if (dst_tuple_lenth(argt) != 2)
+                dst_asm_error(a, "expected 1 argument: (op, label)");
+            instr |= doarg(a, DST_OAT_LABEL, 3, 3, 1, argt[1]);
+            break;
+        }
+        case DIT_SS:
+        {
+            if (dst_tuple_lenth(argt) != 3)
+                dst_asm_error(a, "expected 2 arguments: (op, slot, slot)");
+            instr |= doarg(a, DST_OAT_SLOT, 1, 1, 0, argt[1]);
+            instr |= doarg(a, DST_OAT_SLOT, 3, 2, 0, argt[2]);
+            break;
+        }
+        case DIT_SL:
+        {
+            if (dst_tuple_lenth(argt) != 3)
+                dst_asm_error(a, "expected 2 arguments: (op, slot, label)");
+            instr |= doarg(a, DST_OAT_SLOT, 1, 1, 0, argt[1]);
+            instr |= doarg(a, DST_OAT_LABEL, 3, 2, 1, argt[2]);
+            break;
+        }
+        case DIT_ST:
+        {
+            if (dst_tuple_lenth(argt) != 3)
+                dst_asm_error(a, "expected 2 arguments: (op, slot, type)");
+            instr |= doarg(a, DST_OAT_SLOT, 1, 1, 0, argt[1]);
+            instr |= doarg(a, DST_OAT_TYPE, 3, 2, 0, argt[2]);
+            break;
+        }
+        case DIT_SI:
+        {
+            if (dst_tuple_lenth(argt) != 3)
+                dst_asm_error(a, "expected 2 arguments: (op, slot, integer)");
+            instr |= doarg(a, DST_OAT_SLOT, 1, 1, 0, argt[1]);
+            instr |= doarg(a, DST_OAT_INTEGER, 3, 2, 1, argt[2]);
+            break;
+        }
+        case DIT_SSS:
+        {
+            if (dst_tuple_lenth(argt) != 4)
+                dst_asm_error(a, "expected 3 arguments: (op, slot, slot, slot)");
+            instr |= doarg(a, DST_OAT_SLOT, 1, 1, 0, argt[1]);
+            instr |= doarg(a, DST_OAT_SLOT, 2, 1, 0, argt[2]);
+            instr |= doarg(a, DST_OAT_SLOT, 3, 1, 0, argt[3]);
+            break;
+        }
+        case DIT_SES:
+        {
+            DstAssembler *b = a;
+            uint32_t envn;
+            if (dst_tuple_lenth(argt) != 4)
+                dst_asm_error(a, "expected 3 arguments: (op, slot, environment, envslot)");
+            instr |= doarg(a, DST_OAT_SLOT, 1, 1, 0, argt[1]);
+            envn = doarg(a, DST_OAT_ENVIRONMENT, 0, 1, 0, argt[2]);
+            instr |= envn << 16;
+            for (env += 1; env > 0; env--) {
+                b = b->parent;
+                if (NULL == b)
+                    dst_asm_error(a, "invalid environment index");
+            }
+            instr |= doarg(b, DST_OAT_SLOT, 3, 1, 0, argt[3]);
+            break;
+        }
+        case DIT_SC:
+        {
+            if (dst_tuple_lenth(argt) != 3)
+                dst_asm_error(a, "expected 2 arguments: (op, slot, constant)");
+            instr |= doarg(a, DST_OAT_SLOT, 1, 1, 0, argt[1]);
+            instr |= doarg(a, DST_OAT_CONSTANT, 3, 2, 0, argt[2]);
+            break;
+        }
+    }
+    return instr;
+}
+
+/* Do assembly. Return 0 if successful, else return an error code. */
+static void dst_asm1(DstAssembler *a, DstValue src) {
+    DstTable *t = src.as.table;
+    DstFuncDef *def = a->def;
+    uint32_t i;
+    DstValue x;
+
+    if (src.type != DST_TABLE) 
+        dst_asm_error(a, "expected table");
+    x = dst_table_get(t, dst_wrap_symbol(dst_cstring(a->vm, "arity")));
+    def->arity = x.type == DST_INTEGER ? x.as.integer : 0;
+    x = dst_table_get(t, dst_wrap_symbol(dst_cstring(a->vm, "stack")));
+    def->locals = x.type == DST_INTEGER ? x.as.integer : 0;
+
+    // Check name
+    x = dst_table_get(t, dst_wrap_symbol(dst_cstring(a->vm, "name")));
+    if (x.type == SYMBOL) {
+       a->name = x; 
+    }
+
+    // Create slot aliases
+    x = dst_table_get(t, dst_wrap_symbol(dst_cstring(a->vm, "slots")));
+    if (x.type == DST_ARRAY) {
+        for (i = 0; i < x.as.array->count; i++) {
+            DstValue v = x.as.array->data[i];
+            if (v.type == DST_TUPLE) {
+                uint32_t j; 
+                for (j = 0; j < dst_tuple_length(v.as.tuple); j++) {
+                    if (v.as.tuple[j].type != SYMBOL)
+                        dst_asm_error("slot names must be symbols");
+                    dst_table_put(a->vm, a->slots, v.as.tuple[j], dst_wrap_integer(i));
+                }
+            } else if (v.type == DST_SYMBOL) {
+                dst_table_put(a->vm, a->slots, v, dst_wrap_integer(i));
+            } else {
+                dst_asm_error(a, "slot names must be symbols or tuple of symbols");
+            }
+        }
+    }
+
+    // Create environment aliases
+    x = dst_table_get(t, dst_wrap_symbol(dst_cstring(a->vm, "environments")));
+    if (x.type == DST_ARRAY) {
+        for (i = 0; i < x.as.array->count; i++) {
+            DstAssembler *b = a->parent;
+            DstValue v = x.as.array->data[i];
+            if (v.type != DST_SYMBOL) {
+                dst_asm_error(a, "expected a symbol");
+                while (NULL != b) {
+                    if (dst_equals(b->name, v)) {
+                        break;
+                    }
+                    b = b->parent;
+                }
+            // Check parent assemblers to find the given environment
+        }
+    }
+}
+
+/* Detach an environment that was on the stack from the stack, and
+ * ensure that its environment persists */
+void dst_funcenv_detach_(Dst *vm, DstFuncEnv *env) {
+    DstThread *thread = env->thread;
+    DstValue *stack = thread->data + thread->count;
+    uint32_t size = dst_frame_size(stack);
+    DstValue *values = malloc(sizeof(DstValue * size));
+    if (NULL == values) {
+        DST_OUT_OF_MEMORY;
+    }
+    /* Copy stack into env values (the heap) */
+    memcpy(values, stack, sizeof(DstValue) * size);
+    /* Update env */
+    env->thread = NULL;
+    env->stackOffset = size;
+    env->values = values;
+}
+
+/* Deinitialize an environment */
+void dst_funcenv_deinit(DstFuncEnv *env) {
+    if (NULL == env->thread && NULL != env->values) {
+        free(env->values);
+    }
+}
+
+/* Create the FuncEnv for the current stack frame. */
+DstFuncEnv *dst_funcenv_init_(Dst *vm, DstFuncEnv *env, DstThread *thread, DstValue *stack) {
+    env->thread = thread;
+    env->stackOffset = thread->count;
+    env->values = NULL;
+    dst_frame_env(stack) = env;
+    return env;
+}
+
+
+/* Create a funcdef */
+DstFuncDef *dst_funcdef_init_(Dst *vm, DstFuncDef *def) {
+    uint8_t * byteCode = dst_alloc(c->vm, lastNBytes);
+    def->byteCode = (uint16_t *)byteCode;
+    def->byteCodeLen = lastNBytes / 2;
+    /* Copy the last chunk of bytes in the buffer into the new
+     * memory for the function's byteCOde */
+    dst_memcpy(byteCode, buffer->data + buffer->count - lastNBytes, lastNBytes);
+    /* Remove the byteCode from the end of the buffer */
+    buffer->count -= lastNBytes;
+    /* Create the literals used by this function */
+    if (scope->literalsArray->count) {
+        def->literals = dst_alloc(c->vm, scope->literalsArray->count * sizeof(DstValue));
+        dst_memcpy(def->literals, scope->literalsArray->data,
+                scope->literalsArray->count * sizeof(DstValue));
+    } else {
+        def->literals = NULL;
+    }
+    def->literalsLen = scope->literalsArray->count;
+    /* Delete the sub scope */
+    compiler_pop_scope(c);
+    /* Initialize the new FuncDef */
+    def->locals = scope->frameSize;
+    def->arity = arity;
+    def->flags = (varargs ? DST_FUNCDEF_FLAG_VARARG : 0) |
+        (scope->touchParent ? DST_FUNCDEF_FLAG_NEEDSPARENT : 0) |
+        (scope->touchEnv ? DST_FUNCDEF_FLAG_NEEDSENV : 0);
+    return def;
+}

core/gc.c

@@ -21,179 +21,251 @@
 */
 
 #include "internal.h"
+#include "cache.h"
+#include "wrap.h"
 
-/* The metadata header associated with an allocated block of memory */
-#define gc_header(mem) ((GCMemoryHeader *)(mem) - 1)
+/* Helpers for marking the various gc types */
+static void dst_mark_funcenv(DstFuncEnv *env);
+static void dst_mark_funcdef(DstFuncEnv *def);
+static void dst_mark_function(DstFunction *func);
+static void dst_mark_array(DstArray *array);
+static void dst_mark_table(DstTable *table);
+static void dst_mark_struct(const DstValue *st);
+static void dst_mark_tuple(const DstValue *tuple);
+static void dst_mark_buffer(DstBuffer *buffer);
+static void dst_mark_string(const uint8_t *str);
+static void dst_mark_thread(DstThread *thread);
+static void dst_mark_udata(void *udata);
 
-/* Memory header struct. Node of a linked list of memory blocks. */
-typedef struct GCMemoryHeader GCMemoryHeader;
-struct GCMemoryHeader {
-    GCMemoryHeader * next;
-    uint32_t color : 1;
-    uint32_t tags : 31;
-};
+/* Mark a value */
+void dst_mark(DstValue x) {
+    switch (x.type) {
+        default: break;
+        case DST_STRING:
+        case DST_SYMBOL: dst_mark_string(x.as.string); break;
+        case DST_FUNCTION: dst_mark_function(x.as.function); break;
+        case DST_ARRAY: dst_mark_array(x.as.array); break;
+        case DST_TABLE: dst_mark_table(x.as.table); break;
+        case DST_STRUCT: dst_mark_struct(x.as.st); break;
+        case DST_TUPLE: dst_mark_tuple(x.as.tuple); break;
+        case DST_BUFFER: dst_mark_buffer(x.as.buffer); break;
+        case DST_STRING: dst_mark_string(x.as.string); break;
+        case DST_THREAD: dst_mark_thread(x.as.thread); break;
+        case DST_USERDATA: dst_mark_udata(x.as.pointer); break;
+    }
+}
 
-/* Mark a chunk of memory as reachable for the gc */
-void dst_mark_mem(Dst *vm, void *mem) {
-	gc_header(mem)->color = vm->black;
+/* Unpin a value */
+void dst_unpin(DstValue x) {
+    switch (x.type) {
+        default: break;
+        case DST_STRING:
+        case DST_SYMBOL: dst_unpin_string(x.as.string); break;
+        case DST_FUNCTION: dst_unpin_function(x.as.function); break;
+        case DST_ARRAY: dst_unpin_array(x.as.array); break;
+        case DST_TABLE: dst_unpin_table(x.as.table); break;
+        case DST_STRUCT: dst_unpin_struct(x.as.st); break;
+        case DST_TUPLE: dst_unpin_tuple(x.as.tuple); break;
+        case DST_BUFFER: dst_unpin_buffer(x.as.buffer); break;
+        case DST_STRING: dst_unpin_string(x.as.string); break;
+        case DST_THREAD: dst_unpin_thread(x.as.thread); break;
+        case DST_USERDATA: dst_unpin_udata(x.as.pointer); break;
+    }
+}
+
+/* Pin a value */
+void dst_pin(DstValue x) {
+    switch (x.type) {
+        default: break;
+        case DST_STRING:
+        case DST_SYMBOL: dst_pin_string(x.as.string); break;
+        case DST_FUNCTION: dst_pin_function(x.as.function); break;
+        case DST_ARRAY: dst_pin_array(x.as.array); break;
+        case DST_TABLE: dst_pin_table(x.as.table); break;
+        case DST_STRUCT: dst_pin_struct(x.as.st); break;
+        case DST_TUPLE: dst_pin_tuple(x.as.tuple); break;
+        case DST_BUFFER: dst_pin_buffer(x.as.buffer); break;
+        case DST_STRING: dst_pin_string(x.as.string); break;
+        case DST_THREAD: dst_pin_thread(x.as.thread); break;
+        case DST_USERDATA: dst_pin_udata(x.as.pointer); break;
+    }
+}
+
+static void dst_mark_string(const uint8_t *str) {
+    gc_mark(dst_string_raw(str));
+}
+
+static void dst_mark_buffer(DstBuffer *buffer) {
+    gc_mark(buffer);
+}
+
+static void dst_mark_udata(void *udata) {
+    gc_mark(dst_udata_header(udata));
+}
+
+/* Mark a bunch of items in memory */
+static void dst_mark_many(const DstValue *values, uint32_t n) {
+    const DstValue *end = values + n;
+    while (values < end) {
+        dst_mark(*values)
+        ++values;
+    }
+}
+
+static void dst_mark_array(DstArray *array) {
+    if (gc_reachable(array))
+        return;
+    gc_mark(array);
+    dst_mark_many(array->data, array->count);
+}
+
+static void dst_mark_table(DstTable *table) {
+    if (gc_reachable(table))
+        return;
+    gc_mark(table);
+    dst_mark_many(table->data, table->capacity);
+}
+
+static void dst_mark_struct(const DstValue *st) {
+    if (gc_reachable(dst_struct_raw(st)))
+        return;
+    gc_mark(dst_struct_raw(st));
+    dst_mark_many(st, dst_struct_capacity(st));
+}
+
+static void dst_mark_tuple(const DstValue *tuple) {
+    if (gc_reachable(dst_tuple_raw(tuple)))
+        return;
+    gc_mark(dst_tuple_raw(tuple));
+    dst_mark_many(tuple, dst_tuple_count(tuple));
 }
 
 /* Helper to mark function environments */
-static void dst_mark_funcenv(Dst *vm, DstFuncEnv *env) {
-    if (gc_header(env)->color != vm->black) {
-        gc_header(env)->color = vm->black;
-        if (env->thread) {
-            DstValueUnion x;
-            x.thread = env->thread;
-            dst_mark(vm, x, DST_THREAD);
-        }
+static void dst_mark_funcenv(DstFuncEnv *env) {
+    if (gc_reachable(env))
+        return;
+    gc_mark(env);
     if (env->values) {
         uint32_t count = env->stackOffset;
         uint32_t i;
-            gc_header(env->values)->color = vm->black;
         for (i = 0; i < count; ++i)
-                dst_mark_value(vm, env->values[i]);
+            dst_mark_value(env->values[i]);
+    }
+    if (env->thread)
+        dst_mark_thread(env->thread);
+}
+
+/* GC helper to mark a FuncDef */
+static void dst_mark_funcdef(DstFuncDef *def) {
+    uint32_t count, i;
+    if (gc_reachable(def))
+        return;
+    gc_mark(def);
+    if (def->literals) {
+        count = def->literalsLen;
+        for (i = 0; i < count; ++i) {
+            DstValue v = def->literals[i];
+            /* Funcdefs use boolean literals to store other funcdefs */
+            if (v.type == DST_BOOLEAN) {
+                dst_mark_funcdef((DstFuncDef *) v.as.pointer);
+            } else {
+                dst_mark(v);
+            }
         }
     }
 }
 
-/* GC helper to mark a FuncDef */
-static void dst_mark_funcdef(Dst *vm, DstFuncDef *def) {
-    if (gc_header(def)->color != vm->black) {
-        gc_header(def)->color = vm->black;
-        gc_header(def->byteCode)->color = vm->black;
-        uint32_t count, i;
-        if (def->literals) {
-            count = def->literalsLen;
-            gc_header(def->literals)->color = vm->black;
-            for (i = 0; i < count; ++i)
-                dst_mark_value(vm, def->literals[i]);
-        }
-    }
+static void dst_mark_function(DstFunction *func) {
+    uint32_t i;
+    uint32_t numenvs;
+    if (gc_reachable(func))
+        return;
+    gc_mark(func)
+    numenvs = fun->def->envLen;
+    for (i = 0; i < numenvs; ++i)
+        dst_mark_funcenv(func->envs + i);
+    dst_mark_funcdef(func->def);
 }
 
 /* Helper to mark a stack frame. Returns the next stackframe. */
 static DstValue *dst_mark_stackframe(Dst *vm, DstValue *stack) {
-    uint32_t i;
-    dst_mark_value(vm, dst_frame_callee(stack));
+    dst_mark(dst_frame_callee(stack));
     if (dst_frame_env(stack) != NULL)
-        dst_mark_funcenv(vm, dst_frame_env(stack));
-    for (i = 0; i < dst_frame_size(stack); ++i)
-        dst_mark_value(vm, stack[i]);
+        dst_mark_funcenv(dst_frame_env(stack));
+    /* Mark all values in the stack frame */
+    dst_mark_many(stack, dst_frame_size(stack));
+    /* Return the nexct frame */
     return stack + dst_frame_size(stack) + DST_FRAME_SIZE;
 }
 
-/* Wrapper for marking values */
-void dst_mark_value(Dst *vm, DstValue x) {
-    dst_mark(vm, x.data, x.type);
-}
-
-/* Mark allocated memory associated with a value. This is
- * the main function for doing the garbage collection mark phase. */
-void dst_mark(Dst *vm, DstValueUnion x, DstType type) {
-    /* Allow for explicit tail recursion */
-    begin:
-    switch (type) {
-        default:
-            break;
-
-        case DST_STRING:
-        case DST_SYMBOL:
-            gc_header(dst_string_raw(x.string))->color = vm->black;
-            break;
-
-        case DST_BYTEBUFFER:
-            gc_header(x.buffer)->color = vm->black;
-            gc_header(x.buffer->data)->color = vm->black;
-            break;
-
-        case DST_ARRAY:
-            if (gc_header(x.array)->color != vm->black) {
-                uint32_t i, count;
-                count = x.array->count;
-                gc_header(x.array)->color = vm->black;
-                gc_header(x.array->data)->color = vm->black;
-                for (i = 0; i < count; ++i)
-                    dst_mark_value(vm, x.array->data[i]);
-            }
-            break;
-
-        case DST_TUPLE:
-            if (gc_header(dst_tuple_raw(x.tuple))->color != vm->black) {
-                uint32_t i, count;
-                count = dst_tuple_length(x.tuple);
-                gc_header(dst_tuple_raw(x.tuple))->color = vm->black;
-                for (i = 0; i < count; ++i)
-                    dst_mark_value(vm, x.tuple[i]);
-            }
-            break;
-
-        case DST_STRUCT:
-            if (gc_header(dst_struct_raw(x.st))->color != vm->black) {
-                uint32_t i, count;
-                count = dst_struct_capacity(x.st);
-                gc_header(dst_struct_raw(x.st))->color = vm->black;
-                for (i = 0; i < count; ++i)
-                    dst_mark_value(vm, x.st[i]);
-            }
-            break;
-
-        case DST_THREAD:
-            if (gc_header(x.thread)->color != vm->black) {
-                DstThread *thread = x.thread;
+static void dst_mark_thread(DstThread *thread) {
     DstValue *frame = thread->data + DST_FRAME_SIZE;
     DstValue *end = thread->data + thread->count;
-                gc_header(thread)->color = vm->black;
-                gc_header(thread->data)->color = vm->black;
+    if (gc_reachable(thread))
+        return;
+    gc_mark(thread);
     while (frame <= end)
         frame = dst_mark_stackframe(vm, frame);
-                if (thread->parent) {
-                    x.thread = thread->parent;
-                    goto begin;
-                }
-            }
-            break;
+    if (thread->parent)
+        dst_mark_thread(thread->parent);
+}
 
-        case DST_FUNCTION:
-            if (gc_header(x.function)->color != vm->black) {
-                DstFunction *f = x.function;
-                gc_header(f)->color = vm->black;
-                dst_mark_funcdef(vm, f->def);
-                if (f->env)
-                    dst_mark_funcenv(vm, f->env);
-                if (f->parent) {
-                    DstValueUnion pval;
-                    pval.function = f->parent;
-                    dst_mark(vm, pval, DST_FUNCTION);
-                }
-            }
+/* Deinitialize a block of memory */
+static void dst_deinit_block(Dst *vm, GCMemoryHeader *block) {
+    void *mem = ((char *)(block + 1));
+    DstUserdataHeader *h = (DstUserdataHeader *)mem;
+    void *smem = mem + 2 * sizeof(uint32_t);
+    switch (current->tags) {
+        default:
+            break; /* Do nothing for non gc types */ 
+        case DST_MEMORY_STRING:
+            dst_cache_remove(vm, dst_wrap_string(smem));
             break;
-
-        case DST_TABLE:
-            if (gc_header(x.table)->color != vm->black) {
-                uint32_t i;
-                gc_header(x.table)->color = vm->black;
-                gc_header(x.table->data)->color = vm->black;
-                for (i = 0; i < x.table->capacity; i += 2) {
-                    dst_mark_value(vm, x.table->data[i]);
-                    dst_mark_value(vm, x.table->data[i + 1]);
-                }
-            }
+        case DST_MEMORY_ARRAY:
+            free(((DstArray*) mem)->data);
             break;
-
-        case DST_USERDATA:
+        case DST_MEMORY_TUPLE:
+            dst_cache_remove(vm, dst_wrap_tuple(smem));
+            break;
+        case DST_MEMORY_TABLE:
+            free(((DstTable*) mem)->data);
+            break;
+        case DST_MEMORY_STRUCT:
+            dst_cache_remove(vm, dst_wrap_struct(smem));
+            break;
+        case DST_MEMORY_THREAD:
+            free(((DstThread *) mem)->data);
+            break;
+        case DST_MEMORY_BUFFER:
+            free(((DstBuffer *) mem)->data);
+            break; 
+        case DST_MEMORY_FUNCTION:
             {
-                DstUserdataHeader *h = dst_udata_header(x.pointer);
-                gc_header(h)->color = vm->black;
+                DstFunction *f = (DstFunction *)mem;
+                if (NULL != f->envs)
+                    free(f->envs);
             }
             break;
-
-        case DST_FUNCENV:
-            dst_mark_funcenv(vm, x.env);
+        case DST_MEMORY_USERDATA:
+            if (h->type->finalize)
+                h->type->finalize(vm, (void *)(h + 1), h->size);
             break;
-
-        case DST_FUNCDEF:
-            dst_mark_funcdef(vm, x.def);
+        case DST_MEMORY_FUNCENV:
+            {
+                DstFuncEnv *env = (DstFuncEnv *)mem;
+                if (NULL == env->thread && NULL != env->values)
+                    free(env->values);
+            }
+            break;
+        case DST_MEMORY_FUNCDEF:
+            {
+                DstFunDef *def = (DstFuncDef *)mem;
+                /* TODO - get this all with one alloc and one free */
+                free(def->envSizes);
+                free(def->envCaptures);
+                free(def->literals);
+                free(def->byteCode);
+            }
             break;
     }
 }
@@ -206,124 +278,65 @@ void dst_sweep(Dst *vm) {
     GCMemoryHeader *next;
     while (current) {
         next = current->next;
-        if (current->color != vm->black) {
+        if (current->flags & (DST_MEM_REACHABLE | DST_MEM_DISABLED)) {
+            previous = current;
+        } else {
+            dst_deinit_block(vm, current);
             if (previous) {
                 previous->next = next;
             } else {
                 vm->blocks = next;
             }
-            if (current->tags) {
-                if (current->tags & DST_MEMTAG_STRING)
-                    dst_cache_remove_string(vm, (char *)(current + 1));
-                if (current->tags & DST_MEMTAG_STRUCT)
-                    dst_cache_remove_struct(vm, (char *)(current + 1));
-                if (current->tags & DST_MEMTAG_TUPLE)
-                    dst_cache_remove_tuple(vm, (char *)(current + 1));
-                if (current->tags & DST_MEMTAG_USER) {
-                    DstUserdataHeader *h = (DstUserdataHeader *)(current + 1);
-                    if (h->type->finalize) {
-                        h->type->finalize(vm, h + 1, h->size);
-                    }
-                }
-            }
-            dst_raw_free(current);
-        } else {
-            previous = current;
+            free(current);
         }
+        current->flags &= ~DST_MEM_REACHABLE;
         current = next;
     }
-    /* Rotate flag */
-    vm->black = !vm->black;
-}
-
-/* Prepare a memory block */
-static void *dst_alloc_prepare(Dst *vm, char *rawBlock, uint32_t size) {
-    GCMemoryHeader *mdata;
-    if (rawBlock == NULL) {
-        return NULL;
-    }
-    vm->nextCollection += size;
-    mdata = (GCMemoryHeader *)rawBlock;
-    mdata->next = vm->blocks;
-    vm->blocks = mdata;
-    mdata->color = !vm->black;
-    mdata->tags = 0;
-    return rawBlock + sizeof(GCMemoryHeader);
 }
 
 /* Allocate some memory that is tracked for garbage collection */
-void *dst_alloc(Dst *vm, uint32_t size) {
-    uint32_t totalSize = size + sizeof(GCMemoryHeader);
-    void *mem = dst_alloc_prepare(vm, dst_raw_alloc(totalSize), totalSize);
-    if (!mem) {
-        DST_LOW_MEMORY;
-        dst_collect(vm);
-        mem = dst_alloc_prepare(vm, dst_raw_alloc(totalSize), totalSize);
-        if (!mem) {
+void *dst_alloc(Dst *vm, DstMemoryType type, size_t size) {
+    GCMemoryHeader *mdata;
+    size_t totalSize = size + sizeof(GCMemoryHeader);
+    void *mem = malloc(totalSize);
+
+    /* Check for bad malloc */
+    if (NULL == mem) {
         DST_OUT_OF_MEMORY;
     }
-    }
-    return mem;
-}
 
-/* Allocate some zeroed memory that is tracked for garbage collection */
-void *dst_zalloc(Dst *vm, uint32_t size) {
-    uint32_t totalSize = size + sizeof(GCMemoryHeader);
-    void *mem = dst_alloc_prepare(vm, dst_raw_calloc(1, totalSize), totalSize);
-    if (!mem) {
-        DST_LOW_MEMORY;
-        dst_collect(vm);
-        mem = dst_alloc_prepare(vm, dst_raw_calloc(1, totalSize), totalSize);
-        if (!mem) {
-            DST_OUT_OF_MEMORY;
-        }
-    }
-    return mem;
-}
+    mdata = (GCMemoryHeader *)rawBlock;
 
-/* Tag some memory to mark it with special properties */
-void dst_mem_tag(void *mem, uint32_t tags) {
-    GCMemoryHeader *mh = (GCMemoryHeader *)mem - 1;
-    mh->tags |= tags;
+    /* Configure block */
+    mdata->flags = type;
+
+    /* Prepend block to heap list */
+    vm->nextCollection += size;
+    mdata->next = vm->blocks;
+    vm->blocks = mdata;
+
+    return mem + sizeof(GCMemoryHeader);
 }
 
 /* Run garbage collection */
 void dst_collect(Dst *vm) {
-    DstValue x;
-    /* Thread can be null */
-    if (vm->thread) {
-        x.type = DST_THREAD;
-        x.data.thread = vm->thread;
-        dst_mark_value(vm, x);
-    }
-    x.type = DST_TABLE;
-
-    x.data.table = vm->modules;
-    dst_mark_value(vm, x);
-
-    x.data.table = vm->registry;
-    dst_mark_value(vm, x);
-
-    x.data.table = vm->env;
-    dst_mark_value(vm, x);
-
-    dst_mark_value(vm, vm->ret);
+    if (vm->thread)
+        dst_mark_thread(vm->thread);
+    dst_mark_table(vm->modules);
+    dst_mark_table(vm->registry);
+    dst_mark_table(vm->env);
+    dst_mark(vm->ret);
     dst_sweep(vm);
     vm->nextCollection = 0;
 }
 
-/* Run garbage collection if needed */
-void dst_maybe_collect(Dst *vm) {
-    if (vm->nextCollection >= vm->memoryInterval)
-        dst_collect(vm);
-}
-
 /* Free all allocated memory */
 void dst_clear_memory(Dst *vm) {
     GCMemoryHeader *current = vm->blocks;
     while (current) {
+        dst_deinit_block(vm, current);
         GCMemoryHeader *next = current->next;
-        dst_raw_free(current);
+        free(current);
         current = next;
     }
     vm->blocks = NULL;

core/gc.h

@@ -0,0 +1,120 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#ifndef DST_GC_H_defined
+#define DST_GC_H_defined
+
+#include "internal.h"
+
+/* The metadata header associated with an allocated block of memory */
+#define gc_header(mem) ((GCMemoryHeader *)(mem) - 1)
+
+#define DST_MEM_REACHABLE 0x100
+#define DST_MEM_DISABLED 0x200
+
+#define gc_settype(m, t) ((gc_header(m)->flags |= (0xFF & (t))))
+#define gc_type(m) (gc_header(m)->flags & 0xFF)
+
+#define gc_mark(m) (gc_header(m)->flags |= DST_MEM_REACHABLE)
+#define gc_unmark(m) (gc_header(m)->flags &= ~DST_MEM_COLOR)
+#define gc_reachable(m) (gc_header(m)->flags & DST_MEM_REACHABLE)
+
+
+/* Memory header struct. Node of a linked list of memory blocks. */
+typedef struct GCMemoryHeader GCMemoryHeader;
+struct GCMemoryHeader {
+    GCMemoryHeader *next;
+    uint32_t flags;
+};
+
+/* Memory types for the GC. Different from DstType to include funcenv and funcdef. */
+typedef enum DstMemoryType DstMemoryType;
+enum DstMemoryType {
+    DST_MEMORY_NONE,
+    DST_MEMORY_STRING,
+    DST_MEMORY_ARRAY,
+    DST_MEMORY_TUPLE,
+    DST_MEMORY_TABLE,
+    DST_MEMORY_STRUCT,
+    DST_MEMORY_THREAD,
+    DST_MEMORY_BUFFER,
+    DST_MEMORY_FUNCTION,
+    DST_MEMORY_USERDATA,
+    DST_MEMORY_FUNCENV,
+    DST_MEMORY_FUNCDEF
+}
+
+/* Prevent GC from freeing some memory. */
+#define dst_disablegc(m) (gc_header(m)->flags |= DST_MEM_DISABLED, (m))
+
+/* To allocate collectable memory, one must calk dst_alloc, initialize the memory,
+ * and then call when dst_enablegc when it is initailize and reachable by the gc (on the DST stack) */
+void *dst_alloc(Dst *vm, DstMemoryType type, size_t size);
+#define dst_enablegc(m) (gc_header(m)->flags &= ~DST_MEM_DISABLED, (m))
+
+/* When doing C interop, it is often needed to disable GC on a value.
+ * This is needed when a garbage collection could occur in the middle
+ * of a c function. This could happen, for example, if one calls back
+ * into dst inside of a c function. The pin and unpin functions toggle
+ * garbage collection on a value when needed. Note that no dst functions
+ * will call gc when you don't want it to. GC only happens automatically
+ * in the interpreter loop. */
+void dst_pin(DstValue x);
+void dst_unpin(DstValue x);
+
+/* Specific types can also be pinned and unpinned as well. */
+#define dst_pin_table dst_disablegc
+#define dst_pin_array dst_disablegc
+#define dst_pin_buffer dst_disablegc
+#define dst_pin_function dst_disablegc
+#define dst_pin_thread dst_disablegc
+#define dst_pin_string(s) dst_disablegc(dst_string_raw(s))
+#define dst_pin_symbol(s) dst_disablegc(dst_string_raw(s))
+#define dst_pin_tuple(s) dst_disablegc(dst_tuple_raw(s))
+#define dst_pin_struct(s) dst_disablegc(dst_struct_raw(s))
+#define dst_pin_userdata(s) dst_disablegc(dst_userdata_header(s))
+
+#define dst_unpin_table dst_enablegc
+#define dst_unpin_array dst_enablegc
+#define dst_unpin_buffer dst_enablegc
+#define dst_unpin_function dst_enablegc
+#define dst_unpin_thread dst_enablegc
+#define dst_unpin_string(s) dst_enablegc(dst_string_raw(s))
+#define dst_unpin_symbol(s) dst_enablegc(dst_string_raw(s))
+#define dst_unpin_tuple(s) dst_enablegc(dst_tuple_raw(s))
+#define dst_unpin_struct(s) dst_enablegc(dst_struct_raw(s))
+#define dst_unpin_userdata(s) dst_enablegc(dst_userdata_header(s))
+
+void dst_mark(DstValue x);
+void dst_sweep(Dst *vm);
+
+/* Collect some memory */
+void dst_collect(Dst *vm);
+
+/* Clear all memory. */
+void dst_clear_memory(Dst *vm);
+
+/* Run garbage collection if needed */
+#define dst_maybe_collect(Dst *vm) do {\
+    if (vm->nextCollection >= vm->memoryInterval) dst_collect(vm); } while (0)
+
+#endif

core/internal.h

@@ -24,7 +24,8 @@
 #define DST_INTERNAL_H_defined
 
 #include <dst/dst.h>
-#include <setjmp.h>
+#include <string.h>
+#include <stdlib.h>
 
 /* String utils */
 #define dst_string_raw(s) ((uint32_t *)(s) - 2)
@@ -47,41 +48,6 @@
 #define dst_udata_type(u) (dst_udata_header(u)->type)
 #define dst_udata_size(u) (dst_udata_header(u)->size)
 
-/* Memcpy for moving memory */
-#ifndef dst_memcpy
-#include <string.h>
-#define dst_memcpy memcpy
-#endif
-
-/* Allocation */
-#ifndef dst_raw_alloc
-#include <stdlib.h>
-#define dst_raw_alloc malloc
-#endif
-
-/* Zero allocation */
-#ifndef dst_raw_calloc
-#include <stdlib.h>
-#define dst_raw_calloc calloc
-#endif
-
-/* Realloc */
-#ifndef dst_raw_realloc
-#include <stdlib.h>
-#define dst_raw_realloc realloc
-#endif
-
-/* Free */
-#ifndef dst_raw_free
-#include <stdlib.h>
-#define dst_raw_free free
-#endif
-
-/* Null */
-#ifndef NULL
-#define NULL ((void *)0)
-#endif
-
 /* Stack frame manipulation */
 
 /* Size of stack frame in number of values */
@@ -89,7 +55,7 @@
 
 /* Prevent some recursive functions from recursing too deeply
  * ands crashing. */
-#define DST_RECURSION_GUARD 2056
+#define DST_RECURSION_GUARD 1000
 
 /* Macros for referencing a stack frame given a stack */
 #define dst_frame_callee(s)     (*(s - 1))
@@ -116,24 +82,14 @@
 
 /* What to do when out of memory */
 #ifndef DST_OUT_OF_MEMORY
-#include <stdlib.h>
 #include <stdio.h>
 #define DST_OUT_OF_MEMORY do { printf("out of memory\n"); exit(1); } while (0)
 #endif
 
-/* What to do when memory is low */
-#ifndef DST_LOW_MEMORY
-#include <stdlib.h>
-#include <stdio.h>
-#define DST_LOW_MEMORY do { printf("low memory\n"); } while (0)
-#endif
-
 /* A general dst value type */
 typedef struct DstValue DstValue;
 
 /* All of the dst types */
-typedef double DstReal;
-typedef int64_t DstInteger;
 typedef int DstBoolean;
 typedef struct DstFunction DstFunction;
 typedef struct DstArray DstArray;
@@ -146,22 +102,15 @@ typedef struct DstUserdataHeader DstUserdataHeader;
 typedef struct DstFuncDef DstFuncDef;
 typedef struct DstFuncEnv DstFuncEnv;
 typedef union DstValueUnion DstValueUnion;
-typedef struct DstModuleItem DstModuleItem;
 typedef struct DstUserType DstUserType;
 typedef struct DstParser DstParser;
 typedef struct DstParseState DstParseState;
 
-/* C Api data types */
-struct DstModuleItem {
-    const char *name;
-    DstCFunction data;
-};
-
 /* Union datatype */
 union DstValueUnion {
     DstBoolean boolean;
-    DstReal real;
-    DstInteger integer;
+    double real;
+    int64_t integer;
     DstArray *array;
     DstBuffer *buffer;
     DstTable *table;
@@ -169,8 +118,6 @@ union DstValueUnion {
     const DstValue *tuple;
     DstCFunction cfunction;
     DstFunction *function;
-    DstFuncEnv *env;
-    DstFuncDef *def;
     const DstValue *st;
     const uint8_t *string;
     /* Indirectly used union members */
@@ -186,7 +133,7 @@ union DstValueUnion {
  * the type information of the value */
 struct DstValue {
     DstType type;
-    DstValueUnion data;
+    DstValueUnion as;
 };
 
 /* A lightweight green thread in dst. Does not correspond to
@@ -228,18 +175,21 @@ struct DstTable {
 
 /* Some function defintion flags */
 #define DST_FUNCDEF_FLAG_VARARG 1
-#define DST_FUNCDEF_FLAG_NEEDSPARENT 2
 #define DST_FUNCDEF_FLAG_NEEDSENV 4
 
-/* A function definition. Contains information need to instantiate closures. */
+/* A function definition. Contains information needed to instantiate closures. */
 struct DstFuncDef {
-    uint32_t locals;
-    uint32_t arity; /* Not including varargs */
-    uint32_t literalsLen;
-    uint32_t byteCodeLen;
     uint32_t flags;
+    uint32_t locals; /* The amount of stack space required for the function */
+    uint32_t arity; /* Not including varargs */
+    uint32_t literalsLen; /* Number of literals */
+    uint32_t byteCodeLen; /* Length of bytcode */
+    uint32_t envLen; /* Number of environments */
+
+    uint32_t *envSizes; /* Minimum sizes of environments (for static analysis) */
+    uint32_t *envCaptures; /* Which environments to capture from parent. Is a bitset with the parent's envLen bits. */
     DstValue *literals; /* Contains strings, FuncDefs, etc. */
-    uint16_t *byteCode;
+    uint32_t *byteCode;
 };
 
 /* A fuction environment */
@@ -252,8 +202,7 @@ struct DstFuncEnv {
 /* A function */
 struct DstFunction {
     DstFuncDef *def;
-    DstFuncEnv *env;
-    DstFunction *parent;
+    DstFuncEnv *envs;
 };
 
 /* Defines a type for userdata */
@@ -321,23 +270,31 @@ enum DstOpCode {
 };
 
 /****/
-/* Internal buffer functions */
+/* Value Stack Manipulation */
 /****/
-void dst_value_buffer_ensure(Dst *vm, DstBuffer *buffer, uint32_t capacity);
-void dst_buffer_append_bytes(Dst *vm, DstBuffer *buffer, const uint8_t *string, uint32_t length);
-void dst_buffer_append_cstring(Dst *vm, DstBuffer *buffer, const char *cstring);
-
-/* Define a push function for pushing a certain type to the buffer */
-#define BUFFER_DEFINE(name, type) \
-static void dst_buffer_push_##name(Dst *vm, DstBuffer *buffer, type x) { \
-    union { type t; uint8_t bytes[sizeof(type)]; } u; \
-    u.t = x; dst_buffer_append(vm, buffer, u.bytes, sizeof(type)); \
-}
+void dst_switchv(Dst *vm, DstValue x);
+DstValue dst_popv(Dst *vm);
+DstValue dst_peekv(Dst *vm);
+void dst_pushv(Dst *vm, DstValue x);
+DstValue dst_getv(Dst *vm, int64_t index);
+void dst_setv(Dst *vm, int64_t index, DstValue x);
 
 /****/
-/* Table functions */
+/* Buffer functions */
+/****/
+void dst_buffer_ensure_(Dst *vm, DstBuffer *buffer, uint32_t capacity);
+void dst_buffer_push_u8_(Dst *vm, DstBuffer *buffer, uint8_t x);
+void dst_buffer_push_u16_(Dst *vm, DstBuffer *buffer, uint16_t x);
+void dst_buffer_push_u32_(Dst *vm, DstBuffer *buffer, uint32_t x);
+void dst_buffer_push_u64_(Dst *vm, DstBuffer *buffer, uint64_t x);
+void dst_buffer_push_bytes_(Dst *vm, DstBuffer *buffer, const uint8_t *bytes, uint32_t len);
+void dst_buffer_push_cstring_(Dst *vm, DstBuffer *buffer, const char *string);
+
+/****/
+/* Array functions */
 /****/
 DstArray *dst_make_array(Dst *vm, uint32_t capacity);
+void dst_array_ensure_(Dst *vm, DstArray *array, uint32_t capacity);
 
 /****/
 /* Tuple functions */
@@ -350,8 +307,6 @@ const DstValue *dst_tuple_end(Dst *vm, DstValue *tuple);
 /* String/Symbol functions */
 /****/
 
-uint8_t *dst_string_begin(Dst *vm, uint32_t len);
-void dst_string_end(Dst *vm, uint32_t dest, uint8_t *str);
 const uint8_t *dst_string_b(Dst *vm, const uint8_t *buf, uint32_t len);
 const uint8_t *dst_string_c(Dst *vm, const char *cstring);
 DstValue dst_string_cv(Dst *vm, const char *string);
@@ -409,129 +364,42 @@ const char *dst_deserialize_internal(
 
 const char *dst_serialize_internal(Dst *vm, DstBuffer *buffer, DstValue x);
 
-/****/
-/* GC */
-/****/
-
-#define DST_MEMTAG_STRING 4
-#define DST_MEMTAG_TUPLE 8
-#define DST_MEMTAG_STRUCT 16
-#define DST_MEMTAG_USER 32
-
-void dst_mark_value(Dst *vm, DstValue x);
-void dst_mark(Dst *vm, DstValueUnion x, DstType type);
-void dst_sweep(Dst *vm);
-void *dst_alloc(Dst *vm, uint32_t size);
-void *dst_zalloc(Dst *vm, uint32_t size);
-void dst_mem_tag(void *mem, uint32_t tags);
-void dst_collect(Dst *vm);
-void dst_maybe_collect(Dst *vm);
-void dst_clear_memory(Dst *vm);
-void dst_mark_mem(Dst *vm, void *mem);
-
-/****/
-/* VM */
-/****/
-
-DstValue dst_arg(Dst *vm, uint32_t index);
-void dst_set_arg(Dst *vm, uint32_t index, DstValue x);
-uint32_t dst_args(Dst *vm);
-
-/***/
-/* Stl */
-/***/
-
-void dst_stl_load(Dst *vm);
-
-/****/
-/* C Api */
-/****/
-
-void dst_module(Dst *vm, const char *name, const DstModuleItem *mod);
-void dst_module_mutable(Dst *vm, const char *name, const DstModuleItem *mod);
-void dst_module_put(Dst *vm, const char *packagename, const char *name, DstValue x);
-DstValue dst_module_get(Dst *vm, const char *packagename);
-void dst_register_put(Dst *vm, const char *packagename, DstValue mod);
-DstValue dst_register_get(Dst *vm, const char *name);
-int dst_callc(Dst *vm, DstCFunction fn, int numargs, ...);
-
 /* Treat similar types through uniform interfaces for iteration */
 int dst_seq_view(DstValue seq, const DstValue **data, uint32_t *len);
 int dst_chararray_view(DstValue str, const uint8_t **data, uint32_t *len);
 int dst_hashtable_view(DstValue tab, const DstValue **data, uint32_t *cap);
 
 /****/
-/* Caching for immutable data */
+/* Parse functions */
 /****/
 
-void dst_cache_remove_string(Dst *vm, char *strmem);
-void dst_cache_remove_tuple(Dst *vm, char *tuplemem);
-void dst_cache_remove_struct(Dst *vm, char *structmem);
+#define PARSE_OK 0
+#define PARSE_ERROR 1
+#define PARSE_UNEXPECTED_EOS 2
 
-/****/
-/* Misc */
-/****/
+int dst_parseb(Dst *vm, const uint8_t *src, const uint8_t **newsrc, uint32_t len);
 
-uint32_t dst_index(Dst *vm, int i);
-int dst_read_real(const uint8_t *string, const uint8_t *end, double *ret, int forceInt);
-int dst_read_integer(const uint8_t *string, const uint8_t *end, int64_t *ret);
-DstReal dst_integer_to_real(DstInteger x);
-DstInteger dst_real_to_integer(DstReal x);
-uint32_t dst_startrange(DstInteger raw, uint32_t len);
-uint32_t dst_endrange(DstInteger raw, uint32_t len);
-void dst_env_merge(Dst *vm, DstTable *destEnv, DstTable *srcEnv);
-DstTable *dst_env_nils(Dst *vm, DstTable *env);
-DstTable *dst_env_meta(Dst *vm, DstTable *env);
-void dst_env_put(Dst *vm, DstTable *env, DstValue key, DstValue value);
-void dst_env_putc(Dst *vm, DstTable *env, const char *key, DstValue value);
-void dst_env_putvar(Dst *vm, DstTable *env, DstValue key, DstValue value);
-void dst_env_putvarc(Dst *vm, DstTable *env, const char *key, DstValue value);
+/* Parse a c string */
+int dst_parsec(Dst *vm, const char *src, const char **newsrc);
+
+/* Parse a DST char seq (Buffer, String, Symbol) */
+int dst_parse(Dst *vm);
 
 /****/
 /* Value functions */
 /****/
 
-int dst_value_truthy(DstValue v);
-int dst_value_equals(DstValue x, DstValue y);
-uint32_t dst_value_hash(DstValue x);
-int dst_value_compare(DstValue x, DstValue y);
+int dst_truthy(DstValue v);
+int dst_equals(DstValue x, DstValue y);
+uint32_t dst_hash(DstValue x);
+int dst_compare(DstValue x, DstValue y);
+uint32_t dst_calchash_array(const DstValue *array, uint32_t len);
 
-/* Wrap data in GstValue */
-DstValue dst_wrap_nil();
-DstValue dst_wrap_real(DstReal x);
-DstValue dst_wrap_integer(DstInteger x);
-DstValue dst_wrap_boolean(int x);
-DstValue dst_wrap_string(const uint8_t *x);
-DstValue dst_wrap_symbol(const uint8_t *x);
-DstValue dst_wrap_array(DstArray *x);
-DstValue dst_wrap_tuple(const DstValue *x);
-DstValue dst_wrap_struct(const DstValue *x);
-DstValue dst_wrap_thread(DstThread *x);
-DstValue dst_wrap_buffer(DstBuffer *x);
-DstValue dst_wrap_function(DstFunction *x);
-DstValue dst_wrap_cfunction(DstCFunction x);
-DstValue dst_wrap_table(DstTable *x);
-DstValue dst_wrap_userdata(void *x);
-DstValue dst_wrap_funcenv(DstFuncEnv *x);
-DstValue dst_wrap_funcdef(DstFuncDef *x);
-
-/* Check data from arguments */
-int dst_check_nil(Dst *vm, uint32_t i);
-int dst_check_real(Dst *vm, uint32_t i);
-int dst_check_integer(Dst *vm, uint32_t i);
-int dst_check_boolean(Dst *vm, uint32_t i);
-int dst_check_string(Dst *vm, uint32_t i);
-int dst_check_symbol(Dst *vm, uint32_t i);
-int dst_check_array(Dst *vm, uint32_t i);
-int dst_check_tuple(Dst *vm, uint32_t i);
-int dst_check_struct(Dst *vm, uint32_t i);
-int dst_check_thread(Dst *vm, uint32_t i);
-int dst_check_buffer(Dst *vm, uint32_t i);
-int dst_check_function(Dst *vm, uint32_t i);
-int dst_check_cfunction(Dst *vm, uint32_t i);
-int dst_check_table(Dst *vm, uint32_t i);
-int dst_check_funcenv(Dst *vm, uint32_t i);
-int dst_check_funcdef(Dst *vm, uint32_t i);
-void dst_check_userdata(Dst *vm, uint32_t i;
+/****/
+/* Helper functions */
+/****/
+uint32_t dst_startrange(int64_t index, uint32_t modulo);
+uint32_t dst_endrange(int64_t index, uint32_t modulo);
+int64_t dst_normalize_index(Dst *vm, int64_t index);
 
 #endif /* DST_INTERNAL_H_defined */

core/module.c

@@ -1,101 +0,0 @@
-/*
-* Copyright (c) 2017 Calvin Rose
-*
-* Permission is hereby granted, free of charge, to any person obtaining a copy
-* of this software and associated documentation files (the "Software"), to
-* deal in the Software without restriction, including without limitation the
-* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-* sell copies of the Software, and to permit persons to whom the Software is
-* furnished to do so, subject to the following conditions:
-*
-* The above copyright notice and this permission notice shall be included in
-* all copies or substantial portions of the Software.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-* IN THE SOFTWARE.
-*/
-
-#include "internal.h"
-
-static void dst_cmodule_register(Dst *vm, const char *name, const DstModuleItem *mod) {
-    uint32_t startLength;
-    DstBuffer *buffer = dst_buffer(vm, 10);
-    dst_buffer_append_cstring(vm, buffer, name);
-    dst_buffer_push(vm, buffer, '.');
-    startLength = buffer->count;
-    while (mod->name != NULL) {
-        DstValue key;
-        buffer->count = startLength;
-        dst_buffer_append_cstring(vm, buffer, mod->name);
-        key = dst_wrap_symbol(dst_buffer_to_string(vm, buffer));
-        dst_table_put(vm, vm->registry, key, dst_wrap_cfunction(mod->data));
-        dst_table_put(vm, vm->registry, dst_wrap_cfunction(mod->data), key);
-        mod++;
-    }
-}
-
-static DstValue dst_cmodule_table(Dst *vm, const DstModuleItem *mod) {
-    DstTable *module = dst_table(vm, 10);
-    while (mod->name != NULL) {
-        DstValue key = dst_string_cvs(vm, mod->name);
-        dst_table_put(vm, module, key, dst_wrap_cfunction(mod->data));
-        mod++;
-    }
-    return dst_wrap_table(module);
-}
-
-static DstValue dst_cmodule_struct(Dst *vm, const DstModuleItem *mod) {
-    uint32_t count = 0;
-    const DstModuleItem *m = mod;
-    DstValue *st;
-    while (m->name != NULL) {
-        ++count;
-        ++m;
-    }
-    st = dst_struct_begin(vm, count);
-    m = mod;
-    while (m->name != NULL) {
-        dst_struct_put(st,
-                dst_string_cvs(vm, m->name),
-                dst_wrap_cfunction(m->data));
-        ++m;
-    }
-    return dst_wrap_struct(dst_struct_end(vm, st));
-}
-
-void dst_module(Dst *vm, const char *packagename, const DstModuleItem *mod) {
-    dst_table_put(vm, vm->modules, dst_string_cvs(vm, packagename), dst_cmodule_struct(vm, mod));
-    dst_cmodule_register(vm, packagename, mod);
-}
-
-void dst_module_mutable(Dst *vm, const char *packagename, const DstModuleItem *mod) {
-    dst_table_put(vm, vm->modules, dst_string_cvs(vm, packagename), dst_cmodule_table(vm, mod));
-    dst_cmodule_register(vm, packagename, mod);
-}
-
-void dst_module_put(Dst *vm, const char *packagename, const char *name, DstValue v) {
-    DstValue modtable = dst_table_get(vm->modules, dst_string_cvs(vm, packagename));
-    if (modtable.type == DST_TABLE) {
-        DstTable *table = modtable.data.table;
-        if (v.type == DST_CFUNCTION) {
-            DstValue key;
-            DstBuffer *buffer = dst_buffer(vm, 10);
-            dst_buffer_append_cstring(vm, buffer, packagename);
-            dst_buffer_push(vm, buffer, '.');
-            dst_buffer_append_cstring(vm, buffer, name);
-            key = dst_wrap_string(dst_buffer_to_string(vm, buffer));
-            dst_table_put(vm, vm->registry, key, v);
-            dst_table_put(vm, vm->registry, v, key);
-        }
-        dst_table_put(vm, table, dst_string_cvs(vm, name), v);
-    }
-}
-
-DstValue dst_module_get(Dst *vm, const char *packagename) {
-    return dst_table_get(vm->modules, dst_string_cvs(vm, packagename));
-}

core/parse.c

@@ -21,7 +21,88 @@
 */
 
 #include "internal.h"
-#include "bootstrap.h"
+
+/* Get an integer power of 10 */
+static double exp10(int power) {
+    if (power == 0) return 1;
+    if (power > 0) {
+        double result = 10;
+        int currentPower = 1;
+        while (currentPower * 2 <= power) {
+            result = result * result;
+            currentPower *= 2;
+        }
+        return result * exp10(power - currentPower);
+    } else {
+        return 1 / exp10(-power);
+    }
+}
+
+/* Read an integer */
+static int read_integer(const uint8_t *string, const uint8_t *end, int64_t *ret) {
+    int sign = 1, x = 0;
+    int64_t accum = 0;
+    if (*string == '-') {
+        sign = -1;
+        ++string;
+    } else if (*string == '+') {
+        ++string;
+    }
+    if (string >= end) return 0;
+    while (string < end) {
+        x = *string;
+        if (x < '0' || x > '9') return 0;
+        x -= '0';
+        accum = accum * 10 + x;
+        ++string;
+    }
+    *ret = accum * sign;
+    return 1;
+}
+
+/* Read a real from a string. Returns if successfuly
+ * parsed a real from the enitre input string.
+ * If returned 1, output is int ret.*/
+static int read_real(const uint8_t *string, const uint8_t *end, double *ret, int forceInt) {
+    int sign = 1, x = 0;
+    double accum = 0, exp = 1, place = 1;
+    /* Check the sign */
+    if (*string == '-') {
+        sign = -1;
+        ++string;
+    } else if (*string == '+') {
+        ++string;
+    }
+    if (string >= end) return 0;
+    while (string < end) {
+        if (*string == '.' && !forceInt) {
+            place = 0.1;
+        } else if (!forceInt && (*string == 'e' || *string == 'E')) {
+            /* Read the exponent */
+            ++string;
+            if (string >= end) return 0;
+            if (!read_real(string, end, &exp, 1))
+                return 0;
+            exp = exp10(exp);
+            break;
+        } else {
+            x = *string;
+            if (x < '0' || x > '9') return 0;
+            x -= '0';
+            if (place < 1) {
+                accum += x * place;
+                place *= 0.1;
+            } else {
+                accum *= 10;
+                accum += x;
+            }
+        }
+        ++string;
+    }
+    *ret = accum * sign * exp;
+    return 1;
+}
+
 
 /* Checks if a string slice is equal to a string constant */
 static int check_str_const(const char *ref, const uint8_t *start, const uint8_t *end) {
@@ -34,19 +115,25 @@ static int check_str_const(const char *ref, const uint8_t *start, const uint8_t
 }
 
 /* Quote a value */
-static DstValue quote(Dst *vm, DstValue x) {
-    DstValue *tuple = dst_tuple_begin(vm, 2);
-    tuple[0] = dst_string_cvs(vm, "quote");
-    tuple[1] = x;
-    return dst_wrap_tuple(dst_tuple_end(vm, tuple));
+static void quote(Dst *vm) {
+    dst_cstring(vm, "quote");
+    dst_hoist(vm, -2);
+    dst_tuple(vm, 2);
 }
 
 /* Check if a character is whitespace */
 static int is_whitespace(uint8_t c) {
-    return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\0' || c == ',';
+    return c == ' ' 
+        || c == '\t'
+        || c == '\n'
+        || c == '\r'
+        || c == '\0'
+        || c == ',';
 }
 
 /* Check if a character is a valid symbol character */
+/* TODO - wlloe utf8 - shouldn't be difficult, err on side
+ * of inclusivity */
 static int is_symbol_char(uint8_t c) {
     if (c >= 'a' && c <= 'z') return 1;
     if (c >= 'A' && c <= 'Z') return 1;
@@ -76,12 +163,13 @@ static int to_hex(uint8_t c) {
 typedef struct {
     Dst *vm;
     const uint8_t *end;
-    const char **errmsg;
+    const char *errmsg;
+    int64_t sourcemap;
     int status;
 } ParseArgs;
 
-/* Entry point of recursive descent parser */
-static const uint8_t *parse(
+/* Entry point of the recursive descent parser */
+static const uint8_t *parse_recur(
     ParseArgs *args,
     const uint8_t *src,
     uint32_t recur) {
@@ -89,7 +177,6 @@ static const uint8_t *parse(
     Dst *vm = args->vm;
     const uint8_t *end = args->end;
     uint32_t qcount = 0;
-    uint32_t retindex = dst_args(vm);
 
     /* Prevent stack overflow */
     if (recur == 0) goto too_much_recur;
@@ -110,29 +197,28 @@ static const uint8_t *parse(
 
         /* Numbers, symbols, simple literals */
         default: {
-            DstReal real;
-            DstInteger integer;
+            double real;
+            int64_t integer;
             const uint8_t *tokenend = src;
             if (!is_symbol_char(*src)) goto unexpected_character;
-            dst_setsize(vm, retindex + 1);
             while (tokenend < end && is_symbol_char(*tokenend))
                 tokenend++;
             if (tokenend >= end) goto unexpected_eos;
-            if (dst_read_integer(src, tokenend, &integer)) {
-                dst_set_integer(vm, retindex, integer);
-            } else if (dst_read_real(src, tokenend, &real, 0)) {
-                dst_set_real(vm, retindex, real);
+            if (read_integer(src, tokenend, &integer)) {
+                dst_integer(vm, integer);
+            } else if (read_real(src, tokenend, &real, 0)) {
+                dst_real(vm, real);
             } else if (check_str_const("nil", src, tokenend)) {
-                dst_nil(vm, retindex);
+                dst_nil(vm);
             } else if (check_str_const("false", src, tokenend)) {
-                dst_false(vm, retindex);
+                dst_false(vm);
             } else if (check_str_const("true", src, tokenend)) {
-                dst_true(vm, retindex);
+                dst_true(vm);
             } else {
                 if (*src >= '0' && *src <= '9') {
                     goto sym_nodigits;
                 } else {
-                    dst_symbol(vm, retindex, src, tokenend - src);
+                    dst_symbol(vm, src, tokenend - src);
                 }
             }
             src = tokenend;
@@ -141,11 +227,10 @@ static const uint8_t *parse(
 
         case ':': {
             const uint8_t *tokenend = ++src;
-            dst_setsize(vm, retindex + 1);
             while (tokenend < end && is_symbol_char(*tokenend))
                 tokenend++;
             if (tokenend >= end) goto unexpected_eos;
-            dst_string(vm, retindex, src, tokenend - src);
+            dst_string(vm, src, tokenend - src);
             src = tokenend;
             break;
         }
@@ -153,28 +238,30 @@ static const uint8_t *parse(
         /* String literals */
         case '"': {
             const uint8_t *strend = ++src;
+            const uint8_t *strstart = src;
             uint32_t len = 0;
             int containsEscape = 0;
             /* Preprocess string to check for escapes and string end */
             while (strend < end && *strend != '"') {
                 len++;
                 if (*strend++ == '\\') {
-                    constainsEscape = 1;
+                    containsEscape = 1;
                     if (strend >= end) goto unexpected_eos;
                     if (*strend == 'h') {
                         strend += 2;
                         if (strend >= end) goto unexpected_eos;
+                    } else {
+                        strend++;
                     }
                 }
             }
             if (containsEscape) {
                 uint8_t *buf = dst_string_begin(vm, len);
                 uint8_t *write = buf;
-                const uint8_t *scan = src;
-                while (scan < strend) {
-                    if (*scan == '\\') {
-                        scan++;
-                        switch (*++scan) {
+                while (src < strend) {
+                    if (*src == '\\') {
+                        src++;
+                        switch (*++src) {
                             case 'n': *write++ = '\n'; break;
                             case 'r': *write++ = '\r'; break;
                             case 't': *write++ = '\t'; break;
@@ -185,69 +272,66 @@ static const uint8_t *parse(
                             case 'z': *write++ = '\0'; break;
                             case 'e': *write++ = 27; break;
                             case 'h': {
-                                int d1 = to_hex(scan[0]);
-                                int d2 = to_hex(scan[1]);
+                                int d1 = to_hex(*src++);
+                                int d2 = to_hex(*src++);
                                 if (d1 < 0 || d2 < 0) goto invalid_hex;
-                                *write = 16 * d1 + d2;
+                                *write++ = 16 * d1 + d2;
                                 break;
                             }
                             default:
                                 goto unknown_strescape;
                         }
                     } else {
-                        *write++ = *scan++;
+                        *write++ = *src++;
                     }
                 }
-                dst_string_end(vm, retindex, buf);
+                dst_string_end(vm, buf);
             } else {
-                dst_string(vm, retindex, src, strend - src);
-            }
+                dst_string(vm, strstart, strend - strstart);
                 src = strend + 1;
+            }
             break;
         }
 
         /* Data Structure literals */
-        case: '(':
-        case: '[':
-        case: '{': {
+        case '(':
+        case '[':
+        case '{': {
+            uint32_t n = 0;
             uint8_t close;
-            uint32_t tmpindex;
             switch (*src++) {
-                case '(': close = ')'; break;
                 case '[': close = ']'; break;
                 case '{': close = '}'; break;
                 default: close = ')'; break;
             }
             /* Recursively parse inside literal */
             while (*src != close) {
-                src = parse(args, src, recur - 1);
-                if (*(args->errmsg) || !src) return src;
+                src = parse_recur(args, src, recur - 1);
+                if (args->errmsg || !src) return src;
+                n++;
+                /* Trim trailing whitespace */
+                while (src < end && (is_whitespace(*src)))
+                    ++src;
             }
             src++;
-            tmpindex = dst_args(vm);
-            dst_push_space(vm, 1);
             switch (close) {
                 case ')':
-                    dst_tuple_n(vm, tmpindex, retindex, tmpindex - retindex);
+                    dst_tuple(vm, n);
                     break;
                 case ']':
-                    dst_array_n(vm, tmpindex, retindex, tmpindex - retindex);
+                    dst_arrayn(vm, n);
                     break;
                 case '}':
-                    if ((tmpindex - retindex) % 2) goto struct_oddargs;
-                    dst_struct_n(vm, tmpindex, retindex, tmpindex - retindex);
+                    if (n & 1) goto struct_oddargs;
+                    dst_struct(vm, n/2);
                     break;
             }
-            dst_move(vm, retindex, tmpindex);
-            dst_setsize(vm, retindex + 1);
             break;
         }
     }
 
     /* Quote the returned value qcount times */
-    while (qcount--) {
-        dst_set_arg(vm, retindex, quote(vm, dst_arg(vm, retindex)));
-    }
+    while (qcount--) quote(vm);
 
     /* Return the new source position for further calls */
     return src;
@@ -255,77 +339,89 @@ static const uint8_t *parse(
     /* Errors below */
 
     unexpected_eos:
-    *(args->errmsg) = "unexpected end of source";
+    args->errmsg = "unexpected end of source";
     args->status = PARSE_UNEXPECTED_EOS;
     return NULL;
 
     unexpected_character:
-    *(args->errmsg) = "unexpected character";
+    args->errmsg = "unexpected character";
     args->status = PARSE_ERROR;
     return src;
 
     sym_nodigits:
-    *(args->errmsg) = "symbols cannot start with digits";
+    args->errmsg = "symbols cannot start with digits";
     args->status = PARSE_ERROR;
     return src;
 
     struct_oddargs:
-    *(args->errmsg) = "struct literal needs an even number of arguments";
+    args->errmsg = "struct literal needs an even number of arguments";
     args->status = PARSE_ERROR;
     return src;
 
     unknown_strescape:
-    *(args->errmsg) = "unknown string escape sequence";
+    args->errmsg = "unknown string escape sequence";
     args->status = PARSE_ERROR;
     return src;
 
     invalid_hex:
-    *(args->errmsg) = "invalid hex escape in string";
+    args->errmsg = "invalid hex escape in string";
     args->status = PARSE_ERROR;
     return src;
 
     too_much_recur:
-    *(args->errmsg) = "recursed too deeply in parsing";
+    args->errmsg = "recursed too deeply in parsing";
     args->status = PARSE_ERROR;
     return src;
 }
 
 /* Parse an array of bytes */
-int dst_parseb(Dst *vm, uint32_t dest, const uint8_t *src, const uint8_t **newsrc, uint32_t len) {
+int dst_parseb(Dst *vm, const uint8_t *src, const uint8_t **newsrc, uint32_t len) {
     ParseArgs args;
-    uint32_t toploc = dst_args(vm);
-    const uint8_t *srcrest;
+    uint32_t nargs;
+
+    /* Create a source map */
+    dst_table(vm, 10);
+
+    nargs = dst_stacksize(vm);
 
     args.vm = vm;
     args.status = PARSE_OK;
     args.end = src + len;
     args.errmsg = NULL;
+    args.sourcemap = dst_normalize_index(vm, -1);
+
+    src = parse_recur(&args, src, 2048);
+    if (newsrc) *newsrc = src;
 
-    srcrest = parse(&args, src, 2048) || src;
-    if (newsrc) {
-        *newsrc = srcrest;
-    }
     if (args.errmsg) {
-        /* Error */
-        dst_cstring(vm, dest, args.errmsg);
-    } else {
-        /* Success */
-        dst_move(vm, dest, toploc);
+        /* Unwind the stack */
+        dst_trimstack(vm, nargs);
+        dst_cstring(vm, args.errmsg);
     }
-    dst_setsize(vm, toploc);
+
     return args.status;
 }
 
 /* Parse a c string */
-int dst_parsec(Dst *vm, uint32_t dest, const char *src) {
+int dst_parsec(Dst *vm, const char *src, const char **newsrc) {
     uint32_t len = 0;
+    const uint8_t *ns = NULL;
+    int status;
     while (src[len]) ++len;
-    return dst_parseb(vm, dest, (const uint8_t *)src, NULL, len);
+    status = dst_parseb(vm, (const uint8_t *)src, &ns, len);
+    if (newsrc) {
+        *newsrc = (const char *)ns;
+    }
+    return status;
 }
 
 /* Parse a DST char seq (Buffer, String, Symbol) */
-int dst_parse(Dst *vm, uint32_t dest, uint32_t src) {
+int dst_parse(Dst *vm)  {
+    int status;
     uint32_t len;
-    const uint8_t *bytes = dst_bytes(vm, src, &len);
-    return dst_parseb(vm, dest, bytes, NULL, len);
+    const uint8_t *bytes = dst_bytes(vm, -1, &len);
+    status = dst_parseb(vm, bytes, NULL, len);
+    dst_swap(vm, -1, -2);
+    dst_pop(vm);
+    return status;
 }

core/serialize.c

@@ -24,7 +24,7 @@
 #include "internal.h"
 
 /**
- * Data format
+ * Data format v1
  *
  * Types:
  *
@@ -32,7 +32,7 @@
  * Byte 201: Nil
  * Byte 202: True
  * Byte 203: False
- * Byte 204: Number  - double format
+ * Byte 204: Number  - IEEE 754 double format
  * Byte 205: String  - [u32 length]*[u8... characters]
  * Byte 206: Struct  - [u32 length]*2*[value... kvs]
  * Byte 207: Buffer  - [u32 capacity][u32 length]*[u8... characters]
@@ -52,703 +52,9 @@
  * Byte 217: Ref     - [u32 id]
  * Byte 218: Integer - [i64 value]
  * Byte 219: Symbol  - [u32 length]*[u8... characters]
+ *
+ * This data format needs both a serialization function and deserialization function
+ * written in C, as this will load embeded and precompiled programs into the VM, including
+ * the self hosted parser abd compiler. It is therefor important that it is correct,
+ * does not leak memory, and is fast.
  */
-
-/* Error at buffer end */
-static const char UEB[] = "unexpected end of buffer";
-
-/* Read 4 bytes as an unsigned integer */
-static uint32_t bytes2u32(const uint8_t *bytes) {
-    union {
-        uint8_t bytes[4];
-        uint32_t u32;
-    } u;
-    dst_memcpy(u.bytes, bytes, 4 * sizeof(uint8_t));
-    return u.u32;
-}
-
-/* Read 2 bytes as unsigned short */
-static uint16_t bytes2u16(const uint8_t *bytes) {
-    union {
-        uint8_t bytes[2];
-        uint16_t u16;
-    } u;
-    dst_memcpy(u.bytes, bytes, 2 * sizeof(uint8_t));
-    return u.u16;
-}
-
-/* Read 8 bytes as a double */
-static double bytes2dbl(const uint8_t *bytes) {
-    union {
-        uint8_t bytes[8];
-        double dbl;
-    } u;
-    dst_memcpy(u.bytes, bytes, 8 * sizeof(uint8_t));
-    return u.dbl;
-}
-
-/* Read 8 bytes as a integer */
-static int64_t bytes2int(const uint8_t *bytes) {
-    union {
-        uint8_t bytes[8];
-        int64_t i;
-    } u;
-    dst_memcpy(u.bytes, bytes, 8 * sizeof(uint8_t));
-    return u.i;
-}
-
-/* Read a string and turn it into a dst value. Returns
- * an error message if there is an error message during
- * deserialization. If successful, the resulting value is
- * passed by reference. */
-static const char *dst_deserialize_impl(
-        Dst *vm,
-        const uint8_t *data,
-        const uint8_t *end,
-        const uint8_t **newData,
-        DstArray *visited,
-        DstValue *out,
-        int depth) {
-
-    DstValue ret;
-    ret.type = DST_NIL;
-    DstValue *buffer;
-    uint32_t length, i;
-    const char *err;
-
-    /* Handle errors as needed */
-#define deser_error(e) return (e)
-
-    /* Assertions */
-#define deser_assert(c, e) do{if(!(c))deser_error(e);}while(0)
-
-    /* Assert enough buffer */
-#define deser_datacheck(len) do{if (end < (data + len)) deser_error(UEB);}while(0)
-
-    /* Check for enough space to read uint32_t */
-#define read_u32(out) do{deser_datacheck(4); (out)=bytes2u32(data); data += 4; }while(0)
-#define read_u16(out) do{deser_datacheck(2); (out)=bytes2u16(data); data += 2; }while(0)
-#define read_dbl(out) do{deser_datacheck(8); (out)=bytes2dbl(data); data += 8; }while(0)
-#define read_i64(out) do{deser_datacheck(8); (out)=bytes2int(data); data += 8; }while(0)
-
-    /* Check if we have recursed too deeply */
-    if (--depth == 0) {
-        return "deserialize recursed too deeply";
-    }
-
-    /* Check enough buffer left to read one byte */
-    if (data >= end) deser_error(UEB);
-
-    /* Small integer */
-    if (*data < 201) {
-        ret.type = DST_INTEGER;
-        ret.data.integer = *data - 100;
-        *newData = data + 1;
-        *out = ret;
-        return NULL;
-    }
-
-    /* Main switch for types */
-    switch (*data++) {
-
-        default:
-            return "unable to deserialize";
-
-        case 201: /* Nil */
-            ret.type = DST_NIL;
-            break;
-
-        case 202: /* True */
-            ret.type = DST_BOOLEAN;
-            ret.data.boolean = 1;
-            break;
-
-        case 203: /* False */
-            ret.type = DST_BOOLEAN;
-            ret.data.boolean = 0;
-            break;
-
-        case 204: /* Long number (double) */
-            ret.type = DST_REAL;
-            read_dbl(ret.data.real);
-            break;
-
-        case 205: /* String */
-        case 219: /* Symbol */
-            ret.type = data[-1] == 205 ? DST_STRING : DST_SYMBOL;
-            read_u32(length);
-            deser_datacheck(length);
-            ret.data.string = dst_string_b(vm, data, length);
-            data += length;
-            dst_array_push(vm, visited, ret);
-            break;
-
-        case 206: /* Struct */
-            ret.type = DST_STRUCT;
-            read_u32(length);
-            buffer = dst_struct_begin(vm, length);
-            for (i = 0; i < length; ++i) {
-                DstValue k, v;
-                if ((err = dst_deserialize_impl(vm, data, end, &data, visited, &k, depth)))
-                    return err;
-                if ((err = dst_deserialize_impl(vm, data, end, &data, visited, &v, depth)))
-                    return err;
-                dst_struct_put(buffer, k, v);
-            }
-            ret.data.st = dst_struct_end(vm, buffer);
-            dst_array_push(vm, visited, ret);
-            break;
-
-        case 207: /* Buffer */
-            {
-                uint32_t cap;
-                ret.type = DST_BYTEBUFFER;
-                read_u32(cap);
-                read_u32(length);
-                deser_datacheck(length);
-                ret.data.buffer = dst_alloc(vm, sizeof(DstBuffer));
-                ret.data.buffer->data = dst_alloc(vm, cap);
-                dst_memcpy(ret.data.buffer->data, data, length);
-                ret.data.buffer->count = length;
-                ret.data.buffer->capacity = cap;
-                dst_array_push(vm, visited, ret);
-            }
-            break;
-
-        case 208: /* Array */
-            ret.type = DST_ARRAY;
-            read_u32(length);
-            buffer = dst_alloc(vm, length * sizeof(DstValue));
-            ret.data.array = dst_alloc(vm, sizeof(DstArray));
-            ret.data.array->data = buffer;
-            ret.data.array->count = length;
-            ret.data.array->capacity = length;
-            dst_array_push(vm, visited, ret);
-            for (i = 0; i < length; ++i)
-                if ((err = dst_deserialize_impl(vm, data, end, &data, visited, buffer + i, depth)))
-                    return err;
-            break;
-
-        case 209: /* Tuple */
-            ret.type = DST_TUPLE;
-            read_u32(length);
-            buffer = dst_tuple_begin(vm, length);
-            for (i = 0; i < length; ++i)
-                if ((err = dst_deserialize_impl(vm, data, end, &data, visited, buffer + i, depth)))
-                    return err;
-            ret.type = DST_TUPLE;
-            ret.data.tuple = dst_tuple_end(vm, buffer);
-            dst_array_push(vm, visited, ret);
-            break;
-
-        case 210: /* Thread */
-            {
-                DstThread *t;
-                DstValue *stack;
-                uint16_t prevsize = 0;
-                uint8_t statusbyte;
-                t = dst_thread(vm, dst_wrap_nil(), 64);
-                ret = dst_wrap_thread(t);
-                dst_array_push(vm, visited, ret);
-                err = dst_deserialize_impl(vm, data, end, &data, visited, &ret, depth);
-                if (err != NULL) return err;
-                if (ret.type == DST_NIL) {
-                    t->parent = NULL;
-                } else if (ret.type == DST_THREAD) {
-                    t->parent = ret.data.thread;
-                } else {
-                    return "expected thread parent to be thread";
-                }
-                deser_assert(data < end, UEB);
-                statusbyte = *data++;
-                read_u32(length);
-                /* Set status */
-                t->status = statusbyte % 4;
-                /* Add frames */
-                for (i = 0; i < length; ++i) {
-                    DstValue callee, env;
-                    uint32_t pcoffset;
-                    uint16_t ret, args, size, j;
-                    /* Read the stack */
-                    err = dst_deserialize_impl(vm, data, end, &data, visited, &callee, depth);
-                    if (err != NULL) return err;
-                    err = dst_deserialize_impl(vm, data, end, &data, visited, &env, depth);
-                    if (err != NULL) return err;
-                    if (env.type != DST_FUNCENV && env.type != DST_NIL)
-                        return "expected funcenv in stackframe";
-                    /* Create a new frame */
-                    if (i > 0)
-                        dst_thread_beginframe(vm, t, dst_wrap_nil(), 0);
-                    read_u32(pcoffset);
-                    read_u32(ret);
-                    read_u32(args);
-                    read_u32(size);
-                    /* Set up the stack */
-                    stack = dst_thread_stack(t);
-                    if (callee.type == DST_FUNCTION) {
-                        dst_frame_pc(stack) = callee.data.function->def->byteCode + pcoffset;
-                    }
-                    dst_frame_ret(stack) = ret;
-                    dst_frame_args(stack) = args;
-                    dst_frame_size(stack) = size;
-                    dst_frame_prevsize(stack) = prevsize;
-                    dst_frame_callee(stack) = callee;
-                    if (env.type == DST_NIL)
-                        dst_frame_env(stack) = NULL;
-                    else
-                        dst_frame_env(stack) = env.data.env;
-                    prevsize = size;
-                    /* Push stack args */
-                    for (j = 0; j < size; ++j) {
-                        DstValue temp;
-                        err = dst_deserialize_impl(vm, data, end, &data, visited, &temp, depth);
-                        dst_thread_push(vm, t, temp);
-                    }
-                }
-            }
-            break;
-
-        case 211: /* Table */
-            read_u32(length);
-            ret = dst_wrap_table(dst_table(vm, 2 * length));
-            dst_array_push(vm, visited, ret);
-            for (i = 0; i < length; ++i) {
-                DstValue key, value;
-                err = dst_deserialize_impl(vm, data, end, &data, visited, &key, depth);
-                if (err != NULL) return err;
-                err = dst_deserialize_impl(vm, data, end, &data, visited, &value, depth);
-                if (err != NULL) return err;
-                dst_table_put(vm, ret.data.table, key, value);
-            }
-            break;
-
-        case 212: /* Funcdef */
-            {
-                DstFuncDef *def;
-                uint32_t locals, arity, literalsLen, byteCodeLen, flags;
-                read_u32(locals);
-                read_u32(arity);
-                read_u32(flags);
-                read_u32(literalsLen);
-                def = dst_alloc(vm, sizeof(DstFuncDef));
-                ret = dst_wrap_funcdef(def);
-                dst_array_push(vm, visited, ret);
-                def->locals = locals;
-                def->arity = arity;
-                def->flags = flags;
-                def->literalsLen = literalsLen;
-                if (literalsLen > 0) {
-                    def->literals = dst_alloc(vm, literalsLen * sizeof(DstValue));
-                } else {
-                    def->literals = NULL;
-                }
-                for (i = 0; i < literalsLen; ++i) {
-                    err = dst_deserialize_impl(vm, data, end, &data, visited, def->literals + i, depth);
-                    if (err != NULL) return err;
-                }
-                read_u32(byteCodeLen);
-                deser_datacheck(byteCodeLen * sizeof(uint16_t));
-                def->byteCode = dst_alloc(vm, byteCodeLen * sizeof(uint16_t));
-                def->byteCodeLen = byteCodeLen;
-                for (i = 0; i < byteCodeLen; ++i) {
-                    read_u16(def->byteCode[i]);
-                }
-            }
-            break;
-
-        case 213: /* Funcenv */
-            {
-                DstValue thread;
-                ret.type = DST_FUNCENV;
-                ret.data.env = dst_alloc(vm, sizeof(DstFuncEnv));
-                dst_array_push(vm, visited, ret);
-                err = dst_deserialize_impl(vm, data, end, &data, visited, &thread, depth);
-                if (err != NULL) return err;
-                read_u32(length);
-                ret.data.env->stackOffset = length;
-                if (thread.type == DST_THREAD) {
-                    ret.data.env->thread = thread.data.thread;
-                } else {
-                    ret.data.env->thread = NULL;
-                    ret.data.env->values = dst_alloc(vm, sizeof(DstValue) * length);
-                    for (i = 0; i < length; ++i) {
-                        DstValue item;
-                        err = dst_deserialize_impl(vm, data, end, &data, visited, &item, depth);
-                        if (err != NULL) return err;
-                        ret.data.env->values[i] = item;
-                    }
-                }
-            }
-            break;
-
-        case 214: /* Function */
-            {
-                DstValue parent, def, env;
-                ret.type = DST_FUNCTION;
-                ret.data.function = dst_alloc(vm, sizeof(DstFunction));
-                dst_array_push(vm, visited, ret);
-                err = dst_deserialize_impl(vm, data, end, &data, visited, &parent, depth);
-                if (err != NULL) return err;
-                err = dst_deserialize_impl(vm, data, end, &data, visited, &env, depth);
-                if (err != NULL) return err;
-                err = dst_deserialize_impl(vm, data, end, &data, visited, &def, depth);
-                if (err != NULL) return err;
-                if (parent.type == DST_NIL) {
-                    ret.data.function->parent = NULL;
-                } else if (parent.type == DST_FUNCTION) {
-                    ret.data.function->parent = parent.data.function;
-                } else {
-                    deser_error("expected function");
-                }
-                deser_assert(def.type == DST_FUNCDEF, "expected funcdef");
-                ret.data.function->def = def.data.def;
-                if (env.type == DST_NIL) {
-                    ret.data.function->env = NULL;
-                } else {
-                    deser_assert(env.type == DST_FUNCENV, "expected funcenv");
-                    ret.data.function->env = env.data.env;
-                }
-            }
-            break;
-
-        case 215: /* LUdata */
-            {
-                /* TODO enable deserialization of userdata through registration
-                 * to names in vm. */
-            }
-            break;
-
-        case 216: /* C function */
-            {
-                DstValue id;
-                read_u32(length);
-                deser_datacheck(length);
-                id = dst_wrap_string(dst_string_b(vm, data, length));
-                data += length;
-                ret = dst_table_get(vm->registry, id);
-                if (ret.type != DST_CFUNCTION) {
-                    deser_error("unabled to deserialize c function");
-                }
-                break;
-            }
-            break;
-
-        case 217: /* Reference */
-            read_u32(length);
-            deser_assert(visited->count > length, "invalid reference");
-            ret = visited->data[length];
-            break;
-
-        case 218: /* Integer */
-            ret.type = DST_INTEGER;
-            read_i64(ret.data.integer);
-            break;
-    }
-    *out = ret;
-    *newData = data;
-    return NULL;
-}
-
-/* Load a value from data */
-const char *dst_deserialize_internal(
-        Dst *vm,
-        const uint8_t *data,
-        uint32_t len,
-        DstValue *out,
-        const uint8_t **nextData) {
-    DstValue ret;
-    const char *err;
-    DstArray *visited = dst_array(vm, 10);
-    err = dst_deserialize_impl(vm, data, data + len, nextData, visited, &ret, DST_RECURSION_GUARD);
-    if (err != NULL) return err;
-    *out = ret;
-    return NULL;
-}
-
-/* Allow appending other types to buffers */
-BUFFER_DEFINE(real, DstReal)
-BUFFER_DEFINE(integer, DstInteger)
-BUFFER_DEFINE(u32, uint32_t)
-BUFFER_DEFINE(u16, uint16_t)
-
-/* Serialize a value and write to a buffer. Returns possible
- * error messages. */
-static const char *dst_serialize_impl(
-        Dst *vm,
-        DstBuffer *buffer,
-        DstTable *visited,
-        uint32_t *nextId,
-        DstValue x,
-        int depth) {
-
-    uint32_t i, count;
-    const char *err;
-    DstValue check;
-
-#define write_byte(b) dst_buffer_push(vm, buffer, (b))
-#define write_u32(b) dst_buffer_push_u32(vm, buffer, (b))
-#define write_u16(b) dst_buffer_push_u16(vm, buffer, (b))
-#define write_dbl(b) dst_buffer_push_real(vm, buffer, (b))
-#define write_int(b) dst_buffer_push_integer(vm, buffer, (b))
-    /*printf("Type: %d\n", x.type);*/
-
-    /* Check if we have gone too deep */
-    if (--depth == 0) {
-        return "serialize recursed too deeply";
-    }
-
-    /* Check non reference types - if successful, return NULL */
-    switch (x.type) {
-        case DST_USERDATA:
-        case DST_NIL:
-            write_byte(201);
-            return NULL;
-        case DST_BOOLEAN:
-            write_byte(x.data.boolean ? 202 : 203);
-            return NULL;
-        case DST_REAL:
-            write_byte(204);
-            write_dbl(x.data.real);
-            return NULL;
-        case DST_INTEGER:
-            if (x.data.integer <= 100 && x.data.integer >= -100) {
-                write_byte(x.data.integer + 100);
-            } else {
-                write_byte(218);
-                write_int(x.data.integer);
-            }
-            return NULL;
-        default:
-            break;
-    }
-
-    /* Check if already seen - if so, use reference */
-    check = dst_table_get(visited, x);
-    if (check.type == DST_INTEGER) {
-        write_byte(217);
-        write_u32((uint32_t) check.data.integer);
-        return NULL;
-    }
-
-    /* Check tuples and structs before other reference types.
-     * They are immutable, and thus cannot be referenced by other values
-     * until they are fully constructed. This creates some strange behavior
-     * if they are treated like other reference types because they cannot
-     * be added to the visited table before recursing into serializing their
-     * arguments */
-    if (x.type == DST_STRUCT || x.type == DST_TUPLE) {
-        if (x.type == DST_STRUCT) {
-            const DstValue *data;
-            write_byte(206);
-            dst_hashtable_view(x, &data, &count);
-            write_u32(dst_struct_length(x.data.st));
-            for (i = 0; i < count; i += 2) {
-                if (data[i].type != DST_NIL) {
-                    err = dst_serialize_impl(vm, buffer, visited, nextId, data[i], depth);
-                    if (err != NULL) return err;
-                    err = dst_serialize_impl(vm, buffer, visited, nextId, data[i + 1], depth);
-                    if (err != NULL) return err;
-                }
-            }
-        } else {
-            write_byte(209);
-            count = dst_tuple_length(x.data.tuple);
-            write_u32(count);
-            for (i = 0; i < count; ++i) {
-                err = dst_serialize_impl(vm, buffer, visited, nextId, x.data.tuple[i], depth);
-                if (err != NULL) return err;
-            }
-        }
-        /* Record reference after serialization */
-        dst_table_put(vm, visited, x, dst_wrap_integer(*nextId));
-        *nextId = *nextId + 1;
-        return NULL;
-    }
-
-    /* Record reference before serialization */
-    dst_table_put(vm, visited, x, dst_wrap_integer(*nextId));
-    *nextId = *nextId + 1;
-
-    /* Check reference types */
-    switch (x.type) {
-        default:
-            return "unable to serialize type";
-
-        case DST_STRING:
-        case DST_SYMBOL:
-            write_byte(x.type == DST_STRING ? 205 : 219);
-            count = dst_string_length(x.data.string);
-            write_u32(count);
-            for (i = 0; i < count; ++i) {
-                write_byte(x.data.string[i]);
-            }
-            break;
-
-        case DST_CFUNCTION:
-            write_byte(216);
-            {
-                DstValue id = dst_table_get(vm->registry, x);
-                count = dst_string_length(id.data.string);
-                write_u32(count);
-                for (i = 0; i < count; ++i) {
-                    write_byte(id.data.string[i]);
-                }
-            }
-            break;
-
-        case DST_TABLE:
-            {
-                const DstValue *data;
-                write_byte(211);
-                dst_hashtable_view(x, &data, &count);
-                write_u32(x.data.table->count);
-                for (i = 0; i < count; i += 2) {
-                    if (data[i].type != DST_NIL) {
-                        err = dst_serialize_impl(vm, buffer, visited, nextId, data[i], depth);
-                        if (err != NULL) return err;
-                        err = dst_serialize_impl(vm, buffer, visited, nextId, data[i + 1], depth);
-                        if (err != NULL) return err;
-                    }
-                }
-            }
-            break;
-
-        case DST_BYTEBUFFER:
-            write_byte(207);
-            count = x.data.buffer->count;
-            write_u32(x.data.buffer->capacity);
-            write_u32(count);
-            for (i = 0; i < count; ++i) {
-                write_byte(x.data.buffer->data[i]);
-            }
-            break;
-
-        case DST_ARRAY:
-            write_byte(208);
-            count = x.data.array->count;
-            write_u32(count);
-            for (i = 0; i < count; ++i) {
-                err = dst_serialize_impl(vm, buffer, visited, nextId, x.data.array->data[i], depth);
-                if (err != NULL) return err;
-            }
-            break;
-
-        case DST_THREAD:
-            {
-                DstThread *t = x.data.thread;
-                const DstValue *stack = t->data + DST_FRAME_SIZE;
-                uint32_t framecount = dst_thread_countframes(t);
-                uint32_t i;
-                write_byte(210);
-                if (t->parent)
-                    err = dst_serialize_impl(vm, buffer, visited, nextId, dst_wrap_thread(t->parent), depth);
-                else
-                    err = dst_serialize_impl(vm, buffer, visited, nextId, dst_wrap_nil(), depth);
-                if (err != NULL) return err;
-                /* Write the status byte */
-                write_byte(t->status);
-                /* Write number of stack frames */
-                write_u32(framecount);
-                /* Write stack frames */
-                for (i = 0; i < framecount; ++i) {
-                    uint32_t j, size;
-                    DstValue callee = dst_frame_callee(stack);
-                    DstFuncEnv *env = dst_frame_env(stack);
-                    err = dst_serialize_impl(vm, buffer, visited, nextId, callee, depth);
-                    if (err != NULL) return err;
-                    if (env)
-                        err = dst_serialize_impl(vm, buffer, visited, nextId, dst_wrap_funcenv(env), depth);
-                    else
-                        err = dst_serialize_impl(vm, buffer, visited, nextId, dst_wrap_nil(), depth);
-                    if (err != NULL) return err;
-                    if (callee.type == DST_FUNCTION) {
-                        write_u32(dst_frame_pc(stack) - callee.data.function->def->byteCode);
-                    } else {
-                        write_u32(0);
-                    }
-                    write_u32(dst_frame_ret(stack));
-                    write_u32(dst_frame_args(stack));
-                    size = dst_frame_size(stack);
-                    write_u32(size);
-                    for (j = 0; j < size; ++j) {
-                        err = dst_serialize_impl(vm, buffer, visited, nextId, stack[j], depth);
-                        if (err != NULL) return err;
-                    }
-                    /* Next stack frame */
-                    stack += size + DST_FRAME_SIZE;
-                }
-            }
-            break;
-
-        case DST_FUNCDEF: /* Funcdef */
-            {
-                DstFuncDef *def = x.data.def;
-                write_byte(212);
-                write_u32(def->locals);
-                write_u32(def->arity);
-                write_u32(def->flags);
-                write_u32(def->literalsLen);
-                for (i = 0; i < def->literalsLen; ++i) {
-                    err = dst_serialize_impl(vm, buffer, visited, nextId, def->literals[i], depth);
-                    if (err != NULL) return err;
-                }
-                write_u32(def->byteCodeLen);
-                for (i = 0; i < def->byteCodeLen; ++i) {
-                    write_u16(def->byteCode[i]);
-                }
-            }
-            break;
-
-        case DST_FUNCENV: /* Funcenv */
-            {
-                DstFuncEnv *env = x.data.env;
-                write_byte(213);
-                if (env->thread) {
-                    err = dst_serialize_impl(vm, buffer, visited, nextId, dst_wrap_thread(env->thread), depth);
-                    if (err != NULL) return err;
-                    write_u32(env->stackOffset);
-                } else {
-                    write_byte(201); /* Write nil */
-                    write_u32(env->stackOffset);
-                    for (i = 0; i < env->stackOffset; ++i) {
-                        err = dst_serialize_impl(vm, buffer, visited, nextId, env->values[i], depth);
-                        if (err != NULL) return err;
-                    }
-                }
-            }
-            break;
-
-        case DST_FUNCTION: /* Function */
-            {
-                DstValue pv, ev, dv;
-                DstFunction *fn = x.data.function;
-                write_byte(214);
-                pv = fn->parent ? dst_wrap_function(fn->parent) : dst_wrap_nil();
-                dv = dst_wrap_funcdef(fn->def);
-                ev = fn->env ? dst_wrap_funcenv(fn->env) : dst_wrap_nil();
-                err = dst_serialize_impl(vm, buffer, visited, nextId, pv, depth);
-                if (err != NULL) return err;
-                err = dst_serialize_impl(vm, buffer, visited, nextId, ev, depth);
-                if (err != NULL) return err;
-                err = dst_serialize_impl(vm, buffer, visited, nextId, dv, depth);
-                if (err != NULL) return err;
-            }
-            break;
-    }
-
-    /* Return success */
-    return NULL;
-}
-
-/* Serialize an object */
-const char *dst_serialize_internal(Dst *vm, DstBuffer *buffer, DstValue x) {
-    uint32_t nextId = 0;
-    uint32_t oldCount = buffer->count;
-    const char *err;
-    DstTable *visited = dst_table(vm, 10);
-    err = dst_serialize_impl(vm, buffer, visited, &nextId, x, DST_RECURSION_GUARD);
-    if (err != NULL) {
-        buffer->count = oldCount;
-    }
-    return err;
-}

core/stl.c

@@ -20,7 +20,6 @@
 * IN THE SOFTWARE.
 */
 
-#include <dst/dst.h>
 #include "internal.h"
 
 static const char DST_EXPECTED_INTEGER[] = "expected integer";
@@ -49,28 +48,28 @@ static const char *types[] = {
 static DstValue nil() {
     DstValue n;
     n.type = DST_NIL;
-    n.data.integer = 0;
+    n.as.integer = 0;
     return n;
 }
 
 static DstValue integer(DstInteger i) {
     DstValue n;
     n.type = DST_INTEGER;
-    n.data.integer = i;
+    n.as.integer = i;
     return n;
 }
 
 static DstValue real(DstReal r) {
     DstValue n;
     n.type = DST_REAL;
-    n.data.real = r;
+    n.as.real = r;
     return n;
 }
 
 static DstValue boolean(int b) {
     DstValue n;
     n.type = DST_BOOLEAN;
-    n.data.boolean = b;
+    n.as.boolean = b;
     return n;
 }
 
@@ -82,16 +81,16 @@ static DstValue boolean(int b) {
 static DstValue dst_stl_binop_##name(DstValue lhs, DstValue rhs) {\
     if (lhs.type == DST_INTEGER)\
         if (rhs.type == DST_INTEGER)\
-            return integer(lhs.data.integer op rhs.data.integer);\
+            return integer(lhs.as.integer op rhs.as.integer);\
         else if (rhs.type == DST_REAL)\
-            return real(lhs.data.integer op rhs.data.real);\
+            return real(lhs.as.integer op rhs.as.real);\
         else\
             return nil();\
     else if (lhs.type == DST_REAL)\
         if (rhs.type == DST_INTEGER)\
-            return real(lhs.data.real op rhs.data.integer);\
+            return real(lhs.as.real op rhs.as.integer);\
         else if (rhs.type == DST_REAL)\
-            return real(lhs.data.real op rhs.data.real);\
+            return real(lhs.as.real op rhs.as.real);\
         else\
             return nil();\
     else\
@@ -127,7 +126,7 @@ int dst_stl_div(Dst *vm) {
     lhs = dst_arg(vm, 0);
     for (j = 1; j < count; ++j) {
         rhs = dst_arg(vm, j);
-        if (lhs.type == DST_INTEGER && rhs.type == DST_INTEGER && rhs.data.integer == 0)
+        if (lhs.type == DST_INTEGER && rhs.type == DST_INTEGER && rhs.as.integer == 0)
             dst_c_throwc(vm, "cannot integer divide by 0");
         lhs = dst_stl_binop_div(lhs, rhs);
     }
@@ -155,7 +154,7 @@ int dst_stl_##name(Dst *vm) {\
         if (next.type != DST_INTEGER) {\
             dst_c_throwc(vm, "expected integer");\
         }\
-        ret.data.integer = ret.data.integer op next.data.integer;\
+        ret.as.integer = ret.as.integer op next.as.integer;\
     }\
     dst_c_return(vm, ret);\
 }
@@ -174,7 +173,7 @@ int dst_stl_bnot(Dst *vm) {
     if (count != 1 || in.type != DST_INTEGER) {
         dst_c_throwc(vm, "expected 1 integer argument");
     }
-    in.data.integer = ~in.data.integer;
+    in.as.integer = ~in.as.integer;
     dst_c_return(vm, in);
 }
 
@@ -183,7 +182,7 @@ int dst_stl_##name(Dst *vm) {\
     DstValue ret;\
     uint32_t i, count;\
     count = dst_args(vm);\
-    ret.data.boolean = 1;\
+    ret.as.boolean = 1;\
     ret.type = DST_BOOLEAN;\
     if (count < 2) {\
         dst_c_return(vm, ret);\
@@ -192,7 +191,7 @@ int dst_stl_##name(Dst *vm) {\
         DstValue lhs = dst_arg(vm, i - 1);\
         DstValue rhs = dst_arg(vm, i);\
         if (!(check)) {\
-            ret.data.boolean = 0;\
+            ret.as.boolean = 0;\
             break;\
         }\
     }\
@@ -224,13 +223,13 @@ int dst_stl_clear(Dst *vm) {
     default:
         dst_c_throwc(vm, "cannot clear");
     case DST_ARRAY:
-        x.data.array->count = 0;
+        x.as.array->count = 0;
         break;
-    case DST_BYTEBUFFER:
-        x.data.buffer->count = 0;
+    case DST_BUFFER:
+        x.as.buffer->count = 0;
         break;
     case DST_TABLE:
-        dst_table_clear(x.data.table);
+        dst_table_clear(x.as.table);
         break;
     }
     dst_c_return(vm, x);
@@ -255,7 +254,7 @@ int dst_stl_to_int(Dst *vm) {
     DstValue x = dst_arg(vm, 0);
     if (x.type == DST_INTEGER) dst_c_return(vm, x);
     if (x.type == DST_REAL)
-        dst_c_return(vm, integer((DstInteger) x.data.real));
+        dst_c_return(vm, integer((DstInteger) x.as.real));
     else
        dst_c_throwc(vm, "expected number");
 }
@@ -265,7 +264,7 @@ int dst_stl_to_real(Dst *vm) {
     DstValue x = dst_arg(vm, 0);
     if (x.type == DST_REAL) dst_c_return(vm, x);
     if (x.type == DST_INTEGER)
-        dst_c_return(vm, dst_wrap_real((DstReal) x.data.integer));
+        dst_c_return(vm, dst_wrap_real((DstReal) x.as.integer));
     else
        dst_c_throwc(vm, "expected number");
 }
@@ -322,12 +321,12 @@ int dst_stl_slice(Dst *vm) {
         dst_memcpy(arr->data, data + from, newlength * sizeof(DstValue));
         dst_c_return(vm, dst_wrap_array(arr));
     } else if (x.type == DST_STRING) {
-        dst_c_return(vm, dst_wrap_string(dst_string_b(vm, x.data.string + from, newlength)));
+        dst_c_return(vm, dst_wrap_string(dst_string_b(vm, x.as.string + from, newlength)));
     } else if (x.type == DST_SYMBOL) {
-        dst_c_return(vm, dst_wrap_symbol(dst_string_b(vm, x.data.string + from, newlength)));
+        dst_c_return(vm, dst_wrap_symbol(dst_string_b(vm, x.as.string + from, newlength)));
     } else { /* buffer */
         DstBuffer *b = dst_buffer(vm, newlength);
-        dst_memcpy(b->data, x.data.buffer->data, newlength);
+        dst_memcpy(b->data, x.as.buffer->data, newlength);
         b->count = newlength;
         dst_c_return(vm, dst_wrap_buffer(b));
     }
@@ -368,7 +367,7 @@ int dst_stl_type(Dst *vm) {
     case DST_THREAD:
         typestr = "thread";
         break;
-    case DST_BYTEBUFFER:
+    case DST_BUFFER:
         typestr = "buffer";
         break;
     case DST_FUNCTION:
@@ -471,7 +470,7 @@ int dst_stl_string(Dst *vm) {
             dat = dst_to_string(vm, dst_arg(vm, j));
             slen = dst_string_length(dat);
             newarg.type = DST_STRING;
-            newarg.data.string = dat;
+            newarg.as.string = dat;
             dst_set_arg(vm, j, newarg);
         }
         length += slen;
@@ -505,7 +504,7 @@ int dst_stl_thread(Dst *vm) {
     if (callee.type != DST_FUNCTION && callee.type != DST_CFUNCTION)
         dst_c_throwc(vm, "expected function in thread constructor");
     if (parent.type == DST_THREAD) {
-        t->parent = parent.data.thread;
+        t->parent = parent.as.thread;
     } else if (parent.type != DST_NIL) {
         dst_c_throwc(vm, "expected thread/nil as parent");
     } else {
@@ -588,7 +587,7 @@ int dst_stl_push(Dst *vm) {
     DstValue ds = dst_arg(vm, 0);
     if (ds.type != DST_ARRAY)
         dst_c_throwc(vm, "expected array");
-    dst_array_push(vm, ds.data.array, dst_arg(vm, 1));
+    dst_array_push(vm, ds.as.array, dst_arg(vm, 1));
     dst_c_return(vm, ds);
 }
 
@@ -597,7 +596,7 @@ int dst_stl_pop(Dst *vm) {
     DstValue ds = dst_arg(vm, 0);
     if (ds.type != DST_ARRAY)
         dst_c_throwc(vm, "expected array");
-    dst_c_return(vm, dst_array_pop(ds.data.array));
+    dst_c_return(vm, dst_array_pop(ds.as.array));
 }
 
 /* Peek at end of array */
@@ -605,7 +604,7 @@ int dst_stl_peek(Dst *vm) {
     DstValue ds = dst_arg(vm, 0);
     if (ds.type != DST_ARRAY)
         dst_c_throwc(vm, "expected array");
-    dst_c_return(vm, dst_array_peek(ds.data.array));
+    dst_c_return(vm, dst_array_peek(ds.as.array));
 }
 
 /* Ensure array capacity */
@@ -616,7 +615,7 @@ int dst_stl_ensure(Dst *vm) {
         dst_c_throwc(vm, "expected array");
     if (cap.type != DST_INTEGER)
         dst_c_throwc(vm, DST_EXPECTED_INTEGER);
-    dst_array_ensure(vm, ds.data.array, (uint32_t) cap.data.integer);
+    dst_array_ensure(vm, ds.as.array, (uint32_t) cap.as.integer);
     dst_c_return(vm, ds);
 }
 
@@ -625,9 +624,9 @@ int dst_stl_next(Dst *vm) {
     DstValue ds = dst_arg(vm, 0);
     DstValue key = dst_arg(vm, 1);
     if (ds.type == DST_TABLE) {
-        dst_c_return(vm, dst_table_next(ds.data.table, key));
+        dst_c_return(vm, dst_table_next(ds.as.table, key));
     } else if (ds.type == DST_STRUCT) {
-        dst_c_return(vm, dst_struct_next(ds.data.st, key));
+        dst_c_return(vm, dst_struct_next(ds.as.st, key));
     } else {
         dst_c_throwc(vm, "expected table or struct");
     }
@@ -666,7 +665,7 @@ int dst_stl_short_description(Dst *vm) {
 int dst_stl_exit(Dst *vm) {
     int ret;
     DstValue x = dst_arg(vm, 0);
-    ret = x.type == DST_INTEGER ? x.data.integer : (x.type == DST_REAL ? x.data.real : 0);
+    ret = x.type == DST_INTEGER ? x.as.integer : (x.type == DST_REAL ? x.as.real : 0);
     exit(ret);
     return DST_RETURN_OK;
 }
@@ -676,36 +675,6 @@ int dst_stl_error(Dst *vm) {
     dst_c_throw(vm, dst_arg(vm, 0));
 }
 
-/****/
-/* Serialization */
-/****/
-
-/* Serialize data into buffer */
-int dst_stl_serialize(Dst *vm) {
-    const char *err;
-    DstValue buffer = dst_arg(vm, 1);
-    if (buffer.type != DST_BYTEBUFFER)
-        buffer = dst_wrap_buffer(dst_buffer(vm, 10));
-    err = dst_serialize(vm, buffer.data.buffer, dst_arg(vm, 0));
-    if (err != NULL)
-        dst_c_throwc(vm, err);
-    dst_c_return(vm, buffer);
-}
-
-/* Deserialize data from a buffer */
-int dst_stl_deserialize(Dst *vm) {
-    DstValue ret;
-    uint32_t len;
-    const uint8_t *data;
-    const char *err;
-    if (!dst_chararray_view(dst_arg(vm, 0), &data, &len))
-        dst_c_throwc(vm, "expected string/buffer/symbol");
-    err = dst_deserialize(vm, data, len, &ret, &data);
-    if (err != NULL)
-        dst_c_throwc(vm, err);
-    dst_c_return(vm, ret);
-}
-
 /***/
 /* Function reflection */
 /***/
@@ -884,7 +853,7 @@ static int dst_stl_gensym(Dst *vm) {
 static int dst_stl_compile(Dst *vm) {
     DstTable *env = vm->env;
     if (dst_arg(vm, 1).type == DST_TABLE) {
-        env = dst_arg(vm, 1).data.table;
+        env = dst_arg(vm, 1).as.table;
     }
     dst_c_return(vm, dst_compile(vm, env, dst_arg(vm, 0)));
 }
@@ -900,7 +869,7 @@ static int dst_stl_setenv(Dst *vm) {
     if (newEnv.type != DST_TABLE) {
         dst_c_throwc(vm, "expected table");
     }
-    vm->env = newEnv.data.table;
+    vm->env = newEnv.as.table;
     return DST_RETURN_OK;
 }
 
@@ -966,8 +935,6 @@ static const DstModuleItem std_module[] = {
     {"set!", dst_stl_set},
     {"next", dst_stl_next},
     {"error", dst_stl_error},
-    {"serialize", dst_stl_serialize},
-    {"deserialize", dst_stl_deserialize},
     {"push!", dst_stl_push},
     {"pop!", dst_stl_pop},
     {"peek", dst_stl_peek},
@@ -992,7 +959,7 @@ void dst_stl_load(Dst *vm) {
     DstValue maybeEnv = dst_table_get(vm->modules, dst_string_cvs(vm, "std"));
     if (maybeEnv.type == DST_TABLE) {
         /* Module already created, so merge into main vm. */
-        dst_env_merge(vm, vm->env, maybeEnv.data.table);
+        dst_env_merge(vm, vm->env, maybeEnv.as.table);
     } else {
         /* Module not yet created */
         /* Load the normal c functions */
@@ -1009,6 +976,6 @@ void dst_stl_load(Dst *vm) {
         dst_module_put(vm, "std", "stderr", dst_wrap_userdata(outp));
         /* Now merge */
         maybeEnv = dst_table_get(vm->modules, dst_string_cvs(vm, "std"));
-        dst_env_merge(vm, vm->env, maybeEnv.data.table);
+        dst_env_merge(vm, vm->env, maybeEnv.as.table);
     }
 }

core/string.c

@@ -21,22 +21,164 @@
 */
 
 #include "internal.h"
+#include "cache.h"
+#include "wrap.h"
+#include "gc.h"
+
+static const char *types[] = {
+    "nil",
+    "real",
+    "integer",
+    "boolean",
+    "string",
+    "symbol",
+    "array",
+    "tuple",
+    "table",
+    "struct",
+    "thread",
+    "buffer",
+    "function",
+    "cfunction",
+    "userdata"
+};
+
+static const char base64[] =
+    "0123456789"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    "abcdefghijklmnopqrstuvwxyz"
+    "_=";
+
+/* Calculate hash for string */
+static uint32_t dst_string_calchash(const uint8_t *str, uint32_t len) {
+    const uint8_t *end = str + len;
+    uint32_t hash = 5381;
+    while (str < end)
+        hash = (hash << 5) + hash + *str++;
+    return hash;
+}
+
+/* Begin building a string */
+uint8_t *dst_string_begin(Dst *vm, uint32_t length) {
+    char *data = dst_alloc(vm, DST_MEMORY_NONE, 2 * sizeof(uint32_t) + length + 1);
+    uint8_t *str = (uint8_t *) (data + 2 * sizeof(uint32_t));
+    dst_string_length(str) = length;
+    str[length] = 0;
+    return str;
+}
+
+/* Finish building a string */
+const uint8_t *dst_string_end(Dst *vm, uint8_t *str) {
+    DstValue check = dst_string_calchash(str, dst_string_length(str));
+    const uint8_t *ret = dst_cache_add(vm, dst_wrap_string(check)).as.string;
+    gc_settype(dst_string_raw(ret), DST_MEMORY_STRING);
+    return ret;
+}
+
+/* Load a buffer as a string */
+static const uint8_t *dst_string(Dst *vm, const uint8_t *buf, uint32_t len) {
+    uint32_t hash = dst_string_calchash(buf, len);
+    int status = 0;
+    DstValue *bucket = dst_cache_strfind(vm, buf, len, hash, &status);
+    if (status) {
+        return bucket->data.string;
+    } else {
+        uint32_t newbufsize = len + 2 * sizeof(uint32_t) + 1;
+        uint8_t *str = (uint8_t *)(dst_alloc(vm, DST_MEMORY_STRING, newbufsize) + 2 * sizeof(uint32_t));
+        dst_memcpy(str, buf, len);
+        dst_string_length(str) = len;
+        dst_string_hash(str) = hash;
+        str[len] = 0;
+        return dst_cache_add_bucket(vm, dst_wrap_string(str), bucket).as.string;
+    }
+}
+
+/* Helper for creating a unique string. Increment an integer
+ * represented as an array of integer digits. */
+static void inc_counter(uint8_t *digits, int base, int len) {
+    int i;
+    uint8_t carry = 1;
+    for (i = len - 1; i >= 0; --i) {
+        digits[i] += carry;
+        carry = 0;
+        if (digits[i] == base) {
+            digits[i] = 0;
+            carry = 1;
+        }
+    }
+}
+
+/* Generate a unique symbol. This is used in the library function gensym. The
+ * symbol string data does not have GC enabled on it yet. You must manuallyb enable
+ * it later. */
+const uint8_t *dst_string_unique(Dst *vm, const uint8_t *buf, uint32_t len) {
+    DstValue *bucket;
+    uint32_t hash;
+    uint8_t counter[6] = {63, 63, 63, 63, 63, 63};
+    /* Leave spaces for 6 base 64 digits and two dashes. That means 64^6 possible suffixes, which
+     * is enough for resolving collisions. */
+    uint32_t newlen = len + 8;
+    uint32_t newbufsize = newlen + 2 * sizeof(uint32_t) + 1;
+    uint8_t *str = (uint8_t *)(dst_alloc(vm, DST_MEMORY_STRING, newbufsize) + 2 * sizeof(uint32_t));
+    dst_string_length(str) = newlen;
+    memcpy(str, buf, len);
+    str[len] = '-';
+    str[len + 1] = '-';
+    str[newlen] = 0;
+    uint8_t *saltbuf = str + len + 2;
+    int status = 1;
+    while (status) {
+        int i;
+        inc_counter(counter, 64, 6);
+        for (i = 0; i < 6; ++i)
+            saltbuf[i] = base64[counter[i]];
+        hash = dst_string_calchash(str, newlen);
+        bucket = dst_cache_strfind(vm, str, newlen, hash, &status);
+    }
+    dst_string_hash(str) = hash;
+    return dst_cache_add_bucket(vm, dst_wrap_string(str), bucket).as.string;
+}
+
+/* Generate a unique string from a cstring */
+const uint8_t *dst_cstring_unique(Dst *vm, const char *s) {
+    uint32_t len = 0;
+    while (s[len]) ++len;
+    return dst_string_unique(vm, (const uint8_t *)s, len);
+}
+
+/* Load a c string */
+const uint8_t *dst_cstring(Dst *vm, const char *str) {
+    uint32_t len = 0;
+    while (str[len]) ++len;
+    return dst_string(vm, (const uint8_t *)str, len);
+}
 
 /* Temporary buffer size */
 #define DST_BUFSIZE 36
 
-static const uint8_t *real_to_string(Dst *vm, DstReal x) {
-    uint8_t buf[DST_BUFSIZE];
+static uint32_t real_to_string_impl(uint8_t *buf, double x) {
     int count = snprintf((char *) buf, DST_BUFSIZE, "%.21gF", x);
-    return dst_string_b(vm, buf, (uint32_t) count);
+    return (uint32_t) count;
 }
 
-static const uint8_t *integer_to_string(Dst *vm, DstInteger x) {
+static void real_to_string_b(Dst *vm, DstBuffer *buffer, double x) {
+    dst_buffer_ensure_(vm, buffer, buffer->count + DST_BUFSIZE);
+    buffer->count += real_to_string_impl(buffer->data + buffer->count, x);
+}
+
+static void real_to_string(Dst *vm, double x) {
     uint8_t buf[DST_BUFSIZE];
+    dst_string(vm, buf, real_to_string_impl(buf, x));
+}
+
+static uint32_t integer_to_string_impl(uint8_t *buf, int64_t x) {
     int neg = 0;
     uint8_t *hi, *low;
     uint32_t count = 0;
-    if (x == 0) return dst_string_c(vm, "0");
+    if (x == 0) {
+        buf[0] = '0';
+        return 1;
+    }
     if (x < 0) {
         neg = 1;
         x = -x;
@@ -56,16 +198,24 @@ static const uint8_t *integer_to_string(Dst *vm, DstInteger x) {
         *low++ = *hi;
         *hi-- = temp;
     }
-    return dst_string_b(vm, buf, (uint32_t) count);
+    return count;
 }
 
-static const char *HEX_CHARACTERS = "0123456789abcdef";
-#define HEX(i) (((uint8_t *) HEX_CHARACTERS)[(i)])
+static void integer_to_string_b(Dst *vm, DstBuffer *buffer, int64_t x) {
+    dst_buffer_extra(vm, buffer, DST_BUFSIZE);
+    buffer->count += integer_to_string_impl(buffer->data + buffer->count, x);
+}
 
-/* Returns a string description for a pointer. Max titlelen is DST_BUFSIZE
- * - 5 - 2 * sizeof(void *). */
-static const uint8_t *string_description(Dst *vm, const char *title, void *pointer) {
+static void integer_to_string(Dst *vm, int64_t x) {
     uint8_t buf[DST_BUFSIZE];
+    dst_string(vm, buf, integer_to_string_impl(buf, x));
+}
+
+#define HEX(i) (((uint8_t *) base64)[(i)])
+
+/* Returns a string description for a pointer. Truncates
+ * title to 12 characters */
+static uint32_t string_description_impl(uint8_t *buf, const char *title, void *pointer) {
     uint8_t *c = buf;
     uint32_t i;
     union {
@@ -75,7 +225,7 @@ static const uint8_t *string_description(Dst *vm, const char *title, void *point
 
     pbuf.p = pointer;
     *c++ = '<';
-    for (i = 0; title[i]; ++i)
+    for (i = 0; title[i] && i < 12; ++i)
         *c++ = ((uint8_t *)title) [i];
     *c++ = ' ';
     *c++ = '0';
@@ -87,217 +237,219 @@ static const uint8_t *string_description(Dst *vm, const char *title, void *point
         *c++ = HEX(byte & 0xF);
     }
     *c++ = '>';
-    return dst_string_b(vm, buf, c - buf);
+    return (uint32_t) (c - buf);
 }
 
-/* Gets the string value of userdata. Allocates memory, so is slower than
- * string_description. */
-static const uint8_t *string_udata(Dst *vm, const char *title, void *pointer) {
-    uint32_t strlen = 0;
-    uint8_t *c, *buf;
-    uint32_t i;
-    union {
-        uint8_t bytes[sizeof(void *)];
-        void *p;
-    } pbuf;
-
-    while (title[strlen]) ++strlen;
-    c = buf = dst_alloc(vm, strlen + 5 + 2 * sizeof(void *));
-    pbuf.p = pointer;
-    *c++ = '<';
-    for (i = 0; title[i]; ++i)
-        *c++ = ((uint8_t *)title) [i];
-    *c++ = ' ';
-    *c++ = '0';
-    *c++ = 'x';
-    for (i = sizeof(void *); i > 0; --i) {
-        uint8_t byte = pbuf.bytes[i - 1];
-        if (!byte) continue;
-        *c++ = HEX(byte >> 4);
-        *c++ = HEX(byte & 0xF);
-    }
-    *c++ = '>';
-    return dst_string_b(vm, buf, c - buf);
+static void string_description_b(Dst *vm, DstBuffer *buffer, const char *title, void *pointer) {
+    dst_buffer_ensure_(vm, buffer, buffer->count + DST_BUFSIZE);
+    buffer->count += string_description_impl(buffer->data + buffer->count, title, pointer);
 }
 
+static const uint8_t *string_description(Dst *vm, const char *title, void *pointer) {
+    uint8_t buf[DST_BUFSIZE];
+    return dst_string(vm, buf, string_description_impl(buf, title, pointer));
+}
+
+#undef HEX
 #undef DST_BUFSIZE
 
-void dst_escape_string(Dst *vm, DstBuffer *b, const uint8_t *str) {
+static uint32_t dst_escape_string_length(const uint8_t *str) {
+    uint32_t len = 2;
     uint32_t i;
-    dst_buffer_push(vm, b, '"');
     for (i = 0; i < dst_string_length(str); ++i) {
+        switch (str[i]) {
+            case '"':
+            case '\n':
+            case '\r':
+            case '\0':
+                len += 2;
+                break;
+            default:
+                len += 1;
+                break;
+        }
+    }
+    return len;
+}
+
+static void dst_escape_string_impl(uint8_t *buf, const uint8_t *str) {
+    uint32_t i, j;
+    buf[0] = '"';
+    for (i = 0, j = 1; i < dst_string_length(str); ++i) {
         uint8_t c = str[i];
         switch (c) {
             case '"':
-                dst_buffer_push(vm, b, '\\');
-                dst_buffer_push(vm, b, '"');
+                buf[j++] = '\\';
+                buf[j++] = '"';
                 break;
             case '\n':
-                dst_buffer_push(vm, b, '\\');
-                dst_buffer_push(vm, b, 'n');
+                buf[j++] = '\\';
+                buf[j++] = 'n';
                 break;
             case '\r':
-                dst_buffer_push(vm, b, '\\');
-                dst_buffer_push(vm, b, 'r');
+                buf[j++] = '\\';
+                buf[j++] = 'r';
                 break;
             case '\0':
-                dst_buffer_push(vm, b, '\\');
-                dst_buffer_push(vm, b, '0');
+                buf[j++] = '\\';
+                buf[j++] = '0';
                 break;
             default:
-                dst_buffer_push(vm, b, c);
+                buf[j++] = c;
                 break;
         }
     }
-    dst_buffer_push(vm, b, '"');
+    buf[j++] = '"';
+}
+
+static void dst_escape_string_b(Dst *vm, DstBuffer *buffer, const uint8_t *str) {
+    uint32_t len = dst_escape_string_length(str);
+    dst_buffer_extra(vm, buffer, len);
+    dst_escape_string_impl(buffer->data + buffer->count, str);
+    buffer->count += len;
+}
+
+static const uint8_t *dst_escape_string(Dst *vm, const uint8_t *str) {
+    uint32_t len = dst_escape_string_length(str);
+    uint8_t *buf = dst_string_begin(vm, len);
+    dst_escape_string_impl(buf, str);
+    return dst_string_end(vm, buf);
 }
 
 /* Returns a string pointer with the description of the string */
-const uint8_t *dst_short_description(Dst *vm, DstValue x) {
+static const uint8_t *dst_short_description(Dst *vm) {
     switch (x.type) {
     case DST_NIL:
-        return dst_string_c(vm, "nil");
+        return dst_cstring(vm, "nil");
     case DST_BOOLEAN:
-        if (x.data.boolean)
-            return dst_string_c(vm, "true");
+        if (x.as.boolean)
+            return dst_cstring(vm, "true");
         else
-            return dst_string_c(vm, "false");
+            return dst_cstring(vm, "false");
     case DST_REAL:
-        return real_to_string(vm, x.data.real);
+        return real_to_string(vm, x.as.real);
     case DST_INTEGER:
-        return integer_to_string(vm, x.data.integer);
-    case DST_ARRAY:
-        return string_description(vm, "array", x.data.pointer);
-    case DST_TUPLE:
-        return string_description(vm, "tuple", x.data.pointer);
-    case DST_STRUCT:
-        return string_description(vm, "struct", x.data.pointer);
-    case DST_TABLE:
-        return string_description(vm, "table", x.data.pointer);
+        return integer_to_string(vm, x.as.integer);
     case DST_SYMBOL:
-        return x.data.string;
+        return x.as.string;
     case DST_STRING:
-    {
-        DstBuffer *buf = dst_buffer(vm, dst_string_length(x.data.string) + 4);
-        dst_escape_string(vm, buf, x.data.string);
-        return dst_buffer_to_string(vm, buf);
-    }
-    case DST_BYTEBUFFER:
-        return string_description(vm, "buffer", x.data.pointer);
-    case DST_CFUNCTION:
-        return string_description(vm, "cfunction", x.data.pointer);
-    case DST_FUNCTION:
-        return string_description(vm, "function", x.data.pointer);
-    case DST_THREAD:
-        return string_description(vm, "thread", x.data.pointer);
+        return dst_escape_string(vm, x.as.string);
     case DST_USERDATA:
-        return string_udata(vm, dst_udata_type(x.data.pointer)->name, x.data.pointer);
-    case DST_FUNCENV:
-        return string_description(vm, "funcenv", x.data.pointer);
-    case DST_FUNCDEF:
-        return string_description(vm, "funcdef", x.data.pointer);
+        return string_description(vm, dst_udata_type(x.as.pointer)->name, x.as.pointer);
+    default:
+        return string_description(vm, types[x.type], x.as.pointer);
     }
 }
 
-static DstValue wrap_integer(DstInteger i) {
-    DstValue v;
-    v.type = DST_INTEGER;
-    v.data.integer = i;
-    return v;
+static void dst_short_description_b(Dst *vm, DstBuffer *buffer, DstValue x) {
+    switch (x.type) {
+    case DST_NIL:
+        dst_buffer_push_cstring(vm, buffer, "nil");
+        return;
+    case DST_BOOLEAN:
+        if (x.as.boolean)
+            dst_buffer_push_cstring(vm, buffer, "true");
+        else
+            dst_buffer_push_cstring(vm, buffer, "false");
+        return;
+    case DST_REAL:
+        real_to_string_b(vm, buffer, x.as.real);
+        return;
+    case DST_INTEGER:
+        integer_to_string_b(vm, buffer, x.as.integer);
+        return;
+    case DST_SYMBOL:
+        dst_buffer_push_bytes(vm, buffer, x.as.string, dst_string_length(x.as.string));
+        return;
+    case DST_STRING:
+        dst_escape_string_b(vm, buffer, x.as.string);
+        return;
+    case DST_USERDATA:
+        string_description_b(vm, buffer, dst_udata_type(x.as.pointer)->name, x.as.pointer);
+        return;
+    default:
+        string_description_b(vm, buffer, types[x.type], x.as.pointer);
+        break;
+    }
 }
 
 /* Static debug print helper */
-static DstInteger dst_description_helper(Dst *vm, DstBuffer *b, DstTable *seen, DstValue x, DstInteger next, int depth) {
+static int64_t dst_description_helper(Dst *vm, DstBuffer *b, DstTable *seen, DstValue x, int64_t next, int depth) {
     DstValue check = dst_table_get(seen, x);
-    const uint8_t *str;
-    /* Prevent a stack overflow */
-    if (depth++ > DST_RECURSION_GUARD)
-        return -1;
     if (check.type == DST_INTEGER) {
-        str = integer_to_string(vm, check.data.integer);
-        dst_buffer_append_cstring(vm, b, "<visited ");
-        dst_buffer_append(vm, b, str, dst_string_length(str));
-        dst_buffer_append_cstring(vm, b, ">");
+        dst_buffer_push_cstring(vm, b, "<cycle>");
     } else {
-        uint8_t open, close;
+        const char *open;
+        const char *close;
         uint32_t len, i;
         const DstValue *data;
         switch (x.type) {
             default:
-                str = dst_short_description(vm, x);
-                dst_buffer_append(vm, b, str, dst_string_length(str));
-                return next;
-            case DST_STRING:
-                dst_escape_string(vm, b, x.data.string);
-                return next;
-            case DST_SYMBOL:
-                dst_buffer_append(vm, b, x.data.string, dst_string_length(x.data.string));
-                return next;
-            case DST_NIL:
-                dst_buffer_append_cstring(vm, b, "nil");
-                return next;
-            case DST_BOOLEAN:
-                dst_buffer_append_cstring(vm, b, x.data.boolean ? "true" : "false");
+                dst_short_description_b(vm, b, x);
                 return next;
             case DST_STRUCT:
-                open = '<'; close = '>';
+                open = "{"; close = "}";
                 break;
             case DST_TABLE:
-                open = '{'; close = '}';
+                open = "{"; close = "}";
                 break;
             case DST_TUPLE:
-                open = '('; close = ')';
+                open = "("; close = ")";
                 break;
             case DST_ARRAY:
-                open = '['; close = ']';
+                open = "["; close = "]";
                 break;
         }
-        dst_table_put(vm, seen, x, wrap_integer(next++));
-        dst_buffer_push(vm, b, open);
-        if (dst_hashtable_view(x, &data, &len)) {
+        dst_table_put(vm, seen, x, dst_wrap_integer(next++));
+        dst_buffer_push_cstring(vm, b, open);
+        if (depth == 0) {
+            dst_buffer_push_cstring(vm, b, "...");
+        } else if (dst_hashtable_view(x, &data, &len)) {
             int isfirst = 1;
             for (i = 0; i < len; i += 2) {
                 if (data[i].type != DST_NIL) {
                     if (isfirst)
                         isfirst = 0;
                     else
-                        dst_buffer_push(vm, b, ' ');
-                    next = dst_description_helper(vm, b, seen, data[i], next, depth);
-                    if (next == -1)
-                        dst_buffer_append_cstring(vm, b, "...");
-                    dst_buffer_push(vm, b, ' ');
-                    next = dst_description_helper(vm, b, seen, data[i + 1], next, depth);
-                    if (next == -1)
-                        dst_buffer_append_cstring(vm, b, "...");
+                        dst_buffer_push_u8(vm, b, ' ');
+                    next = dst_description_helper(vm, b, seen, data[i], next, depth - 1);
+                    dst_buffer_push_u8(vm, b, ' ');
+                    next = dst_description_helper(vm, b, seen, data[i + 1], next, depth - 1);
                 }
             }
         } else if (dst_seq_view(x, &data, &len)) {
             for (i = 0; i < len; ++i) {
-                next = dst_description_helper(vm, b, seen, data[i], next, depth);
-                if (next == -1)
-                    return -1;
+                next = dst_description_helper(vm, b, seen, data[i], next, depth - 1);
                 if (i != len - 1)
-                    dst_buffer_push(vm, b, ' ');
+                    dst_buffer_push_u8(vm, b, ' ');
             }
         }
-        dst_buffer_push(vm, b, close);
+        dst_buffer_push_cstring(vm, b, close);
     }
     return next;
 }
 
-/* Debug print. Returns a description of an object as a buffer. */
+/* Debug print. Returns a description of an object as a string. */
 const uint8_t *dst_description(Dst *vm, DstValue x) {
     DstBuffer *buf = dst_buffer(vm, 10);
-    dst_description_helper(vm, buf, dst_table(vm, 10), x, 0, 0);
-    return dst_buffer_to_string(vm, buf);
+    DstTable *seen = dst_table(vm, 10);
+
+    /* Only print description up to a depth of 4 */
+    dst_description_helper(vm, buf, seen, x, 0, 4);
+
+    return dst_string(vm, buf->data, buf->count);
 }
 
+/* Convert any value to a dst string */
 const uint8_t *dst_to_string(Dst *vm, DstValue x) {
-    if (x.type == DST_STRING || x.type == DST_SYMBOL) {
-        return x.data.string;
-    } else if (x.type == DST_BYTEBUFFER) {
-        return dst_buffer_to_string(vm, x.data.buffer);
-    } else {
+    DstValue x = dst_getv(vm, -1);
+    switch (x.type) {
+        default:
             return dst_description(vm, x);
+        case DST_STRING:
+        case DST_SYMBOL:
+            return x.as.string;
+        case DST_BUFFER:
+            return dst_string(vm, x.as.buffer->data, x.as.buffer->count);
     }
 }

core/struct.c

@@ -0,0 +1,184 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#include "internal.h"
+#include "cache.h"
+
+/* Begin creation of a struct */
+DstValue *dst_struct_begin(Dst *vm, uint32_t count) {
+    /* This expression determines size of structs. It must be a pure
+     * function of count, and hold at least enough space for count 
+     * key value pairs. The minimum it could be is 
+     * sizeof(uint32_t) * 2 + 2 * count * sizeof(DstValue). Adding more space
+     * ensures that structs are less likely to have hash collisions. If more space
+     * is added or s is changed, change the macro dst_struct_capacity in internal.h */
+    size_t s = sizeof(uint32_t) * 2 + 4 * count * sizeof(DstValue);
+    char *data = dst_alloc(vm, DST_MEMORY_STRUCT, s);
+    memset(data, 0, s)
+    DstValue *st = (DstValue *) (data + 2 * sizeof(uint32_t));
+    dst_struct_length(st) = count;
+    /* Use the hash storage space as a counter to see how many items
+     * were successfully added. If this number is not equal to the
+     * original number, we will need to remake the struct using
+     * only the kv pairs in the struct */
+    dst_struct_hash(st) = 0;
+    return st;
+}
+
+/* Find an item in a struct */
+static const DstValue *dst_struct_find(const DstValue *st, DstValue key) {
+    uint32_t cap = dst_struct_capacity(st);
+    uint32_t index = (dst_value_hash(key) % (cap / 2)) * 2;
+    uint32_t i;
+    for (i = index; i < cap; i += 2)
+        if (st[i].type == DST_NIL || dst_equals(st[i], key))
+            return st + i;
+    for (i = 0; i < index; i += 2)
+        if (st[i].type == DST_NIL || dst_equals(st[i], key))
+            return st + i;
+    return NULL;
+}
+
+/* Put a kv pair into a struct that has not yet been fully constructed.
+ * Nil keys and values are ignored, extra keys are ignore, and duplicate keys are
+ * ignored. */
+void dst_struct_put(DstValue *st, DstValue key, DstValue value) {
+    uint32_t cap = dst_struct_capacity(st);
+    uint32_t hash = dst_hash(key);
+    uint32_t index = (hash % (cap / 2)) * 2;
+    uint32_t i, j, dist;
+    uint32_t bounds[4] = {index, cap, 0, index};
+    if (key.type == DST_NIL || value.type == DST_NIL) return;
+    /* Avoid extra items */
+    if (dst_struct_hash(st) == dst_struct_length(st)) return;
+    for (dist = 0, j = 0; j < 4; j += 2)
+    for (i = bounds[j]; i < bounds[j + 1]; i += 2, dist += 2) {
+        int status;
+        uint32_t otherhash, otherindex, otherdist;
+        /* We found an empty slot, so just add key and value */
+        if (st[i].type == DST_NIL) {
+            st[i] = key;
+            st[i + 1] = value;
+            /* Update the temporary count */
+            dst_struct_hash(st)++;
+            return;
+        }
+        /* Robinhood hashing - check if colliding kv pair
+         * is closer to their source than current. We use robinhood
+         * hashing to ensure that equivalent structs that are contsructed
+         * with different order have the same internal layout, and therefor
+         * will compare properly - i.e., {1 2 3 4} should equal {3 4 1 2}. */
+        otherhash = dst_hash(st[i]);
+        otherindex = (otherhash % (cap / 2)) * 2;
+        otherdist = (i + cap - otherindex) % cap;
+        if (dist < otherdist)
+            status = -1;
+        else if (otherdist < dist)
+            status = 1;
+        else if (hash < otherhash)
+            status = -1;
+        else if (otherhash < hash)
+            status = 1;
+        else
+            status = dst_compare(key, st[i]);
+        /* If other is closer to their ideal slot */
+        if (status == 1) {
+            /* Swap current kv pair with pair in slot */
+            DstValue t1, t2;
+            t1 = st[i];
+            t2 = st[i + 1];
+            st[i] = key;
+            st[i + 1] = value;
+            key = t1;
+            value = t2;
+            /* Save dist and hash of new kv pair */
+            dist = otherdist;
+            hash = otherhash;
+        } else {
+            /* This should not happen - it means
+             * than a key was added to the struct more than once */
+            return;
+        }
+    }
+}
+
+/* Finish building a struct */
+static const DstValue *dst_struct_end(Dst *vm, DstValue *st) {
+    DstValue cached;
+    DstValue check;
+    /* For explicit tail recursion */
+    recur:
+    if (dst_struct_hash(st) != dst_struct_length(st)) {
+        /* Error building struct, probably duplicate values. We need to rebuild
+         * the struct using only the values that went in. The second creation should always
+         * succeed. */
+        uint32_t i, realCount;
+        DstValue *newst;
+        realCount = 0;
+        for (i = 0; i < dst_struct_capacity(st); i += 2) {
+            realCount += st[i].type != DST_NIL;
+        }
+        newst = dst_struct_begin(vm, realCount);
+        for (i = 0; i < dst_struct_capacity(st); i += 2) {
+            if (st[i].type != DST_NIL) {
+                dst_struct_put(newst, st[i], st[i + 1]);
+            }
+        }
+        st = newst;
+        goto recur;
+    }
+    dst_struct_hash(st) = dst_calchash_array(st, dst_struct_capacity(st));
+    check.type = DST_STRUCT;
+    check.as.st = (const DstValue *) st;
+    return dst_cache_add(vm, check).as.st;
+}
+
+/* Get an item from a struct */
+DstValue dst_struct_get(const DstValue *st, DstValue key) {
+    const DstValue *bucket = dst_struct_find(st, key);
+    if (!bucket || bucket[0].type == DST_NIL) {
+        DstValue ret;
+        ret.type = DST_NIL;
+        return  ret;
+    } else {
+        return bucket[1];
+    }
+}
+
+/* Get the next key in a struct */
+DstValue dst_struct_next(const DstValue *st, DstValue key) {
+    const DstValue *bucket, *end;
+    end = st + dst_struct_capacity(st);
+    if (key.type == DST_NIL) {
+        bucket = st;
+    } else {
+        bucket = dst_struct_find(st, key);
+        if (!bucket || bucket[0].type == DST_NIL)
+            return dst_wrap_nil();
+        bucket += 2;
+    }
+    for (; bucket < end; bucket += 2) {
+        if (bucket[0].type != DST_NIL)
+            return bucket[0];
+    }
+    return dst_wrap_nil();
+}

core/table.c

@@ -0,0 +1,163 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#include "internal.h"
+#include "wrap.h"
+#include "gc.h"
+
+/* Initialize a table */
+DstTable *dst_table(Dst *vm, uint32_t capacity) {
+    DstTable *table = dst_alloc(vm, DST_MEMORY_TABLE, sizeof(DstTable));
+    DstValue *data = calloc(sizeof(DstValue), capacity);
+    if (capacity < 2) capacity = 2;
+    if (NULL == data) {
+        DST_OUT_OF_MEMORY;
+    }
+    table->data = data;
+    table->capacity = capacity;
+    table->count = 0;
+    table->deleted = 0;
+    return table;
+}
+
+/* Find the bucket that contains the given key. Will also return
+ * bucket where key should go if not in the table. */
+static DstValue *dst_table_find(DstTable *t, DstValue key) {
+    uint32_t index = (dst_hash(key) % (t->capacity / 2)) * 2;
+    uint32_t i, j;
+    uint32_t start[2], end[2];
+    start[0] = index; end[0] = t->capacity;
+    start[1] = 0; end[1] = index;
+    for (j = 0; j < 2; ++j) {
+        for (i = start[j]; i < end[j]; i += 2) {
+            if (t->data[i].type == DST_NIL) {
+                if (t->data[i + 1].type == DST_NIL) {
+                    /* Empty */
+                    return t->data + i;
+                }
+            } else if (dst_equals(t->data[i], key)) {
+                return t->data + i;
+            }
+        }
+    }
+    return NULL;
+}
+
+/* Resize the dictionary table. */
+static void dst_table_rehash(Dst *vm, DstTable *t, uint32_t size) {
+    DstValue *olddata = t->data;
+    DstValue *newdata = calloc(sizeof(DstValue), size);
+    if (NULL == newdata) {
+        DST_OUT_OF_MEMORY;
+    }
+    uint32_t i, oldcapacity;
+    oldcapacity = t->capacity;
+    t->data = newdata;
+    t->capacity = size;
+    t->deleted = 0;
+    for (i = 0; i < oldcapacity; i += 2) {
+        if (olddata[i].type != DST_NIL) {
+            DstValue *bucket = dst_table_find(t, olddata[i]);
+            bucket[0] = olddata[i];
+            bucket[1] = olddata[i + 1];
+        }
+    }
+    free(olddata);
+}
+
+/* Get a value out of the object */
+DstValue dst_table_get(DstTable *t, DstValue key) {
+    DstValue *bucket = dst_table_find(t, key);
+    if (bucket && bucket[0].type != DST_NIL)
+        return bucket[1];
+    else
+        return dst_wrap_nil();
+}
+
+/* Remove an entry from the dictionary. Return the value that
+ * was removed. */
+DstValue dst_table_remove(DstTable *t, DstValue key) {
+    DstValue *bucket = dst_table_find(t, key);
+    if (bucket && bucket[0].type != DST_NIL) {
+        DstValue ret = bucket[1];
+        t->count--;
+        t->deleted++;
+        bucket[0].type = DST_NIL;
+        bucket[1].type = DST_BOOLEAN;
+        return ret;
+    } else {
+        return dst_wrap_nil();
+    }
+}
+
+/* Put a value into the object */
+void dst_table_put(Dst *vm, DstTable *t, DstValue key, DstValue value) {
+    if (key.type == DST_NIL) return;
+    if (value.type == DST_NIL) {
+        dst_table_remove(t, key);
+    } else {
+        DstValue *bucket = dst_table_find(t, key);
+        if (bucket && bucket[0].type != DST_NIL) {
+            bucket[1] = value;
+        } else {
+            if (!bucket || 4 * (t->count + t->deleted) >= t->capacity) {
+                dst_table_rehash(vm, t, 4 * t->count + 6);
+            }
+            bucket = dst_table_find(t, key);
+            if (bucket[1].type == DST_BOOLEAN)
+                --t->deleted;
+            bucket[0] = key;
+            bucket[1] = value;
+            ++t->count;
+        }
+    }
+}
+
+/* Clear a table */
+void dst_table_clear(DstTable *t) {
+    uint32_t capacity = t->capacity;
+    uint32_t i;
+    DstValue *data = t->data;
+    for (i = 0; i < capacity; i += 2)
+        data[i].type = DST_NIL;
+    t->count = 0;
+    t->deleted = 0;
+}
+
+/* Find next key in an object. Returns nil if no next key. */
+DstValue dst_table_next(DstTable *t, DstValue key) {
+    const DstValue *bucket, *end;
+    end = t->data + t->capacity;
+    if (key.type == DST_NIL) {
+        bucket = t->data;
+    } else {
+        bucket = dst_table_find(t, key);
+        if (!bucket || bucket[0].type == DST_NIL)
+            return dst_wrap_nil();
+        bucket += 2;
+    }
+    for (; bucket < end; bucket += 2) {
+        if (bucket[0].type != DST_NIL)
+            return bucket[0];
+    }
+    return dst_wrap_nil();
+}

core/thread.c

@@ -21,13 +21,19 @@
 */
 
 #include "internal.h"
+#include "wrap.h"
+#include "gc.h"
 
-/* Create a new thread */
+/* Initialize a new thread */
 DstThread *dst_thread(Dst *vm, DstValue callee, uint32_t capacity) {
-    DstThread *thread = dst_alloc(vm, sizeof(DstThread));
+    DstThread *thread = dst_alloc(vm, DST_MEMORY_THREAD, sizeof(DstThread));
     if (capacity < DST_FRAME_SIZE) capacity = DST_FRAME_SIZE;
-    thread->data = dst_alloc(vm, sizeof(DstValue) * capacity);
     thread->capacity = capacity;
+    DstValue *data = malloc(vm, sizeof(DstValue) * capacity);
+    if (NULL == data) {
+        DST_OUT_OF_MEMORY;
+    }
+    thread->data = data;
     return dst_thread_reset(vm, thread, callee);
 }
 
@@ -49,7 +55,7 @@ DstThread *dst_thread_reset(Dst *vm, DstThread *thread, DstValue callee) {
     return thread;
 }
 
-/* Ensure that the thread has enough EXTRA capacity */
+/* Ensure that the thread has enough extra capacity */
 void dst_thread_ensure_extra(Dst *vm, DstThread *thread, uint32_t extra) {
     DstValue *newData, *stack;
     uint32_t usedCapacity, neededCapacity, newCapacity;
@@ -58,8 +64,12 @@ void dst_thread_ensure_extra(Dst *vm, DstThread *thread, uint32_t extra) {
     neededCapacity = usedCapacity + extra;
     if (thread->capacity >= neededCapacity) return;
     newCapacity = 2 * neededCapacity;
-    newData = dst_alloc(vm, sizeof(DstValue) * newCapacity);
-    dst_memcpy(newData, thread->data, sizeof(DstValue) * usedCapacity);
+
+    newData = realloc(thread->data, sizeof(DstValue) * newCapacity);
+    if (NULL == newData) {
+        DST_OUT_OF_MEMORY;
+    }
+
     thread->data = newData;
     thread->capacity = newCapacity;
 }
@@ -85,7 +95,8 @@ void dst_thread_pushnil(Dst *vm, DstThread *thread, uint32_t n) {
     dst_frame_size(stack) += n;
 }
 
-/* Package up extra args after and including n into tuple at n*/
+/* Package up extra args after and including n into tuple at n. Used for
+ * packing up varargs to variadic functions. */
 void dst_thread_tuplepack(Dst *vm, DstThread *thread, uint32_t n) {
     DstValue *stack = thread->data + thread->count;
     uint32_t size = dst_frame_size(stack);
@@ -94,7 +105,7 @@ void dst_thread_tuplepack(Dst *vm, DstThread *thread, uint32_t n) {
         dst_thread_pushnil(vm, thread, n - size + 1);
         stack = thread->data + thread->count;
         stack[n].type = DST_TUPLE;
-        stack[n].data.tuple = dst_tuple_end(vm, dst_tuple_begin(vm, 0));
+        stack[n].as.tuple = dst_tuple_end(vm, dst_tuple_begin(vm, 0));
         dst_frame_size(stack) = n + 1;
     } else {
         uint32_t i;
@@ -102,7 +113,7 @@ void dst_thread_tuplepack(Dst *vm, DstThread *thread, uint32_t n) {
         for (i = n; i < size; ++i)
             tuple[i - n] = stack[i];
         stack[n].type = DST_TUPLE;
-        stack[n].data.tuple = dst_tuple_end(vm, tuple);
+        stack[n].as.tuple = dst_tuple_end(vm, tuple);
     }
 }
 
@@ -136,7 +147,7 @@ void dst_thread_endframe(Dst *vm, DstThread *thread) {
     DstValue *stack = thread->data + thread->count;
     DstValue callee = dst_frame_callee(stack);
     if (callee.type == DST_FUNCTION) {
-        DstFunction *fn = callee.data.function;
+        DstFunction *fn = callee.as.function;
         uint32_t locals = fn->def->locals;
         dst_frame_pc(stack) = fn->def->byteCode;
         if (fn->def->flags & DST_FUNCDEF_FLAG_VARARG) {
@@ -162,8 +173,8 @@ DstValue *dst_thread_popframe(Dst *vm, DstThread *thread) {
         uint32_t size = dst_frame_size(stack);
         env->thread = NULL;
         env->stackOffset = size;
-        env->values = dst_alloc(vm, sizeof(DstValue) * size);
-        dst_memcpy(env->values, stack, sizeof(DstValue) * size);
+        env->values = malloc(sizeof(DstValue) * size);
+        memcpy(env->values, stack, sizeof(DstValue) * size);
     }
 
     /* Shrink stack */

core/tuple.c

@@ -0,0 +1,47 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#include "internal.h"
+#include "cache.h"
+#include "wrap.h"
+
+/* Create a new empty tuple of the given size. This will return memory
+ * which should be filled with DstValues. The memory will not be collected until
+ * dst_tuple_end is called. */
+DstValue *dst_tuple_begin(Dst *vm, uint32_t length) {
+    char *data = dst_alloc(vm, DST_MEMORY_TUPLE, 2 * sizeof(uint32_t) + length * sizeof(DstValue));
+    DstValue *tuple = (DstValue *)(data + (2 * sizeof(uint32_t)));
+    dst_tuple_length(tuple) = length;
+    return tuple;
+}
+
+/* Finish building a tuple */
+const DstValue *dst_tuple_end(Dst *vm, DstValue *tuple) {
+    dst_tuple_hash(tuple) = dst_calchash_array(tuple, dst_tuple_length(tuple));
+    return dst_cache_add(vm, dst_wrap_tuple((const DstValue *) tuple)).as.tuple;
+}
+
+const DstValue *dst_tuple_n(Dst *vm, DstValue *values, uint32_t n) {
+    DstValue *t = dst_tuple_begin(vm, n);
+    memcpy(t, values, sizeof(DstValue) * n);
+    return dst_tuple_end(vm, t);
+}

core/userdata.c

@@ -0,0 +1,35 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#include "internal.h"
+#include "wrap.h"
+
+/* Create new userdata */
+void *dst_userdata(Dst *vm, uint32_t size, const DstUserType *utype) {
+    DstValue ud;
+    char *data = dst_alloc(vm, DST_USERDATA, sizeof(DstUserdataHeader) + size);
+    DstUserdataHeader *header = (DstUserdataHeader *)data;
+    void *user = data + sizeof(DstUserdataHeader);
+    header->size = size;
+    header->type = utype;
+    return dst_wrap_userdata(user);
+}

core/util.c

@@ -22,102 +22,32 @@
 
 #include "internal.h"
 
-/****/
-/* Parsing utils */
-/****/
+int dst_checkerr(Dst *vm) { return !!vm->flags; }
 
-/* Get an integer power of 10 */
-static double exp10(int power) {
-    if (power == 0) return 1;
-    if (power > 0) {
-        double result = 10;
-        int currentPower = 1;
-        while (currentPower * 2 <= power) {
-            result = result * result;
-            currentPower *= 2;
-        }
-        return result * exp10(power - currentPower);
-    } else {
-        return 1 / exp10(-power);
-    }
+void dst_return(Dst *vm, DstValue x) {
+    vm->ret = x;
 }
 
-int dst_read_integer(const uint8_t *string, const uint8_t *end, int64_t *ret) {
-    int sign = 1, x = 0;
-    int64_t accum = 0;
-    if (*string == '-') {
-        sign = -1;
-        ++string;
-    } else if (*string == '+') {
-        ++string;
-    }
-    if (string >= end) return 0;
-    while (string < end) {
-        x = *string;
-        if (x < '0' || x > '9') return 0;
-        x -= '0';
-        accum = accum * 10 + x;
-        ++string;
-    }
-    *ret = accum * sign;
-    return 1;
+void dst_throw(Dst *vm) {
+    vm->flags = 1;
+    vm->ret = dst_popv(vm);
 }
 
-/* Read a real from a string. Returns if successfuly
- * parsed a real from the enitre input string.
- * If returned 1, output is int ret.*/
-int dst_read_real(const uint8_t *string, const uint8_t *end, double *ret, int forceInt) {
-    int sign = 1, x = 0;
-    double accum = 0, exp = 1, place = 1;
-    /* Check the sign */
-    if (*string == '-') {
-        sign = -1;
-        ++string;
-    } else if (*string == '+') {
-        ++string;
-    }
-    if (string >= end) return 0;
-    while (string < end) {
-        if (*string == '.' && !forceInt) {
-            place = 0.1;
-        } else if (!forceInt && (*string == 'e' || *string == 'E')) {
-            /* Read the exponent */
-            ++string;
-            if (string >= end) return 0;
-            if (!dst_read_real(string, end, &exp, 1))
-                return 0;
-            exp = exp10(exp);
-            break;
-        } else {
-            x = *string;
-            if (x < '0' || x > '9') return 0;
-            x -= '0';
-            if (place < 1) {
-                accum += x * place;
-                place *= 0.1;
-            } else {
-                accum *= 10;
-                accum += x;
-            }
-        }
-        ++string;
-    }
-    *ret = accum * sign * exp;
-    return 1;
+void dst_cerr(Dst *vm, const char *message) {
+    vm->flags = 1;
+    vm->ret = dst_string_cv(vm, message);
 }
 
-/* Utilities for manipulating different types with the same semantics */
-
 /* Read both tuples and arrays as c pointers + uint32_t length. Return 1 if the
  * view can be constructed, 0 if an invalid type. */
 int dst_seq_view(DstValue seq, const DstValue **data, uint32_t *len) {
     if (seq.type == DST_ARRAY) {
-        *data = seq.data.array->data;
-        *len = seq.data.array->count;
+        *data = seq.as.array->data;
+        *len = seq.as.array->count;
         return 1;
     } else if (seq.type == DST_TUPLE) {
-        *data = seq.data.st;
-        *len = dst_tuple_length(seq.data.st);
+        *data = seq.as.st;
+        *len = dst_tuple_length(seq.as.st);
         return 1;
     }
     return 0;
@@ -127,12 +57,12 @@ int dst_seq_view(DstValue seq, const DstValue **data, uint32_t *len) {
  * Returns 1 if the view can be constructed and 0 if the type is invalid. */
 int dst_chararray_view(DstValue str, const uint8_t **data, uint32_t *len) {
     if (str.type == DST_STRING || str.type == DST_SYMBOL) {
-        *data = str.data.string;
-        *len = dst_string_length(str.data.string);
+        *data = str.as.string;
+        *len = dst_string_length(str.as.string);
         return 1;
-    } else if (str.type == DST_BYTEBUFFER) {
-        *data = str.data.buffer->data;
-        *len = str.data.buffer->count;
+    } else if (str.type == DST_BUFFER) {
+        *data = str.as.buffer->data;
+        *len = str.as.buffer->count;
         return 1;
     }
     return 0;
@@ -143,211 +73,32 @@ int dst_chararray_view(DstValue str, const uint8_t **data, uint32_t *len) {
  * 0 if the type is invalid. */
 int dst_hashtable_view(DstValue tab, const DstValue **data, uint32_t *cap) {
     if (tab.type == DST_TABLE) {
-        *data = tab.data.table->data;
-        *cap = tab.data.table->capacity;
+        *data = tab.as.table->data;
+        *cap = tab.as.table->capacity;
         return 1;
     } else if (tab.type == DST_STRUCT) {
-        *data = tab.data.st;
-        *cap = dst_struct_capacity(tab.data.st);
+        *data = tab.as.st;
+        *cap = dst_struct_capacity(tab.as.st);
         return 1;
     }
     return 0;
 }
 
-DstReal dst_integer_to_real(DstInteger x) {
-    return (DstReal) x;
+/* Convert an index used by the capi to an absolute index */
+uint32_t dst_startrange(int64_t index, uint32_t modulo) {
+    if (index < 0) index += modulo;
+    return ((index >= 0 && index < modulo)) ? ((uint32_t) index) : 0;
 }
 
-DstInteger dst_real_to_integer(DstReal x) {
-    return (DstInteger) x;
+/* Convert an index used by the capi to an absolute index */
+uint32_t dst_endrange(int64_t index, uint32_t modulo) {
+    return dst_startrange(index, modulo + 1);
 }
 
-uint32_t dst_startrange(DstInteger raw, uint32_t len) {
-    if (raw >= len)
-        return -1;
-    if (raw < 0)
-        return len + raw;
-    return raw;
-}
-
-uint32_t dst_endrange(DstInteger raw, uint32_t len) {
-    if (raw > len)
-        return -1;
-    if (raw < 0)
-        return len + raw + 1;
-    return raw;
-}
-
-static DstValue cfunction(DstCFunction fn) {
-    DstValue n;
-    n.type = DST_CFUNCTION;
-    n.data.cfunction = fn;
-    return n;
-}
-
-int dst_callc(Dst *vm, DstCFunction fn, int numargs, ...) {
-    int result, i;
-    va_list args;
-    DstValue *stack;
-    va_start(args, numargs);
-    stack = dst_thread_beginframe(vm, vm->thread, cfunction(fn), numargs);
-    for (i = 0; i < numargs; ++i) {
-        stack[i] = va_arg(args, DstValue);
+/* Convert a possibly negative index to a positive index on the current stack. */
+int64_t dst_normalize_index(Dst *vm, int64_t index) {
+    if (index < 0) {
+        index += dst_frame_size(dst_thread_stack(vm->thread));
     }
-    va_end(args);
-    result = fn(vm);
-    dst_thread_popframe(vm, vm->thread);
-    return result;
-}
-
-
-/* Stack manipulation functions */
-int dst_checkerr(Dst *vm) {
-    return !!vm->flags;
-}
-
-/* Get an argument from the stack */
-DstValue dst_arg(Dst *vm, uint32_t index) {
-    DstValue *stack = dst_thread_stack(vm->thread);
-    uint32_t frameSize = dst_frame_size(stack);
-    if (frameSize <= index) {
-        DstValue ret;
-        ret.type = DST_NIL;
-        return ret;
-    }
-    return stack[index];
-}
-
-/* Put a value on the stack */
-void dst_set_arg(Dst* vm, uint32_t index, DstValue x) {
-    DstValue *stack = dst_thread_stack(vm->thread);
-    uint32_t frameSize = dst_frame_size(stack);
-    if (frameSize <= index) return;
-    stack[index] = x;
-}
-
-/* Get the size of the VMStack */
-uint32_t dst_args(Dst *vm) {
-    DstValue *stack = dst_thread_stack(vm->thread);
-    return dst_frame_size(stack);
-}
-
-void dst_addsize(Dst *vm, uint32_t n) {
-    dst_thread_pushnil(vm, vm->thread, n);
-}
-
-void dst_setsize(Dst *vm, uint32_t n) {
-    DstValue *stack = dst_thread_stack(vm->thread);
-    uint32_t frameSize = dst_frame_size(stack);
-    if (frameSize < n) {
-        dst_thread_ensure_extra(vm, vm->thread, n - frameSize);
-    }
-    dst_frame_size(stack) = n;
-}
-
-void dst_swap(Dst *vm, uint32_t x, uint32_t y) {
-    DstValue oldx = dst_arg(vm, x);
-    DstValue oldy = dst_arg(vm, y);
-    dst_set_arg(vm, x, oldy);
-    dst_set_arg(vm, y, oldx);
-}
-
-void dst_move(Dst *vm, uint32_t dest, uint32_t src) {
-    dst_set_arg(vm, dest, dst_arg(vm, src));
-}
-
-void dst_nil(Dst *vm, uint32_t dest) {
-    DstValue n;
-    n.type = DST_NIL;
-    dst_set_arg(vm, dest, n);
-}
-
-void dst_true(Dst *vm, uint32_t dest) {
-    dst_set_boolean(vm, dest, 1);
-}
-
-void dst_false(Dst *vm, uint32_t dest) {
-    dst_set_boolean(vm, dest, 0);
-}
-
-/* Boolean Functions */
-void dst_set_boolean(Dst *vm, uint32_t dest, int val) {
-    DstValue n;
-    n.type = DST_BOOLEAN;
-    n.data.boolean = val;
-    dst_set_arg(vm, dest, n);
-}
-
-int dst_get_boolean(Dst *vm, uint32_t b) {
-    DstValue x = dst_arg(vm, b);
-    if (x.type != DST_BOOLEAN) {
-        return 0;
-    }
-    return x.data.boolean;
-}
-
-/* Integer functions */
-void dst_set_integer(Dst *vm, uint32_t dest, int64_t val) {
-    DstValue n;
-    n.type = DST_INTEGER;
-    n.data.integer = val;
-    dst_set_arg(vm, dest, n);
-}
-
-int64_t dst_get_integer(Dst *vm, uint32_t i) {
-    DstValue x = dst_arg(vm, i);
-    if (x.type != DST_INTEGER) {
-        return 0;
-    }
-    return x.data.integer;
-}
-
-/* Real functions */
-void dst_set_real(Dst *vm, uint32_t dest, double val) {
-    DstValue n;
-    n.type = DST_REAL;
-    n.data.real = val;
-    dst_set_arg(vm, dest, n);
-}
-
-double dst_get_real(Dst *vm, uint32_t r) {
-    DstValue x = dst_arg(vm, r);
-    if (x.type != DST_REAL) {
-        return 0.0;
-    }
-    return x.data.real;
-}
-
-/* CFunction functions */
-void dst_set_cfunction(Dst *vm, uint32_t dest, DstCFunction cfn) {
-    DstValue n;
-    n.type = DST_CFUNCTION;
-    n.data.cfunction = cfn;
-    dst_set_arg(vm, dest, n);
-}
-
-DstCFunction dst_get_cfunction(Dst *vm, uint32_t cfn) {
-    DstValue x = dst_arg(vm, cfn);
-    if (x.type != DST_CFUNCTION) {
-        return NULL;
-    }
-    return x.data.cfunction;
-}
-
-void dst_return(Dst *vm, uint32_t index) {
-    vm->ret = dst_arg(vm, index);
-}
-
-void dst_throw(Dst *vm, uint32_t index) {
-    vm->flags = 1;
-    vm->ret = dst_arg(vm, index);
-}
-
-void dst_cerr(Dst *vm, const char *message) {
-    vm->flags = 1;
-    vm->ret = dst_string_cv(vm, message);
-}
-
-int dst_checktype(Dst *vm, uint32_t n, DstType type) {
-    return dst_arg(vm, n).type == type;
+    return index;
 }

core/value.c

@@ -22,13 +22,17 @@
 
 #include "internal.h"
 
+/*
+ * Define a number of functions that can be used internally on ANY DstValue.
+ */
+
 /* Boolean truth definition */
-int dst_value_truthy(DstValue v) {
-    return v.type != DST_NIL && !(v.type == DST_BOOLEAN && !v.data.boolean);
+int dst_truthy(DstValue v) {
+    return v.type != DST_NIL && !(v.type == DST_BOOLEAN && !v.as.boolean);
 }
 
 /* Check if two values are equal. This is strict equality with no conversion. */
-int dst_value_equals(DstValue x, DstValue y) {
+int dst_equals(DstValue x, DstValue y) {
     int result = 0;
     if (x.type != y.type) {
         result = 0;
@@ -38,17 +42,17 @@ int dst_value_equals(DstValue x, DstValue y) {
             result = 1;
             break;
         case DST_BOOLEAN:
-            result = (x.data.boolean == y.data.boolean);
+            result = (x.as.boolean == y.as.boolean);
             break;
         case DST_REAL:
-            result = (x.data.real == y.data.real);
+            result = (x.as.real == y.as.real);
             break;
         case DST_INTEGER:
-            result = (x.data.integer == y.data.integer);
+            result = (x.as.integer == y.as.integer);
             break;
         default:
             /* compare pointers */
-            result = (x.data.pointer == y.data.pointer);
+            result = (x.as.pointer == y.as.pointer);
             break;
         }
     }
@@ -56,75 +60,106 @@ int dst_value_equals(DstValue x, DstValue y) {
 }
 
 /* Computes a hash value for a function */
-uint32_t dst_value_hash(DstValue x) {
+uint32_t dst_hash(DstValue x) {
     uint32_t hash = 0;
     switch (x.type) {
     case DST_NIL:
         hash = 0;
         break;
     case DST_BOOLEAN:
-        hash = x.data.boolean;
+        hash = x.as.boolean;
         break;
     case DST_STRING:
     case DST_SYMBOL:
-        hash = dst_string_hash(x.data.string);
+        hash = dst_string_hash(x.as.string);
         break;
     case DST_TUPLE:
-        hash = dst_tuple_hash(x.data.tuple);
+        hash = dst_tuple_hash(x.as.tuple);
         break;
     case DST_STRUCT:
-        hash = dst_struct_hash(x.data.st);
+        hash = dst_struct_hash(x.as.st);
         break;
     default:
         if (sizeof(double) == sizeof(void *)) {
             /* Assuming 8 byte pointer */
-            hash = x.data.dwords[0] ^ x.data.dwords[1];
+            hash = x.as.dwords[0] ^ x.as.dwords[1];
         } else {
             /* Assuming 4 byte pointer (or smaller) */
-            hash = (uint32_t) x.data.pointer;
+            hash = (uint32_t) x.as.pointer;
         }
         break;
     }
     return hash;
 }
 
+/* Computes hash of an array of values */
+uint32_t dst_calchash_array(const DstValue *array, uint32_t len) {
+    const DstValue *end = array + len;
+    uint32_t hash = 5381;
+    while (array < end)
+        hash = (hash << 5) + hash + dst_value_hash(*array++);
+    return hash;
+}
+
+/* Compare two strings */
+int dst_string_compare(const uint8_t *lhs, const uint8_t *rhs) {
+    uint32_t xlen = dst_string_length(lhs);
+    uint32_t ylen = dst_string_length(rhs);
+    uint32_t len = xlen > ylen ? ylen : xlen;
+    uint32_t i;
+    for (i = 0; i < len; ++i) {
+        if (lhs[i] == rhs[i]) {
+            continue;
+        } else if (lhs[i] < rhs[i]) {
+            return -1; /* x is less than y */
+        } else {
+            return 1; /* y is less than x */
+        }
+    }
+    if (xlen == ylen) {
+        return 0;
+    } else {
+        return xlen < ylen ? -1 : 1;
+    }
+}
+
 /* Compares x to y. If they are equal retuns 0. If x is less, returns -1.
  * If y is less, returns 1. All types are comparable
  * and should have strict ordering. */
-int dst_value_compare(DstValue x, DstValue y) {
+int dst_compare(DstValue x, DstValue y) {
     if (x.type == y.type) {
         switch (x.type) {
             case DST_NIL:
                 return 0;
             case DST_BOOLEAN:
-                if (x.data.boolean == y.data.boolean) {
+                if (x.as.boolean == y.as.boolean) {
                     return 0;
                 } else {
-                    return x.data.boolean ? 1 : -1;
+                    return x.as.boolean ? 1 : -1;
                 }
             case DST_REAL:
-                if (x.data.real == y.data.real) {
+                if (x.as.real == y.as.real) {
                     return 0;
                 } else {
-                    return x.data.real > y.data.real ? 1 : -1;
+                    return x.as.real > y.as.real ? 1 : -1;
                 }
             case DST_INTEGER:
-                if (x.data.integer == y.data.integer) {
+                if (x.as.integer == y.as.integer) {
                     return 0;
                 } else {
-                    return x.data.integer > y.data.integer ? 1 : -1;
+                    return x.as.integer > y.as.integer ? 1 : -1;
                 }
             case DST_STRING:
-                return dst_string_compare(x.data.string, y.data.string);
+                return dst_string_compare(x.as.string, y.as.string);
                 /* Lower indices are most significant */
             case DST_TUPLE:
                 {
                     uint32_t i;
-                    uint32_t xlen = dst_tuple_length(x.data.tuple);
-                    uint32_t ylen = dst_tuple_length(y.data.tuple);
+                    uint32_t xlen = dst_tuple_length(x.as.tuple);
+                    uint32_t ylen = dst_tuple_length(y.as.tuple);
                     uint32_t count = xlen < ylen ? xlen : ylen;
                     for (i = 0; i < count; ++i) {
-                        int comp = dst_value_compare(x.data.tuple[i], y.data.tuple[i]);
+                        int comp = dst_value_compare(x.as.tuple[i], y.as.tuple[i]);
                         if (comp != 0) return comp;
                     }
                     if (xlen < ylen)
@@ -135,10 +170,10 @@ int dst_value_compare(DstValue x, DstValue y) {
                 }
                 break;
             default:
-                if (x.data.string == y.data.string) {
+                if (x.as.string == y.as.string) {
                     return 0;
                 } else {
-                    return x.data.string > y.data.string ? 1 : -1;
+                    return x.as.string > y.as.string ? 1 : -1;
                 }
         }
     } else if (x.type < y.type) {
@@ -146,78 +181,3 @@ int dst_value_compare(DstValue x, DstValue y) {
     }
     return 1;
 }
-
-int dst_truthy(Dst *vm, uint32_t x) {
-    return dst_value_truthy(dst_arg(vm, x));
-}
-uint32_t dst_hash(Dst *vm, uint32_t x) {
-    return dst_value_hash(dst_arg(vm, x));
-}
-int dst_compare(Dst *vm, uint32_t x, uint32_t y) {
-    return dst_value_compare(dst_arg(vm, x), dst_arg(vm, y));
-}
-int dst_equals(Dst *vm, uint32_t x, uint32_t y) {
-    return dst_value_equals(dst_arg(vm, x), dst_arg(vm, y));
-}
-
-/* Get the length of an object. Returns errors for invalid types */
-uint32_t dst_length(Dst *vm, uint32_t n) {
-    DstValue x = dst_arg(vm, n);
-    uint32_t length;
-    switch (x.type) {
-        default:
-            vm->ret = dst_string_cv(vm, "cannot get length");
-            vm->flags = 1;
-            return 0;
-        case DST_STRING:
-            length = dst_string_length(x.data.string);
-            break;
-        case DST_ARRAY:
-            length = x.data.array->count;
-            break;
-        case DST_BYTEBUFFER:
-            length = x.data.buffer->count;
-            break;
-        case DST_TUPLE:
-            length = dst_tuple_length(x.data.tuple);
-            break;
-        case DST_STRUCT:
-            length = dst_struct_length(x.data.st);
-            break;
-        case DST_TABLE:
-            length = x.data.table->count;
-            break;
-    }
-    return length;
-}
-
-/* Get the capacity of an object. Returns errors for invalid types */
-uint32_t dst_capacity(Dst *vm, uint32_t n) {
-    DstValue x = dst_arg(vm, n);
-    uint32_t cap;
-    switch (x.type) {
-        default:
-            vm->ret = dst_string_cv(vm, "cannot get capacity");
-            vm->flags = 1;
-            return 0;
-        case DST_STRING:
-            cap = dst_string_length(x.data.string);
-            break;
-        case DST_ARRAY:
-            cap = x.data.array->capacity;
-            break;
-        case DST_BYTEBUFFER:
-            cap = x.data.buffer->capacity;
-            break;
-        case DST_TUPLE:
-            cap = dst_tuple_length(x.data.tuple);
-            break;
-        case DST_STRUCT:
-            cap = dst_struct_length(x.data.st);
-            break;
-        case DST_TABLE:
-            cap = x.data.table->capacity;
-            break;
-    }
-    return cap;
-}

core/vm.c

@@ -21,6 +21,7 @@
 */
 
 #include "internal.h"
+#include "wrap.h"
 
 static const char DST_NO_UPVALUE[] = "no upvalue";
 static const char DST_EXPECTED_FUNCTION[] = "expected function";
@@ -54,14 +55,14 @@ int dst_continue(Dst *vm) {
 
         case DST_OP_FLS: /* Load False */
             temp.type = DST_BOOLEAN;
-            temp.data.boolean = 0;
+            temp.as.boolean = 0;
             stack[pc[1]] = temp;
             pc += 2;
             continue;
 
         case DST_OP_TRU: /* Load True */
             temp.type = DST_BOOLEAN;
-            temp.data.boolean = 1;
+            temp.as.boolean = 1;
             stack[pc[1]] = temp;
             pc += 2;
             continue;
@@ -74,7 +75,7 @@ int dst_continue(Dst *vm) {
 
         case DST_OP_I16: /* Load Small Integer */
             temp.type = DST_INTEGER;
-            temp.data.integer = ((int16_t *)(pc))[2];
+            temp.as.integer = ((int16_t *)(pc))[2];
             stack[pc[1]] = temp;
             pc += 3;
             continue;
@@ -88,7 +89,7 @@ int dst_continue(Dst *vm) {
                 uint16_t level = pc[2];
                 temp = dst_frame_callee(stack);
                 dst_assert(vm, temp.type == DST_FUNCTION, DST_EXPECTED_FUNCTION);
-                fn = temp.data.function;
+                fn = temp.as.function;
                 if (level == 0)
                     upv = stack + pc[3];
                 else {
@@ -125,29 +126,29 @@ int dst_continue(Dst *vm) {
         case DST_OP_CST: /* Load constant value */
             v1 = dst_frame_callee(stack);
             dst_assert(vm, v1.type == DST_FUNCTION, DST_EXPECTED_FUNCTION);
-            if (pc[2] > v1.data.function->def->literalsLen)
+            if (pc[2] > v1.as.function->def->literalsLen)
                 dst_error(vm, DST_NO_UPVALUE);
-            stack[pc[1]] = v1.data.function->def->literals[pc[2]];
+            stack[pc[1]] = v1.as.function->def->literals[pc[2]];
             pc += 3;
             continue;
 
         case DST_OP_I32: /* Load 32 bit integer */
             temp.type = DST_INTEGER;
-            temp.data.integer = *((int32_t *)(pc + 2));
+            temp.as.integer = *((int32_t *)(pc + 2));
             stack[pc[1]] = temp;
             pc += 4;
             continue;
 
         case DST_OP_I64: /* Load 64 bit integer */
             temp.type = DST_INTEGER;
-            temp.data.integer = (DstInteger) *((int64_t *)(pc + 2));
+            temp.as.integer = *((int64_t *)(pc + 2));
             stack[pc[1]] = temp;
             pc += 6;
             continue;
 
         case DST_OP_F64: /* Load 64 bit float */
             temp.type = DST_REAL;
-            temp.data.real = (DstReal) *((double *)(pc + 2));
+            temp.as.real = *((double *)(pc + 2));
             stack[pc[1]] = temp;
             pc += 6;
             continue;
@@ -161,31 +162,32 @@ int dst_continue(Dst *vm) {
             {
                 DstFunction *fn;
                 v1 = dst_frame_callee(stack);
-                temp = v1.data.function->def->literals[pc[2]];
+                temp = v1.as.function->def->literals[pc[2]];
                 if (temp.type != DST_FUNCDEF)
                     dst_error(vm, "cannot create closure from non-funcdef");
-                fn = dst_alloc(vm, sizeof(DstFunction));
-                fn->def = temp.data.def;
-                if (temp.data.def->flags & DST_FUNCDEF_FLAG_NEEDSPARENT)
-                    fn->parent = v1.data.function;
+                fn = dst_mem_resumegc(dst_alloc(vm, sizeof(DstFunction)));
+                fn->def = temp.as.def;
+                /* Don't always set the parent. We might want to let the gc get it */
+                if (temp.as.def->flags & DST_FUNCDEF_FLAG_NEEDSPARENT)
+                    fn->parent = v1.as.function;
                 else
                     fn->parent = NULL;
                 if (v1.type != DST_FUNCTION)
                     dst_error(vm, DST_EXPECTED_FUNCTION);
                 if (dst_frame_env(stack) == NULL && (fn->def->flags & DST_FUNCDEF_FLAG_NEEDSENV)) {
-                    dst_frame_env(stack) = dst_alloc(vm, sizeof(DstFuncEnv));
+                    dst_frame_env(stack) = dst_mem_resumegc(dst_alloc(vm, sizeof(DstFuncEnv)));
                     dst_frame_env(stack)->thread = vm->thread;
                     dst_frame_env(stack)->stackOffset = vm->thread->count;
                     dst_frame_env(stack)->values = NULL;
                 }
-                if (pc[2] > v1.data.function->def->literalsLen)
+                if (pc[2] > v1.as.function->def->literalsLen)
                     dst_error(vm, DST_NO_UPVALUE);
                 if (fn->def->flags & DST_FUNCDEF_FLAG_NEEDSENV)
                     fn->env = dst_frame_env(stack);
                 else
                     fn->env = NULL;
                 temp.type = DST_FUNCTION;
-                temp.data.function = fn;
+                temp.as.function = fn;
                 stack[pc[1]] = temp;
                 pc += 3;
             }
@@ -224,11 +226,11 @@ int dst_continue(Dst *vm) {
                 const DstValue *data;
                 temp = stack[pc[1]];
                 if (temp.type == DST_TUPLE) {
-                    count = dst_tuple_length(temp.data.tuple);
-                    data = temp.data.tuple;
+                    count = dst_tuple_length(temp.as.tuple);
+                    data = temp.as.tuple;
                 } else if (temp.type == DST_ARRAY){
-                    count = temp.data.array->count;
-                    data = temp.data.array->data;
+                    count = temp.as.array->count;
+                    data = temp.as.array->data;
                 } else {
                     dst_error(vm, "expected array or tuple");
                 }
@@ -271,7 +273,7 @@ int dst_continue(Dst *vm) {
                     DstFuncEnv *env = dst_frame_env(stack);
                     env->thread = NULL;
                     env->stackOffset = size;
-                    env->values = dst_alloc(vm, sizeof(DstValue) * size);
+                    env->values = dst_mem_resumegc(dst_alloc(vm, sizeof(DstValue) * size));
                     dst_memcpy(env->values, stack, sizeof(DstValue) * size);
                 }
                 if (newStackIndex)
@@ -290,11 +292,11 @@ int dst_continue(Dst *vm) {
             stack = vm->thread->data + vm->thread->count;
             temp = dst_frame_callee(stack);
             if (temp.type == DST_FUNCTION) {
-                pc = temp.data.function->def->byteCode;
+                pc = temp.as.function->def->byteCode;
             } else if (temp.type == DST_CFUNCTION) {
                 int status;
                 vm->ret.type = DST_NIL;
-                status = temp.data.cfunction(vm);
+                status = temp.as.cfunction(vm);
                 if (status) {
                     goto vm_error;
                 } else {
@@ -315,7 +317,7 @@ int dst_continue(Dst *vm) {
                 for (i = 0; i < arrayLen; ++i)
                     array->data[i] = stack[pc[3 + i]];
                 temp.type = DST_ARRAY;
-                temp.data.array = array;
+                temp.as.array = array;
                 stack[pc[1]] = temp;
                 pc += 3 + arrayLen;
             }
@@ -326,6 +328,8 @@ int dst_continue(Dst *vm) {
                 uint32_t i = 3;
                 uint32_t kvs = pc[2];
                 DstTable *t = dst_make_table(vm, 2 * kvs);
+                dst_mem_suspendgc(t);
+                dst_mem_suspendgc(t->data);
                 kvs = kvs + 3;
                 while (i < kvs) {
                     v1 = stack[pc[i++]];
@@ -333,8 +337,10 @@ int dst_continue(Dst *vm) {
                     dst_table_put(vm, t, v1, v2);
                 }
                 temp.type = DST_TABLE;
-                temp.data.table = t;
+                temp.as.table = t;
                 stack[pc[1]] = temp;
+                dst_mem_resumegc(t);
+                dst_mem_resumegc(t->data);
                 pc += kvs;
             }
             break;
@@ -347,7 +353,7 @@ int dst_continue(Dst *vm) {
                 for (i = 0; i < len; ++i)
                     tuple[i] = stack[pc[3 + i]];
                 temp.type = DST_TUPLE;
-                temp.data.tuple = dst_tuple_end(vm, tuple);
+                temp.as.tuple = dst_tuple_end(vm, tuple);
                 stack[pc[1]] = temp;
                 pc += 3 + len;
             }
@@ -360,10 +366,10 @@ int dst_continue(Dst *vm) {
                 dst_error(vm, "expected thread");
             if (temp.type == DST_NIL && vm->thread->parent) {
                 temp.type = DST_THREAD;
-                temp.data.thread = vm->thread->parent;
+                temp.as.thread = vm->thread->parent;
             }
             if (temp.type == DST_THREAD) {
-                if (temp.data.thread->status != DST_THREAD_PENDING)
+                if (temp.as.thread->status != DST_THREAD_PENDING)
                     dst_error(vm, "can only enter pending thread");
             }
             dst_frame_ret(stack) = pc[1];
@@ -373,9 +379,9 @@ int dst_continue(Dst *vm) {
                 vm->ret = v1;
                 return 0;
             }
-            temp.data.thread->status = DST_THREAD_ALIVE;
-            vm->thread = temp.data.thread;
-            stack = dst_thread_stack(temp.data.thread);
+            temp.as.thread->status = DST_THREAD_ALIVE;
+            vm->thread = temp.as.thread;
+            stack = dst_thread_stack(temp.as.thread);
             if (dst_frame_callee(stack).type != DST_FUNCTION)
                 goto vm_return;
             stack[dst_frame_ret(stack)] = v1;
@@ -455,7 +461,7 @@ int dst_run(Dst *vm, DstValue callee) {
     }
     if (callee.type == DST_CFUNCTION) {
         vm->ret.type = DST_NIL;
-        result = callee.data.cfunction(vm);
+        result = callee.as.cfunction(vm);
     } else if (callee.type == DST_FUNCTION) {
         result = dst_continue(vm);
     } else {
@@ -475,7 +481,10 @@ int dst_run(Dst *vm, DstValue callee) {
 
 /* Setup functions */
 Dst *dst_init() {
-    Dst *vm = dst_raw_alloc(sizeof(Dst));
+    Dst *vm = malloc(sizeof(Dst));
+    if (NULL == vm) {
+        DST_OUT_OF_MEMORY;
+    }
     vm->ret.type = DST_NIL;
     /* Garbage collection */
     vm->blocks = NULL;
@@ -485,9 +494,8 @@ Dst *dst_init() {
      * horrible for performance, but helps ensure
      * there are no memory bugs during dev */
     vm->memoryInterval = 0;
-    vm->black = 0;
     /* Set up the cache */
-    vm->cache = dst_raw_calloc(1, 128 * sizeof(DstValue));
+    vm->cache = calloc(1, 128 * sizeof(DstValue));
     vm->cache_capacity = vm->cache == NULL ? 0 : 128;
     vm->cache_count = 0;
     vm->cache_deleted = 0;
@@ -509,11 +517,11 @@ void dst_deinit(Dst *vm) {
     vm->registry = NULL;
     vm->ret.type = DST_NIL;
     /* Deinit the cache */
-    dst_raw_free(vm->cache);
+    free(vm->cache);
     vm->cache = NULL;
     vm->cache_count = 0;
     vm->cache_capacity = 0;
     vm->cache_deleted = 0;
     /* Free the vm */
-    dst_raw_free(vm);
+    free(vm);
 }

core/wrap.c

@@ -20,37 +20,34 @@
 * IN THE SOFTWARE.
 */
 
-#include <dst/dst.h>
-#include "internal.h"
+#include "wrap.h"
 
 /* Wrapper functions wrap a data type that is used from C into a
- * gst value, which can then be used in gst. */
+ * dst value, which can then be used in dst internal functions. Use
+ * these functions sparingly, as these function will let the programmer
+ * leak memory, where as the stack based API ensures that all values can
+ * be collected by the garbage collector. */
 
 DstValue dst_wrap_nil() {
-    GstValue y;
-    y.type = GST_NIL;
+    DstValue y;
+    y.type = DST_NIL;
     return y;
 }
 
-int dst_check_nil(Gst *vm, uint32_t i) {
-    DstValue a = dst_arg(vm, i);
-    return a.type == DST_NIL;
-}
-
 #define DST_WRAP_DEFINE(NAME, TYPE, DTYPE, UM)\
 DstValue dst_wrap_##NAME(TYPE x) {\
     DstValue y;\
     y.type = DTYPE;\
-    y.data.UM = x;\
+    y.as.UM = x;\
     return y;\
 }\
 \
-int dst_check_##NAME(Dst *vm, uint32_t i) {\
-    return dst_arg(vm, i).type == DTYPE;\
+TYPE dst_unwrap_##NAME(Dst *vm, int64_t i) {\
+    return dst_getv(vm, i).as.UM;\
 }\
 
-DST_WRAP_DEFINE(real, DstReal, DST_REAL, real)
-DST_WRAP_DEFINE(integer, DstInteger, DST_INTEGER, integer)
+DST_WRAP_DEFINE(real, double, DST_REAL, real)
+DST_WRAP_DEFINE(integer, int64_t, DST_INTEGER, integer)
 DST_WRAP_DEFINE(boolean, int, DST_BOOLEAN, boolean)
 DST_WRAP_DEFINE(string, const uint8_t *, DST_STRING, string)
 DST_WRAP_DEFINE(symbol, const uint8_t *, DST_SYMBOL, string)
@@ -58,27 +55,10 @@ DST_WRAP_DEFINE(array, DstArray *, DST_ARRAY, array)
 DST_WRAP_DEFINE(tuple, const DstValue *, DST_TUPLE, tuple)
 DST_WRAP_DEFINE(struct, const DstValue *, DST_STRUCT, st)
 DST_WRAP_DEFINE(thread, DstThread *, DST_THREAD, thread)
-DST_WRAP_DEFINE(buffer, DstBuffer *, DST_BYTEBUFFER, buffer)
+DST_WRAP_DEFINE(buffer, DstBuffer *, DST_BUFFER, buffer)
 DST_WRAP_DEFINE(function, DstFunction *, DST_FUNCTION, function)
 DST_WRAP_DEFINE(cfunction, DstCFunction, DST_CFUNCTION, cfunction)
 DST_WRAP_DEFINE(table, DstTable *, DST_TABLE, table)
-DST_WRAP_DEFINE(funcenv, DstFuncEnv *, DST_FUNCENV, env)
-DST_WRAP_DEFINE(funcdef, DstFuncDef *, DST_FUNCDEF, def)
+DST_WRAP_DEFINE(userdata, void *, DST_USERDATA, pointer)
 
 #undef DST_WRAP_DEFINE
-
-DstValue dst_wrap_userdata(void *x) {
-    DstValue ret;
-    ret.type = DST_USERDATA;
-    ret.data.pointer = x;
-    return ret;
-}
-
-void *dst_check_userdata(Dst *vm, uint32_t i, const DstUserType *type) {
-    DstValue x = dst_arg(vm, i);
-    DstUserdataHeader *h;
-    if (x.type != DST_USERDATA) return NULL;
-    h = dst_udata_header(x.data.pointer);
-    if (h->type != type) return NULL;
-    return x.data.pointer;
-}

core/wrap.h

@@ -0,0 +1,61 @@
+/*
+* Copyright (c) 2017 Calvin Rose
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to
+* deal in the Software without restriction, including without limitation the
+* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+* sell copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+* IN THE SOFTWARE.
+*/
+
+#ifndef DST_WRAP_H_defined
+#define DST_WRAP_H_defined
+
+#include <dst/dst.h>
+
+/* Wrap data in DstValue */
+DstValue dst_wrap_nil();
+DstValue dst_wrap_real(double x);
+DstValue dst_wrap_integer(int64_t x);
+DstValue dst_wrap_boolean(int x);
+DstValue dst_wrap_string(const uint8_t *x);
+DstValue dst_wrap_symbol(const uint8_t *x);
+DstValue dst_wrap_array(DstArray *x);
+DstValue dst_wrap_tuple(const DstValue *x);
+DstValue dst_wrap_struct(const DstValue *x);
+DstValue dst_wrap_thread(DstThread *x);
+DstValue dst_wrap_buffer(DstBuffer *x);
+DstValue dst_wrap_function(DstFunction *x);
+DstValue dst_wrap_cfunction(DstCFunction x);
+DstValue dst_wrap_table(DstTable *x);
+DstValue dst_wrap_userdata(void *x);
+
+/* Unwrap values */
+double dst_unwrap_real(Dst *vm, int64_t i);
+int64_t dst_unwrap_integer(Dst *vm, int64_t i);
+int dst_unwrap_boolean(Dst *vm, int64_t i);
+const uint8_t *dst_unwrap_string(Dst *vm, int64_t i);
+const uint8_t *dst_unwrap_symbol(Dst *vm, int64_t i);
+DstArray *dst_unwrap_array(Dst *vm, int64_t i);
+const DstValue *dst_unwrap_tuple(Dst *vm, int64_t i);
+const DstValue *dst_unwrap_struct(Dst *vm, int64_t i);
+DstThread *dst_unwrap_thread(Dst *vm, int64_t i);
+DstBuffer *dst_unwrap_buffer(Dst *vm, int64_t i);
+DstFunction *dst_unwrap_function(Dst *vm, int64_t i);
+DstCFunction dst_unwrap_cfunction(Dst *vm, int64_t i);
+DstTable *dst_unwrap_table(Dst *vm, int64_t i);
+void *dst_unwrap_userdata(Dst *vm, int64_t i);
+
+#endif

libs/sample.dsts

@@ -0,0 +1,68 @@
+# A .dsts file will contain VM, assembly for a dst function. This includes
+# associated constants and what not. The assembler should be able to
+# simply construct a FuncDef from this file. This file is also parsed
+# in the same markup as dst itself (extended S expressions)
+{
+    stack 10
+    arity 3
+    signature (integer integer integer integer)
+    source "source file path"
+    varargs false
+# Name for reference by nested funcdefs
+    name outerfunc
+# Contains the bytecode for this function. This can be assembly
+# instructions or integers. Assembly will be converted to integer bytecodes immediately.
+    bytecode [
+        (typecheck 0 integer)
+        (typecheck 1 integer)
+        (typecheck 2 integer)
+        :checked-entry
+        (load-constant 3 bork)
+        (load-constant 4 bip)
+        (add-integer-unchecked 5 0 1)
+        (add-integer-unchecked 5 5 2)
+        (add-integer-unchecked 5 5 3)
+        (add-integer-unchecked 5 5 4)
+        (return 5)
+    ]
+# A source map is optional. The sourcemap corresponds with the byte code.
+# Each number is a 64 bit integer, the concatenation of two 32 bit integers.
+# These integers represent source line, source column for each instruction.
+# This format may change.
+     source-map [
+        0x0000123400000123
+        0x0000123400000123
+        0x0000123400000125
+        0x0000123400000134
+        0x0000123400000134
+        0x0000123400000135
+        ...
+     ]
+# Slots can be named as well for convenience.
+     slots [
+        x
+        y
+        z
+     ]
+# Captured outer environments that are referenced
+     environments [
+        outer1
+        outer2
+     ]
+# Constants are an array or tuple. For named constants, use a tuple that begins with let
+# For a literal tuple, use (quote tuple), or 'tuple. Without names, constants must be indexed
+# from their number
+     constants [
+         "hello"
+         (let bork 123)
+         (let bip 456)
+         '(1 2 3)
+         (let atuple (1 2 3))
+         (567)
+# Functions can be recursively defined.
+         (funcdef funcname {
+            ...
+         })
+# Literal FuncEnvs and Functions may be possible later
+     ]
+}