Work on serialization. Move clibs into core.

Makefile

@@ -7,9 +7,8 @@ CFLAGS=-std=c99 -Wall -Wextra -Wpedantic -g -I./include
 PREFIX=/usr/local
 GST_TARGET=client/gst
 GST_CORELIB=core/libgst.a
-GST_AUXLIB=clibs/libgstaux.a
 GST_HEADERS=$(addprefix include/gst/,\
-		vm.h ds.h value.h datatypes.h gc.h util.h gst.h stl.h thread.h)
+		vm.h ds.h value.h datatypes.h gc.h util.h gst.h stl.h thread.h serialize.h)
 
 all: $(GST_TARGET)
 
@@ -17,27 +16,19 @@ all: $(GST_TARGET)
 ##### The core vm and runtime #####
 ###################################
 GST_CORE_SOURCES=$(addprefix core/,\
-				 value.c vm.c ds.c gc.c thread.c)
+				 compile.c disasm.c parse.c stl.c\
+				 value.c vm.c ds.c gc.c thread.c serialize.c)
 GST_CORE_OBJECTS=$(patsubst %.c,%.o,$(GST_CORE_SOURCES))
 $(GST_CORELIB): $(GST_CORE_OBJECTS) $(GST_HEADERS)
 	ar rcs $(GST_CORELIB) $(GST_CORE_OBJECTS)
 
-#################################
-##### The auxiliary library #####
-#################################
-GST_AUX_SOURCES=$(addprefix clibs/,\
-				compile.c disasm.c parse.c stl.c)
-GST_AUX_OBJECTS=$(patsubst %.c,%.o,$(GST_AUX_SOURCES))
-$(GST_AUXLIB): $(GST_AUX_OBJECTS) $(GST_HEADERS)
-	ar rcs $(GST_AUXLIB) $(GST_AUX_OBJECTS)
-
 ##############################
 ##### The example client #####
 ##############################
 GST_CLIENT_SOURCES=client/main.c
 GST_CLIENT_OBJECTS=$(patsubst %.c,%.o,$(GST_CLIENT_SOURCES))
-$(GST_TARGET): $(GST_CLIENT_OBJECTS) $(GST_HEADERS) $(GST_AUXLIB) $(GST_CORELIB)
+$(GST_TARGET): $(GST_CLIENT_OBJECTS) $(GST_HEADERS) $(GST_CORELIB)
-	$(CC) $(CFLAGS) -o $(GST_TARGET) $(GST_CLIENT_OBJECTS) $(GST_CORELIB) $(GST_AUXLIB)
+	$(CC) $(CFLAGS) -o $(GST_TARGET) $(GST_CLIENT_OBJECTS) $(GST_CORELIB)
 
 # Compile all .c to .o
 %.o : %.c $(GST_HEADERS)
@@ -55,9 +46,7 @@ valgrind: $(GST_TARGET)
 clean:
 	rm $(GST_TARGET) || true
 	rm $(GST_CORELIB) || true
-	rm $(GST_AUXLIB) || true
 	rm $(GST_CORE_OBJECTS) || true
-	rm $(GST_AUX_OBJECTS) || true
 	rm $(GST_CLIENT_OBJECTS) || true
 
 .PHONY: clean install run debug valgrind

core/compile.c

@@ -597,6 +597,7 @@ MAKE_SPECIAL(gte, 1, 1, GST_OP_LTE, -1, OP_0_BOOLEAN | OP_1_BOOLEAN | OP_REVERSE
 MAKE_SPECIAL(not, -1, GST_OP_NOT, -1, -1, 0)
 MAKE_SPECIAL(get, -1, -1, GST_OP_GET, -1, 0)
 MAKE_SPECIAL(make_tuple, -1, -1, -1, GST_OP_TUP, 0)
+MAKE_SPECIAL(length, -1, GST_OP_LEN, -1, -1, 0)
 
 #undef MAKE_SPECIAL
 
@@ -1030,6 +1031,7 @@ static SpecialFormHelper get_special(GstValue *form) {
             case '>': return compile_gt;
             case '<': return compile_lt;
             case '=': return compile_equals;
+            case ':': return compile_var;
             default:
                 break;
         }
@@ -1094,6 +1096,17 @@ static SpecialFormHelper get_special(GstValue *form) {
                 }
             }
             break;
+        case 'l':
+            {
+                if (gst_string_length(name) == 6 &&
+                        name[1] == 'e' &&
+                        name[2] == 'n' &&
+                        name[3] == 'g' &&
+                        name[4] == 't' &&
+                        name[5] == 'h') {
+                    return compile_length;
+                }
+            }
         case 'n':
             {
                 if (gst_string_length(name) == 3 &&
@@ -1148,14 +1161,6 @@ static SpecialFormHelper get_special(GstValue *form) {
                 }
             }
             break;
-        case ':':
-            {
-                if (gst_string_length(name) == 2 &&
-                        name[1] == '=') {
-                    return compile_var;
-                }
-            }
-            break;
         default:
             break;
     }

core/deserialize.c

@@ -1,382 +0,0 @@
-#include <gst/datatypes.h>
-#include <gst/vm.h>
-#include <gst/values.h>
-#include <gst/ds.h>
-#include <gst/util.h>
-
-/**
- * Data format
- * State is encoded as a string of unsigned bytes.
- *
- * Types:
- *
- * Byte 0 to 200: small integer byte - 100
- * Byte 201: Nil
- * Byte 202: True
- * Byte 203: False
- * Byte 204: Number  - double format
- * Byte 205: String  - [u32 length]*[u8... characters]
- * Byte 206: Symbol  - [u32 length]*[u8... characters]
- * Byte 207: Buffer  - [u32 length]*[u8... characters]
- * Byte 208: Array   - [u32 length]*[value... elements]
- * Byte 209: Tuple   - [u32 length]*[value... elements]
- * Byte 210: Thread  - [u8 state][u32 frames]*[[value callee][value env]
- *  [u32 pcoffset][u32 erroffset][u16 ret][u16 errloc][u16 size]*[value ...stack]
- * Byte 211: Object  - [value meta][u32 length]*2*[value... kvs]
- * Byte 212: FuncDef - [u32 locals][u32 arity][u32 flags][u32 literallen]*[value...
- *  literals][u32 bytecodelen]*[u16... bytecode]
- * Byte 213: FunEnv  - [value thread][u32 length]*[value ...upvalues]
- *  (upvalues is not read if thread is a thread object)
- * Byte 214: Func    - [value parent][value def][value env]
- *  (nil values indicate empty)
- * Byte 215: LUdata  - [value meta][u32 length]*[u8... bytes]
- * Byte 216: CFunc   - [u32 length]*[u8... idstring]
- * Byte 217: Ref     - [u32 id]
- */
-
-/* Error at buffer end */
-static const char UEB[] = "unexpected end of buffer";
-
-/* Read 4 bytes as an unsigned integer */
-static uint32_t bytes2u32(uint8_t bytes[4]) {
-    union {
-        uint8_t bytes[4];
-        uint32_t u32;
-    } u;
-    u.bytes = bytes;
-    return u.u32;
-}
-
-/* Read 2 bytes as unsigned short */
-static uint16_t bytes2u16(uint8_t bytes[2]) {
-    union {
-        uint8_t bytes[2];
-        uint16_t u16;
-    } u;
-    u.bytes = bytes;
-    return u.u16;
-}
-
-/* Read 8 bytes as a double */
-static uint32_t bytes2dbl(uint8_t bytes[8]) {
-    union {
-        uint8_t bytes[8];
-        double dbl;
-    } u;
-    u.bytes = bytes;
-    return u.dbl;
-}
-
-/* Read a string and turn it into a gst value */
-static GstValue gst_deserialize_impl(
-        Gst *vm, 
-        uint8_t *data,
-        uint8_t *end,
-        uint8_t **newData,
-        GstArray *visited) {
-
-    GstValue ret;
-    ret.type = GST_NIL;
-    GstValue *buffer;
-    uint32_t length, i;
-
-    /* Handle errors as needed */
-#define deser_error(e) (exit(-1), NULL)
-
-    /* Assertions */
-#define deser_assert(c, e) do{if(!(c))deser_error(e);}while(0)
-
-    /* Assert enough buffer */
-#define deser_datacheck(len) (end - data < (len) ? deser_error(UEB) : data)
-
-    /* Check for enough space to read uint32_t */
-#define read_u32() (deser_datacheck(4), bytes2u32(data))
-
-    /* Check for enough space to read uint16_t */
-#define read_u16() (deser_datacheck(2), bytes2u16(data))
-
-    /* Check for enough space to read uint32_t */
-#define read_dbl() (deser_datacheck(8), bytes2dbl(data))
-
-    /* Check enough buffer left to read one byte */
-    if (data >= end) deser_error(UEB);
-
-    /* Small integer */
-    if (*data < 201) {
-        ret.type = GST_NUMBER;
-        ret.data.number = *data - 100; 
-        newData = data + 1;
-        return ret;
-    } 
-
-    /* Main switch for types */
-    switch (*data++) {
-
-        case 201: /* Nil */
-            ret.type = GST_NIL;
-            *newData = data;
-            return ret;
-
-        case 202: /* True */
-            ret.type = GST_BOOLEAN;
-            ret.data.boolean = 1;
-            *newData = data;
-            return ret;
-
-        case 203: /* False */
-            ret.type = GST_BOOLEAN;
-            ret.data.boolean = 0;
-            *newData = data;
-            return ret;
-
-        case 204: /* Long number (double) */
-            ret.type = GST_NUMBER; 
-            ret.data.number = read_dbl();
-            *newData = data + 8;
-            return ret;
-
-        case 205: /* String */
-        case 206: /* Symbol */
-            ret.type = data[-1] == 205 ? GST_STRING : GST_SYMBOL;
-            length = read_u32(); data += 4;
-            data = deser_datacheck(length);
-            ret.data.string = 
-                gst_alloc(vm, 2 * sizeof(uint32_t) + length + 1) + 2 * sizeof(uint32_t);
-            gst_string_length(ret.data.string) = length;
-            gst_string_hash(ret.data.string) = 0;
-            gst_memcpy(ret.data.string, data, length);
-            ret.data.string[length] = 0;
-            *newData = data + length;
-            gst_array_push(vm, visited, ret);
-            return ret;
-
-        case 207: /* Buffer */
-            ret.type = GST_BUFFER;
-            length = read_u32(); data += 4;
-            data = deser_datacheck(length);
-            ret.data.buffer = gst_alloc(vm, sizeof(GstBuffer));
-            ret.data.buffer->data = gst_alloc(vm, length);
-            gst_memcpy(ret.data.string, data, length;
-            ret.data.buffer->count = length;
-            ret.data.buffer->capacity = length;
-            *newData = data + length;
-            gst_array_push(vm, visited, ret);
-            return ret;
-
-        case 208: /* Array */
-            ret.type = GST_ARRAY;
-            length = read_u32(); data += 4;
-            buffer = gst_alloc(vm, length * sizeof(GstValue));
-            for (i = 0; i < length; ++i)
-                buffer[i] = gst_deserialize_impl(vm, data, end, &data, visited);
-            ret.data.array = gst_alloc(vm, sizeof(GstArray));
-            ret.data.array->data = buffer;
-            ret.data.array->count = length;
-            ret.data.array->capacity = length;
-            *newData = data;
-            gst_array_push(vm, visited, ret);
-            return ret;
-
-        case 209: /* Tuple */
-            ret.type = GST_TUPLE;
-            length = read_u32(); data += 4;
-            buffer = gst_alloc(vm, length * sizeof(GstValue) + 2 * sizeof(uint32_t))
-                + 2 * sizeof(uint32_t);
-            for (i = 0; i < length; ++i)
-                buffer[i] = gst_deserialize_impl(vm, data, end, &data, visited);
-            gst_tuple_hash(buffer) = 0;
-            gst_tuple_length(buffer) = length;
-            ret.data.tuple = buffer;
-            *newData = data;
-            gst_array_push(vm, visited, ret);
-            return ret;
-
-        case 210: /* Thread */
-            {
-                GstValue nil;
-                GstThread *t;
-                GstValue *stack;
-                uint16_t prevsize = 0;
-                uint8 statusbyte;
-                nil.type = GST_NIL;
-                t = gst_thread(vm, 64, nil);
-                ret.type = GST_THREAD;
-                ret.data.thread = t;
-                deser_assert(data < end, UEB);
-                statusbyte = *data++;
-                length = read_u32(); data += 4;
-                /* Check for empty thread */
-                if (length == 0) {
-                    *newData = data;
-                    return ret;
-                } 
-                /* Set status */
-                if (statusbyte == 0) t->status = GST_THREAD_PENDING;
-                else if (statusbyte == 1) t->status = GST_THREAD_ALIVE;
-                else t->status = GST_THREAD_DEAD;
-                /* Add frames */
-                for (i = 0; i < length; ++i) {
-                    GstValue callee, env;
-                    uint32_t pcoffset, erroffset;
-                    uint16_t ret, errloc, size, j;
-                    /* Create a new frame */
-                    if (i > 0)
-                        gst_thread_beginframe(vm, t, nil, 0);
-                    /* Read the stack */
-                    callee = gst_deserialize_impl(vm, data, end, &data, visited);
-                    env = gst_deserialize_impl(vm, data, end, &data, visited);
-                    pcoffset = read_u32(); data += 4;
-                    erroffset = read_u32(); data += 4;
-                    ret = read_u16(); data += 2;
-                    errloc = read_u16(); data += 2;
-                    size = read_u16(); data += 2;
-                    /* Set up the stack */
-                    stack = gst_thread_stack(t);
-                    if (callee.type = GST_FUNCTION) {
-                        gst_frame_pc(stack) = callee.data.function->def->byteCode + pcoffset;
-                        gst_frame_errjmp(stack) = callee.data.function->def->byteCode + erroffset;
-                        if (env.type == GST_FUNCENV)
-                            gst_frame_env(stack) = env.data.env;
-                    }
-                    gst_frame_ret(stack) = ret;
-                    gst_frame_errloc(stack) = errloc;
-                    gst_frame_size(stack) = size;
-                    gst_frame_prevsize(stack) = prevsize;
-                    prevsize = size;
-                    /* Push stack args */
-                    for (j = 0; j < size; ++j) {
-                        GstValue temp = gst_deserialize_impl(vm, data, end, &data, visited);
-                        gst_thread_push(vm, t, temp);
-                    }
-                }
-                *newData = data;
-            }
-            return ret;
-
-        case 211: /* Object */
-            {
-                GstValue meta;
-                ret.type = GST_OBJECT;
-                ret.data.object = gst_object(vm, 10);
-                meta = gst_deserialize_impl(vm, data, end, &data, visited);
-                length = read_u32(); data += 4;
-                for (i = 0; i < length; i += 2) {
-                    GstValue key, value;
-                    key = gst_deserialize_impl(vm, data, end, &data, visited);
-                    value = gst_deserialize_impl(vm, data, end, &data, visited);
-                    gst_object_put(vm, ret.data.object, key, value);
-                }
-                if (meta.type == GST_OBJECT) {
-                    ret.data.object->meta = meta.data.object; 
-                }
-                *newData = data;
-                gst_array_push(vm, visited, ret);
-            }
-            return ret;
-
-        case 212: /* Funcdef */
-            {
-                GstFuncDef *def;
-                uint32_t locals, arity, literalsLen, byteCodeLen, flags;
-                locals = read_u32(); data += 4;
-                arity = read_u32(); data += 4;
-                flags = read_u32(); data += 4;
-                literalsLen = read_u32(); data += 4;
-                def = gst_alloc(vm, sizeof(GstFuncDef));
-                ret.type = GST_FUNCDEF;
-                ret.data.def = def;
-                def->locals = locals;
-                def->arity = arity;
-                def->flags = flags;
-                def->literalsLen = literalsLen;
-                if (literalsLen > 0) {
-                    def->literals = gst_alloc(vm, literalsLen * sizeof(GstValue));
-                }
-                for (i = 0; i < literalsLen; ++i) {
-                    def->literals[i] = gst_deserialize_impl(vm, data, end, &data, visited);
-                }
-                byteCodeLen = read_u32(); data += 4;
-                deser_datacheck(byteCodeLen);
-                def->byteCode = vm_alloc(vm, byteCodeLen * sizeof(uint16_t));
-                def->byteCodeLen = byteCodeLen;
-                for (i = 0; i < byteCodeLen; ++i) {
-                    def->byteCode[i] = read_u16();
-                    data += 2;    
-                }
-                *newData = data;
-                gst_array_push(vm, visited, ret);
-            }
-            return ret;
-
-        case 213: /* Funcenv */
-            {
-                GstValue thread = gst_deserialize_impl(vm, deata, &data, visited);
-                length = read_u32(); data += 4;
-                ret.type = GST_FUNCENV;
-                ret.data.env = gst_alloc(vm, sizeof(GstFuncEnv));
-                ret.data.env->stackOffset = length;
-                if (thread.type == GST_THREAD) {
-                    ret.data.env->thread = thread.data.thread;
-                } else {
-                    ret.data.env->thread = NULL;
-                    ret.data.env->values = vm_alloc(vm, sizeof(GstValue) * length);
-                    for (i = 0; i < length; ++i) {
-                        GstValue item = gst_deserialize_impl(vm, data, end, &data, visited);
-                        ret.data.env->values[i] = item;
-                    }
-                }
-                *newData = data;
-                gst_array_push(vm, visited, ret);
-            }
-            return ret;
-
-        case 214: /* Function */
-            {
-                GstValue parent, def, env;
-                parent = gst_deserialize_impl(vm, data, end, &data, visited);
-                def = gst_deserialize_impl(vm, data, end, &data, visited);
-                env = gst_deserialize_impl(vm, data, end, &data, visited);
-                ret.type = GST_FUNCTION;
-                ret.data.function = gst_alloc(vm, sizeof(GstFunction));
-                if (parent->type == GST_NIL) {
-                    ret.data.function->parent = NULL;
-                } else if (parent->type == GST_FUNCTION) {
-                    ret.data.function->parent = parent.data.function;
-                } else {
-                    deser_error("expected function");
-                }
-                gst_assert(def->type == GST_FUNCDEF, "expected funcdef");
-                gst_assert(env->type == GST_FUNCENV, "expected funcenv");
-                ret.data.function->env = env.data.env;
-                ret.data.function->def = env.data.def;
-                *newData = data;
-                gst_array_push(vm, visited, ret);
-            }
-            return ret;
-
-        case 215: /* LUdata */
-            {
-                GstValue meta;
-                ret.type = GST_USERDATA;
-                meta = gst_deserialize_impl(vm, data, end, &data, visited);
-                deser_assert(meta.type == GST_OBJECT, "userdata requires valid meta");
-                length = read_u32(); data += 4;
-                data = deser_datacheck(length);
-                ret.data.pointer = gst_userdata(vm, length, meta.data.object);
-                gst_memcpy(ret.data.pointer, data, length);
-                *newData = data + length;
-                gst_array_push(vm, visited, ret);
-            }
-            return ret;
-
-        case 216: /* C function */
-
-            return ret;
-
-        case 217: /* Reference */
-            length = read_u32(); data += 4;
-            deser_assert(visited->count > length, "invalid reference");
-            *newData = data;
-            return visited->data[length];
-    }
-}

core/serialize.c

@@ -0,0 +1,535 @@
+#include <gst/serialize.h>
+#include <gst/datatypes.h>
+#include <gst/vm.h>
+#include <gst/value.h>
+#include <gst/ds.h>
+#include <gst/util.h>
+#include <gst/thread.h>
+
+/**
+ * Data format
+ * State is encoded as a string of unsigned bytes.
+ *
+ * Types:
+ *
+ * Byte 0 to 200: small integer byte - 100
+ * Byte 201: Nil
+ * Byte 202: True
+ * Byte 203: False
+ * Byte 204: Number  - double format
+ * Byte 205: String  - [u32 length]*[u8... characters]
+ * Byte 206: Symbol  - [u32 length]*[u8... characters]
+ * Byte 207: Buffer  - [u32 length]*[u8... characters]
+ * Byte 208: Array   - [u32 length]*[value... elements]
+ * Byte 209: Tuple   - [u32 length]*[value... elements]
+ * Byte 210: Thread  - [u8 state][u32 frames]*[[value callee][value env]
+ *  [u32 pcoffset][u32 erroffset][u16 ret][u16 errloc][u16 size]*[value ...stack]
+ * Byte 211: Object  - [value meta][u32 length]*2*[value... kvs]
+ * Byte 212: FuncDef - [u32 locals][u32 arity][u32 flags][u32 literallen]*[value...
+ *  literals][u32 bytecodelen]*[u16... bytecode]
+ * Byte 213: FunEnv  - [value thread][u32 length]*[value ...upvalues]
+ *  (upvalues is not read if thread is a thread object)
+ * Byte 214: Func    - [value parent][value def][value env]
+ *  (nil values indicate empty)
+ * Byte 215: LUdata  - [value meta][u32 length]*[u8... bytes]
+ * Byte 216: CFunc   - [u32 length]*[u8... idstring]
+ * Byte 217: Ref     - [u32 id]
+ */
+
+/* Error at buffer end */
+static const char UEB[] = "unexpected end of buffer";
+
+/* Read 4 bytes as an unsigned integer */
+static uint32_t bytes2u32(uint8_t *bytes) {
+    union {
+        uint8_t bytes[4];
+        uint32_t u32;
+    } u;
+    gst_memcpy(u.bytes, bytes, 4 * sizeof(uint8_t));
+    return u.u32;
+}
+
+/* Read 2 bytes as unsigned short */
+static uint16_t bytes2u16(uint8_t *bytes) {
+    union {
+        uint8_t bytes[2];
+        uint16_t u16;
+    } u;
+    gst_memcpy(u.bytes, bytes, 2 * sizeof(uint8_t));
+    return u.u16;
+}
+
+/* Read 8 bytes as a double */
+static uint32_t bytes2dbl(uint8_t *bytes) {
+    union {
+        uint8_t bytes[8];
+        double dbl;
+    } u;
+    gst_memcpy(u.bytes, bytes, 8 * sizeof(uint8_t));
+    return u.dbl;
+}
+
+/* Read a string and turn it into a gst value. Returns
+ * an error message if there is an error message during
+ * deserialization. If successful, the resulting value is
+ * passed by reference. */
+static const char *gst_deserialize_impl(
+        Gst *vm, 
+        uint8_t *data,
+        uint8_t *end,
+        uint8_t **newData,
+        GstArray *visited,
+        GstValue *out) {
+
+    GstValue ret;
+    ret.type = GST_NIL;
+    GstValue *buffer;
+    uint8_t *bytebuf;
+    uint32_t length, i;
+    const char *err;
+
+    /* Handle errors as needed */
+#define deser_error(e) return (e)
+
+    /* Assertions */
+#define deser_assert(c, e) do{if(!(c))deser_error(e);}while(0)
+
+    /* Assert enough buffer */
+#define deser_datacheck(len) do{if (end - data < (len)) deser_error(UEB);}while(0)
+
+    /* Check for enough space to read uint32_t */
+#define read_u32(out) do{deser_datacheck(4); (out)=bytes2u32(data); data += 4; }while(0)
+#define read_u16(out) do{deser_datacheck(2); (out)=bytes2u16(data); data += 2; }while(0)
+#define read_dbl(out) do{deser_datacheck(8); (out)=bytes2dbl(data); data += 8; }while(0)
+
+    /* Check enough buffer left to read one byte */
+    if (data >= end) deser_error(UEB);
+
+    /* Small integer */
+    if (*data < 201) {
+        ret.type = GST_NUMBER;
+        ret.data.number = *data - 100; 
+        *newData = data + 1;
+        *out = ret;
+        return NULL;
+    } 
+
+    /* Main switch for types */
+    switch (*data++) {
+
+        case 201: /* Nil */
+            ret.type = GST_NIL;
+            break;
+
+        case 202: /* True */
+            ret.type = GST_BOOLEAN;
+            ret.data.boolean = 1;
+            break;
+
+        case 203: /* False */
+            ret.type = GST_BOOLEAN;
+            ret.data.boolean = 0;
+            break;
+
+        case 204: /* Long number (double) */
+            ret.type = GST_NUMBER; 
+            read_dbl(ret.data.number);
+            break;
+
+        case 205: /* String */
+        case 206: /* Symbol */
+            ret.type = data[-1] == 205 ? GST_STRING : GST_SYMBOL;
+            read_u32(length);
+            deser_datacheck(length);
+            ret.data.string = gst_alloc(vm, 2 * sizeof(uint32_t) + length + 1);
+            ret.data.string += 2 * sizeof(uint32_t);
+            gst_string_length(ret.data.string) = length;
+            gst_string_hash(ret.data.string) = 0;
+            gst_memcpy(ret.data.string, data, length);
+            ret.data.string[length] = 0;
+            data += length;
+            gst_array_push(vm, visited, ret);
+            break;
+
+        case 207: /* Buffer */
+            ret.type = GST_BYTEBUFFER;
+            read_u32(length);
+            deser_datacheck(length);
+            ret.data.buffer = gst_alloc(vm, sizeof(GstBuffer));
+            ret.data.buffer->data = gst_alloc(vm, length);
+            gst_memcpy(ret.data.string, data, length);
+            ret.data.buffer->count = length;
+            ret.data.buffer->capacity = length;
+            data += length;
+            gst_array_push(vm, visited, ret);
+            break;
+
+        case 208: /* Array */
+            ret.type = GST_ARRAY;
+            read_u32(length);
+            buffer = gst_alloc(vm, length * sizeof(GstValue));
+            for (i = 0; i < length; ++i)
+                if ((err = gst_deserialize_impl(vm, data, end, &data, visited, buffer + i)))
+                    return err; 
+            ret.data.array = gst_alloc(vm, sizeof(GstArray));
+            ret.data.array->data = buffer;
+            ret.data.array->count = length;
+            ret.data.array->capacity = length;
+            gst_array_push(vm, visited, ret);
+            break;
+
+        case 209: /* Tuple */
+            ret.type = GST_TUPLE;
+            read_u32(length);
+            bytebuf = gst_alloc(vm, length * sizeof(GstValue) + 2 * sizeof(uint32_t));
+            buffer = (GstValue *)(bytebuf + 2 * sizeof(uint32_t));
+            for (i = 0; i < length; ++i)
+                if ((err = gst_deserialize_impl(vm, data, end, &data, visited, buffer + i)))
+                    return err;
+            gst_tuple_hash(buffer) = 0;
+            gst_tuple_length(buffer) = length;
+            ret.data.tuple = buffer;
+            gst_array_push(vm, visited, ret);
+            break;
+
+        case 210: /* Thread */
+            {
+                GstValue nil;
+                GstThread *t;
+                GstValue *stack;
+                uint16_t prevsize = 0;
+                uint8_t statusbyte;
+                nil.type = GST_NIL;
+                t = gst_thread(vm, nil, 64);
+                ret.type = GST_THREAD;
+                ret.data.thread = t;
+                deser_assert(data < end, UEB);
+                statusbyte = *data++;
+                read_u32(length);
+                /* Check for empty thread - TODO check for valid state */
+                if (length == 0)
+                    break;
+                /* Set status */
+                if (statusbyte == 0) t->status = GST_THREAD_PENDING;
+                else if (statusbyte == 1) t->status = GST_THREAD_ALIVE;
+                else t->status = GST_THREAD_DEAD;
+                /* Add frames */
+                for (i = 0; i < length; ++i) {
+                    GstValue callee, env;
+                    uint32_t pcoffset, erroffset;
+                    uint16_t ret, errloc, size, j;
+                    /* Create a new frame */
+                    if (i > 0)
+                        gst_thread_beginframe(vm, t, nil, 0);
+                    /* Read the stack */
+                    err = gst_deserialize_impl(vm, data, end, &data, visited, &callee);
+                    if (err != NULL) return err;
+                    err = gst_deserialize_impl(vm, data, end, &data, visited, &env);
+                    if (err != NULL) return err;
+                    read_u32(pcoffset);
+                    read_u32(erroffset);
+                    read_u16(ret);
+                    read_u16(errloc);
+                    read_u16(size);
+                    /* Set up the stack */
+                    stack = gst_thread_stack(t);
+                    if (callee.type == GST_FUNCTION) {
+                        gst_frame_pc(stack) = callee.data.function->def->byteCode + pcoffset;
+                        gst_frame_errjmp(stack) = callee.data.function->def->byteCode + erroffset;
+                        if (env.type == GST_FUNCENV)
+                            gst_frame_env(stack) = env.data.env;
+                    }
+                    gst_frame_ret(stack) = ret;
+                    gst_frame_errloc(stack) = errloc;
+                    gst_frame_size(stack) = size;
+                    gst_frame_prevsize(stack) = prevsize;
+                    prevsize = size;
+                    /* Push stack args */
+                    for (j = 0; j < size; ++j) {
+                        GstValue temp;
+                        err = gst_deserialize_impl(vm, data, end, &data, visited, &temp);
+                        gst_thread_push(vm, t, temp);
+                    }
+                }
+            }
+            break;
+
+        case 211: /* Object */
+            {
+                GstValue meta;
+                ret.type = GST_OBJECT;
+                ret.data.object = gst_object(vm, 10);
+                err = gst_deserialize_impl(vm, data, end, &data, visited, &meta);
+                if (err != NULL) return err;
+                read_u32(length);
+                for (i = 0; i < length; i += 2) {
+                    GstValue key, value;
+                    err = gst_deserialize_impl(vm, data, end, &data, visited, &key);
+                    if (err != NULL) return err;
+                    err = gst_deserialize_impl(vm, data, end, &data, visited, &value);
+                    if (err != NULL) return err;
+                    gst_object_put(vm, ret.data.object, key, value);
+                }
+                if (meta.type == GST_OBJECT)
+                    ret.data.object->meta = meta.data.object; 
+                gst_array_push(vm, visited, ret);
+            }
+            break;
+
+        case 212: /* Funcdef */
+            {
+                GstFuncDef *def;
+                uint32_t locals, arity, literalsLen, byteCodeLen, flags;
+                read_u32(locals);
+                read_u32(arity);
+                read_u32(flags);
+                read_u32(literalsLen);
+                def = gst_alloc(vm, sizeof(GstFuncDef));
+                ret.type = GST_FUNCDEF;
+                ret.data.def = def;
+                def->locals = locals;
+                def->arity = arity;
+                def->flags = flags;
+                def->literalsLen = literalsLen;
+                if (literalsLen > 0) {
+                    def->literals = gst_alloc(vm, literalsLen * sizeof(GstValue));
+                }
+                for (i = 0; i < literalsLen; ++i) {
+                    err = gst_deserialize_impl(vm, data, end, &data, visited, def->literals + i);
+                    if (err != NULL) return err;
+                }
+                read_u32(byteCodeLen);
+                deser_datacheck(byteCodeLen);
+                def->byteCode = gst_alloc(vm, byteCodeLen * sizeof(uint16_t));
+                def->byteCodeLen = byteCodeLen;
+                for (i = 0; i < byteCodeLen; ++i) {
+                    read_u16(def->byteCode[i]);
+                }
+                gst_array_push(vm, visited, ret);
+            }
+            break;
+
+        case 213: /* Funcenv */
+            {
+                GstValue thread;
+                err = gst_deserialize_impl(vm, data, end, &data, visited, &thread);
+                if (err != NULL) return err;
+                read_u32(length);
+                ret.type = GST_FUNCENV;
+                ret.data.env = gst_alloc(vm, sizeof(GstFuncEnv));
+                ret.data.env->stackOffset = length;
+                if (thread.type == GST_THREAD) {
+                    ret.data.env->thread = thread.data.thread;
+                } else {
+                    ret.data.env->thread = NULL;
+                    ret.data.env->values = gst_alloc(vm, sizeof(GstValue) * length);
+                    for (i = 0; i < length; ++i) {
+                        GstValue item;
+                        err = gst_deserialize_impl(vm, data, end, &data, visited, &item);
+                        if (err != NULL) return err;
+                        ret.data.env->values[i] = item;
+                    }
+                }
+                gst_array_push(vm, visited, ret);
+            }
+            break;
+
+        case 214: /* Function */
+            {
+                GstValue parent, def, env;
+                err = gst_deserialize_impl(vm, data, end, &data, visited, &parent);
+                if (err != NULL) return err;
+                err = gst_deserialize_impl(vm, data, end, &data, visited, &def);
+                if (err != NULL) return err;
+                err = gst_deserialize_impl(vm, data, end, &data, visited, &env);
+                if (err != NULL) return err;
+                ret.type = GST_FUNCTION;
+                ret.data.function = gst_alloc(vm, sizeof(GstFunction));
+                if (parent.type == GST_NIL) {
+                    ret.data.function->parent = NULL;
+                } else if (parent.type == GST_FUNCTION) {
+                    ret.data.function->parent = parent.data.function;
+                } else {
+                    deser_error("expected function");
+                }
+                deser_assert(def.type == GST_FUNCDEF, "expected funcdef");
+                deser_assert(env.type == GST_FUNCENV, "expected funcenv");
+                ret.data.function->env = env.data.env;
+                ret.data.function->def = env.data.def;
+                gst_array_push(vm, visited, ret);
+            }
+            break;
+
+        case 215: /* LUdata */
+            {
+                GstValue meta;
+                ret.type = GST_USERDATA;
+                err = gst_deserialize_impl(vm, data, end, &data, visited, &meta);
+                if (err != NULL) return err;
+                deser_assert(meta.type == GST_OBJECT, "userdata requires valid meta");
+                read_u32(length);
+                deser_datacheck(length);
+                ret.data.pointer = gst_userdata(vm, length, meta.data.object);
+                gst_memcpy(ret.data.pointer, data, length);
+                gst_array_push(vm, visited, ret);
+            }
+            break;
+
+        case 216: /* C function */
+            /* TODO - add registry for c functions */
+            read_u32(length);
+            ret.type = GST_NIL;
+            break;
+
+        case 217: /* Reference */
+            read_u32(length);
+            deser_assert(visited->count > length, "invalid reference");
+            ret = visited->data[length];
+            break;
+    }
+
+    /* Handle a successful return */
+    *out = ret;
+    *newData = data;
+    return NULL;
+}
+
+/* Load a value from data */
+const char *gst_deserialize(
+        Gst *vm,
+        uint8_t *data,
+        uint32_t len,
+        GstValue *out,
+        uint8_t *nextData) {
+    GstValue ret;
+    const char *err;
+    GstArray *visited = gst_array(vm, 10);
+    err = gst_deserialize_impl(vm, data, data + len, &nextData, visited, &ret);
+    if (err != NULL) return err;
+    *out = ret;
+    return NULL;
+}
+    
+/* Allow appending other types to buffers */
+BUFFER_DEFINE(number, GstNumber)
+/*BUFFER_DEFINE(u16, uint16_t)*/
+BUFFER_DEFINE(u32, uint32_t)
+
+/* Serialize a value and write to a buffer. Returns possible
+ * error messages. */
+const char *gst_serialize_impl(
+        Gst *vm,
+        GstBuffer *buffer,
+        GstObject *visited,
+        uint32_t *nextId,
+        GstValue x) {
+
+    uint32_t i, count;
+    const char *err;
+    GstValue check;
+
+#define write_byte(b) gst_buffer_push(vm, buffer, (b))
+#define write_u32(b) gst_buffer_push_u32(vm, buffer, (b))
+#define write_u16(b) gst_buffer_push_u16(vm, buffer, (b))
+#define write_dbl(b) gst_buffer_push_number(vm, buffer, (b))
+
+    /* Check non reference types - if successful, return NULL */
+    switch (x.type) {
+        case GST_NIL:
+            write_byte(201);
+            return NULL;
+        case GST_BOOLEAN:
+            write_byte(x.data.boolean ? 202 : 203);
+            return NULL;
+        case GST_NUMBER: 
+            {
+                GstNumber number = x.data.number;
+                int32_t int32Num = (int32_t) number;
+                if (number == (GstNumber) int32Num &&
+                        int32Num <= 100 && int32Num >= -100) {
+                    write_byte(int32Num + 100);
+                } else {
+                    write_byte(204);
+                    write_dbl(number);
+                }
+            }
+            return NULL;
+        case GST_CFUNCTION:
+            /* TODO */
+            break;
+        default:
+            break;
+    }
+
+    /* Check if already seen - if so, use reference */
+    check = gst_object_get(visited, x);
+    if (check.type == GST_NUMBER) {
+        write_byte(217);
+        write_u32((uint32_t) check.data.number);
+        return NULL;
+    }
+
+    /* Check reference types */
+    switch (x.type) {
+        default:
+           return "unable to serialize type"; 
+        case GST_STRING:
+        case GST_SYMBOL:
+           write_byte(x.type == GST_STRING ? 205 : 206);
+           count = gst_string_length(x.data.string);
+           write_u32(count);
+           for (i = 0; i < count; ++i) {
+               write_byte(x.data.string[i]);
+           }
+           break;
+        case GST_BYTEBUFFER:
+           write_byte(207);
+           count = x.data.buffer->count;
+           write_u32(count);
+           for (i = 0; i < count; ++i) {
+                write_byte(x.data.buffer->data[i]);
+           }
+           break;
+        case GST_ARRAY:
+           write_byte(208);
+           count = x.data.array->count;
+           write_u32(count);
+           for (i = 0; i < count; ++i) {
+               err = gst_serialize_impl(vm, buffer, visited, nextId, x.data.array->data[i]); 
+               if (err != NULL) return err;
+           }
+           break;
+        case GST_TUPLE:
+           write_byte(209);
+           count = gst_tuple_length(x.data.tuple);
+           write_u32(count);
+           for (i = 0; i < count; ++i) {
+               err = gst_serialize_impl(vm, buffer, visited, nextId, x.data.tuple[i]); 
+               if (err != NULL) return err;
+           }
+           break;
+        /*case GST_THREAD:*/
+           /*break;*/
+    }
+
+    /* Record reference */
+    check.type = GST_NUMBER;
+    check.data.number = *nextId++;
+    gst_object_put(vm, visited, x, check);
+
+    /* Return success */
+    return NULL;
+}
+
+/* Serialize an object */
+const char *gst_serialize(Gst *vm, GstBuffer *buffer, GstValue x) {
+    uint32_t nextId = 0;
+    uint32_t oldCount = buffer->count;
+    const char *err;
+    GstObject *visited = gst_object(vm, 10);
+    err = gst_serialize_impl(vm, buffer, visited, &nextId, x);
+    if (err != NULL) {
+        buffer->count = oldCount;
+    }
+    return err;
+}

core/stl.c

@@ -2,6 +2,7 @@
  * will eventually be ported over to gst if possible */
 #include <gst/stl.h>
 #include <gst/gst.h>
+#include <gst/serialize.h>
 
 /****/
 /* Core */
@@ -53,6 +54,29 @@ int gst_stl_callforeach(Gst *vm) {
     }
 }
 
+/* Create a buffer */
+int gst_stl_make_buffer(Gst *vm) {
+    uint32_t i, count;
+    GstValue buf;
+    buf.type = GST_BYTEBUFFER;
+    buf.data.buffer = gst_buffer(vm, 10);
+    count = gst_count_args(vm);
+    for (i = 0; i < count; ++i) {
+        uint8_t *string = gst_to_string(vm, gst_arg(vm, i));
+        gst_buffer_append(vm, buf.data.buffer, string, gst_string_length(string));
+    }
+    gst_c_return(vm, buf);
+}
+
+/* To string */
+int gst_stl_tostring(Gst *vm) {
+    GstValue ret;
+    uint8_t *string = gst_to_string(vm, gst_arg(vm, 0));
+    ret.type = GST_STRING;
+    ret.data.string = string;
+    gst_c_return(vm, ret);
+}
+
 /* Exit */
 int gst_stl_exit(Gst *vm) {
     int ret;
@@ -68,6 +92,8 @@ void gst_stl_load_core(GstCompiler *c) {
     gst_compiler_add_global_cfunction(c, "get-class", gst_stl_getclass);
     gst_compiler_add_global_cfunction(c, "set-class", gst_stl_setclass);
     gst_compiler_add_global_cfunction(c, "call-for-each", gst_stl_callforeach);
+    gst_compiler_add_global_cfunction(c, "make-buffer", gst_stl_make_buffer);
+    gst_compiler_add_global_cfunction(c, "tostring", gst_stl_tostring);
     gst_compiler_add_global_cfunction(c, "exit", gst_stl_exit);
 }
 
@@ -130,6 +156,32 @@ void gst_stl_load_compile(GstCompiler *c) {
     gst_compiler_add_global_cfunction(c, "compile", gst_stl_compile);
 }
 
+/****/
+/* Serialization */
+/****/
+
+/* Serialize data into buffer */
+int gst_stl_serialize(Gst *vm) {
+    const char *err;
+    uint32_t i;
+    GstValue buffer = gst_arg(vm, 0);
+    if (buffer.type != GST_BYTEBUFFER)
+        gst_c_throwc(vm, "expected buffer");
+    for (i = 1; i < gst_count_args(vm); ++i) {
+        err = gst_serialize(vm, buffer.data.buffer, gst_arg(vm, i));
+        if (err != NULL)
+            gst_c_throwc(vm, err);
+    }
+    gst_c_return(vm, buffer);
+}
+
+/* Load serilization */
+void gst_stl_load_serialization(GstCompiler *c) {
+    gst_compiler_add_global_cfunction(c, "serialize", gst_stl_serialize);
+}
+
+/* Read data from a linear sequence of memory */
+
 /****/
 /* IO */
 /****/
@@ -145,4 +197,5 @@ void gst_stl_load(GstCompiler *c) {
     gst_stl_load_core(c);
     gst_stl_load_parse(c);
     gst_stl_load_compile(c);
+    gst_stl_load_serialization(c);
 }

core/value.c

@@ -477,3 +477,44 @@ const char *gst_set_class(GstValue x, GstValue class) {
     return NULL;
 }
 
+/* Get the length of an object. Returns errors for invalid types */
+int gst_length(Gst *vm, GstValue x, GstValue *len) {
+    uint32_t length;
+    switch (x.type) {
+        default:
+            vm->ret = gst_load_cstring(vm, "cannot get length");
+            return GST_RETURN_ERROR;
+        case GST_STRING:
+        case GST_SYMBOL:
+            length = gst_string_length(x.data.string);
+            break;
+        case GST_ARRAY:
+            length = x.data.array->count;
+            break;
+        case GST_BYTEBUFFER:
+            length = x.data.buffer->count;
+            break;
+        case GST_TUPLE:
+            length = gst_tuple_length(x.data.tuple);
+            break;
+        case GST_OBJECT:
+            /* TODO - Check for class override */
+            if (x.data.object->meta != NULL) {
+                GstValue check = gst_object_get_cstring(
+                        x.data.object->meta, "length");
+                if (check.type != GST_NIL) {
+                    int status = gst_call(vm, check, 1, &x);
+                    if (status == GST_RETURN_OK)
+                        *len = vm->ret;
+                    return status;
+                }
+            }
+            length = x.data.object->count;
+            break;
+    }
+    /* Normal numeric return */
+    len->type = GST_NUMBER;
+    len->data.number = (GstNumber) length;
+    return GST_RETURN_OK;
+}
+

core/vm.c

@@ -107,6 +107,17 @@ static int gst_continue_size(Gst *vm, uint32_t stackBase) {
             pc += 3;
             continue;
 
+        case GST_OP_LEN: /* Length */
+            {
+                int status = gst_length(vm, stack[pc[2]], &v1);
+                if (status == GST_RETURN_OK)
+                    stack[pc[1]] = v1;
+                else
+                    goto vm_error;
+                pc += 3;
+            }
+            continue;
+
         case GST_OP_FLS: /* Load False */
             temp.type = GST_BOOLEAN;
             temp.data.boolean = 0;
@@ -420,6 +431,8 @@ static int gst_continue_size(Gst *vm, uint32_t stackBase) {
 
         /* Handle errors from c functions and vm opcodes */
         vm_error:
+            if (stack == NULL)
+                return GST_RETURN_ERROR;
             while (gst_frame_errjmp(stack) == NULL) {
                 stack = gst_thread_popframe(vm, &thread);
                 if (thread.count < stackBase)