Fix some stack overflow bugs.

CHANGELOG.md

@@ -2,6 +2,7 @@
 All notable changes to this project will be documented in this file.
 
 ## Unreleased
+- Change default fiber stack limit to the maximum value of a 32 bit signed integer.
 - Some bug fixes with `jpm`
 - Add `os/arch` to get ISA that janet was compiled for
 - Add color to stacktraces via `(dyn :err-color)`

meson_options.txt

@@ -14,7 +14,7 @@ option('int_types', type : 'boolean', value : true)
 option('recursion_guard', type : 'integer', min : 10, max : 8000, value : 1024)
 option('max_proto_depth', type : 'integer', min : 10, max : 8000, value : 200)
 option('max_macro_expand', type : 'integer', min : 1, max : 8000, value : 200)
-option('stack_max', type : 'integer', min : 8096, max : 1000000000, value : 16384)
+option('stack_max', type : 'integer', min : 8096, max : 0x7fffffff, value : 0x7fffffff)
 
 option('arch_name', type : 'string', value: '')
 option('os_name', type : 'string', value: '')

src/core/array.c

@@ -64,7 +64,9 @@ void janet_array_ensure(JanetArray *array, int32_t capacity, int32_t growth) {
     Janet *newData;
     Janet *old = array->data;
     if (capacity <= array->capacity) return;
-    capacity *= growth;
+    int64_t new_capacity = capacity * growth;
+    if (new_capacity > INT32_MAX) new_capacity = INT32_MAX;
+    capacity = (int32_t) new_capacity;
     newData = realloc(old, capacity * sizeof(Janet));
     if (NULL == newData) {
         JANET_OUT_OF_MEMORY;

src/core/fiber.c

@@ -87,19 +87,27 @@ void janet_fiber_setcapacity(JanetFiber *fiber, int32_t n) {
     fiber->capacity = n;
 }
 
+/* Grow fiber if needed */
+static void janet_fiber_grow(JanetFiber *fiber, int32_t needed) {
+    int32_t cap = needed > (INT32_MAX / 2) ? INT32_MAX : 2 * needed;
+    janet_fiber_setcapacity(fiber, cap);
+}
+
 /* Push a value on the next stack frame */
 void janet_fiber_push(JanetFiber *fiber, Janet x) {
+    if (fiber->stacktop == INT32_MAX) janet_panic("stack overflow");
     if (fiber->stacktop >= fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * fiber->stacktop);
+        janet_fiber_grow(fiber, fiber->stacktop);
     }
     fiber->data[fiber->stacktop++] = x;
 }
 
 /* Push 2 values on the next stack frame */
 void janet_fiber_push2(JanetFiber *fiber, Janet x, Janet y) {
+    if (fiber->stacktop >= INT32_MAX - 1) janet_panic("stack overflow");
     int32_t newtop = fiber->stacktop + 2;
     if (newtop > fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * newtop);
+        janet_fiber_grow(fiber, newtop);
     }
     fiber->data[fiber->stacktop] = x;
     fiber->data[fiber->stacktop + 1] = y;
@@ -108,9 +116,10 @@ void janet_fiber_push2(JanetFiber *fiber, Janet x, Janet y) {
 
 /* Push 3 values on the next stack frame */
 void janet_fiber_push3(JanetFiber *fiber, Janet x, Janet y, Janet z) {
+    if (fiber->stacktop >= INT32_MAX - 2) janet_panic("stack overflow");
     int32_t newtop = fiber->stacktop + 3;
     if (newtop > fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * newtop);
+        janet_fiber_grow(fiber, newtop);
     }
     fiber->data[fiber->stacktop] = x;
     fiber->data[fiber->stacktop + 1] = y;
@@ -120,9 +129,10 @@ void janet_fiber_push3(JanetFiber *fiber, Janet x, Janet y, Janet z) {
 
 /* Push an array on the next stack frame */
 void janet_fiber_pushn(JanetFiber *fiber, const Janet *arr, int32_t n) {
+    if (fiber->stacktop > INT32_MAX - n) janet_panic("stack overflow");
     int32_t newtop = fiber->stacktop + n;
     if (newtop > fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * newtop);
+        janet_fiber_grow(fiber, newtop);
     }
     memcpy(fiber->data + fiber->stacktop, arr, n * sizeof(Janet));
     fiber->stacktop = newtop;

src/core/vm.c

@@ -628,6 +628,9 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in, JanetFiberStatus status)
 
     VM_OP(JOP_TAILCALL) {
         Janet callee = stack[D];
+        if (fiber->stacktop > fiber->maxstack) {
+            vm_throw("stack overflow");
+        }
         if (janet_checktype(callee, JANET_KEYWORD)) {
             vm_commit();
             callee = resolve_method(callee, fiber);

src/include/janet.h

@@ -171,11 +171,10 @@ extern "C" {
 /* Maximum depth to follow table prototypes before giving up and returning nil. */
 #define JANET_MAX_MACRO_EXPAND 200
 
-/* Define max stack size for stacks before raising a stack overflow error.
- * If this is not defined, fiber stacks can grow without limit (until memory
- * runs out) */
+/* Define default max stack size for stacks before raising a stack overflow error.
+ * This can also be set on a per fiber basis. */
 #ifndef JANET_STACK_MAX
-#define JANET_STACK_MAX 16384
+#define JANET_STACK_MAX 0x7fffffff
 #endif
 
 /* Use nanboxed values - uses 8 bytes per value instead of 12 or 16.