From 292be33b9d4f75c5e4da9628234c07d7a5a19774 Mon Sep 17 00:00:00 2001
From: Calvin Rose <calsrose@gmail.com>
Date: Mon, 19 Aug 2019 01:19:51 -0400
Subject: [PATCH] Fix some stack overflow bugs.

---
 CHANGELOG.md        |  1 +
 meson_options.txt   |  2 +-
 src/core/array.c    |  4 +++-
 src/core/fiber.c    | 18 ++++++++++++++----
 src/core/vm.c       |  3 +++
 src/include/janet.h |  7 +++----
 6 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 063b8712..7bb0b8a0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 All notable changes to this project will be documented in this file.
 
 ## Unreleased
+- Change default fiber stack limit to the maximum value of a 32 bit signed integer.
 - Some bug fixes with `jpm`
 - Add `os/arch` to get ISA that janet was compiled for
 - Add color to stacktraces via `(dyn :err-color)`
diff --git a/meson_options.txt b/meson_options.txt
index afa8f0b9..b63c8ec1 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -14,7 +14,7 @@ option('int_types', type : 'boolean', value : true)
 option('recursion_guard', type : 'integer', min : 10, max : 8000, value : 1024)
 option('max_proto_depth', type : 'integer', min : 10, max : 8000, value : 200)
 option('max_macro_expand', type : 'integer', min : 1, max : 8000, value : 200)
-option('stack_max', type : 'integer', min : 8096, max : 1000000000, value : 16384)
+option('stack_max', type : 'integer', min : 8096, max : 0x7fffffff, value : 0x7fffffff)
 
 option('arch_name', type : 'string', value: '')
 option('os_name', type : 'string', value: '')
diff --git a/src/core/array.c b/src/core/array.c
index 62fa7844..39acff80 100644
--- a/src/core/array.c
+++ b/src/core/array.c
@@ -64,7 +64,9 @@ void janet_array_ensure(JanetArray *array, int32_t capacity, int32_t growth) {
     Janet *newData;
     Janet *old = array->data;
     if (capacity <= array->capacity) return;
-    capacity *= growth;
+    int64_t new_capacity = capacity * growth;
+    if (new_capacity > INT32_MAX) new_capacity = INT32_MAX;
+    capacity = (int32_t) new_capacity;
     newData = realloc(old, capacity * sizeof(Janet));
     if (NULL == newData) {
         JANET_OUT_OF_MEMORY;
diff --git a/src/core/fiber.c b/src/core/fiber.c
index d4fa5e82..cb1eea93 100644
--- a/src/core/fiber.c
+++ b/src/core/fiber.c
@@ -87,19 +87,27 @@ void janet_fiber_setcapacity(JanetFiber *fiber, int32_t n) {
     fiber->capacity = n;
 }
 
+/* Grow fiber if needed */
+static void janet_fiber_grow(JanetFiber *fiber, int32_t needed) {
+    int32_t cap = needed > (INT32_MAX / 2) ? INT32_MAX : 2 * needed;
+    janet_fiber_setcapacity(fiber, cap);
+}
+
 /* Push a value on the next stack frame */
 void janet_fiber_push(JanetFiber *fiber, Janet x) {
+    if (fiber->stacktop == INT32_MAX) janet_panic("stack overflow");
     if (fiber->stacktop >= fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * fiber->stacktop);
+        janet_fiber_grow(fiber, fiber->stacktop);
     }
     fiber->data[fiber->stacktop++] = x;
 }
 
 /* Push 2 values on the next stack frame */
 void janet_fiber_push2(JanetFiber *fiber, Janet x, Janet y) {
+    if (fiber->stacktop >= INT32_MAX - 1) janet_panic("stack overflow");
     int32_t newtop = fiber->stacktop + 2;
     if (newtop > fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * newtop);
+        janet_fiber_grow(fiber, newtop);
     }
     fiber->data[fiber->stacktop] = x;
     fiber->data[fiber->stacktop + 1] = y;
@@ -108,9 +116,10 @@ void janet_fiber_push2(JanetFiber *fiber, Janet x, Janet y) {
 
 /* Push 3 values on the next stack frame */
 void janet_fiber_push3(JanetFiber *fiber, Janet x, Janet y, Janet z) {
+    if (fiber->stacktop >= INT32_MAX - 2) janet_panic("stack overflow");
     int32_t newtop = fiber->stacktop + 3;
     if (newtop > fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * newtop);
+        janet_fiber_grow(fiber, newtop);
     }
     fiber->data[fiber->stacktop] = x;
     fiber->data[fiber->stacktop + 1] = y;
@@ -120,9 +129,10 @@ void janet_fiber_push3(JanetFiber *fiber, Janet x, Janet y, Janet z) {
 
 /* Push an array on the next stack frame */
 void janet_fiber_pushn(JanetFiber *fiber, const Janet *arr, int32_t n) {
+    if (fiber->stacktop > INT32_MAX - n) janet_panic("stack overflow");
     int32_t newtop = fiber->stacktop + n;
     if (newtop > fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * newtop);
+        janet_fiber_grow(fiber, newtop);
     }
     memcpy(fiber->data + fiber->stacktop, arr, n * sizeof(Janet));
     fiber->stacktop = newtop;
diff --git a/src/core/vm.c b/src/core/vm.c
index 21bec2e8..7a2016a8 100644
--- a/src/core/vm.c
+++ b/src/core/vm.c
@@ -628,6 +628,9 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in, JanetFiberStatus status)
 
     VM_OP(JOP_TAILCALL) {
         Janet callee = stack[D];
+        if (fiber->stacktop > fiber->maxstack) {
+            vm_throw("stack overflow");
+        }
         if (janet_checktype(callee, JANET_KEYWORD)) {
             vm_commit();
             callee = resolve_method(callee, fiber);
diff --git a/src/include/janet.h b/src/include/janet.h
index b8c651db..1d597ae9 100644
--- a/src/include/janet.h
+++ b/src/include/janet.h
@@ -171,11 +171,10 @@ extern "C" {
 /* Maximum depth to follow table prototypes before giving up and returning nil. */
 #define JANET_MAX_MACRO_EXPAND 200
 
-/* Define max stack size for stacks before raising a stack overflow error.
- * If this is not defined, fiber stacks can grow without limit (until memory
- * runs out) */
+/* Define default max stack size for stacks before raising a stack overflow error.
+ * This can also be set on a per fiber basis. */
 #ifndef JANET_STACK_MAX
-#define JANET_STACK_MAX 16384
+#define JANET_STACK_MAX 0x7fffffff
 #endif
 
 /* Use nanboxed values - uses 8 bytes per value instead of 12 or 16.