Replace janet_malloc(A * X) with array_allocate(A, X) to avoid overflow

on 32 bit systems in some cases.

Since most internal arrays in Janet have 32 bit limits, this is is not
hit in practice on 64 bit systems, except in cases of user input. Apply
a fix more generally to simply avoid the issue by coding "defensively".
Some applications are not strictly needed.

src/core/array.c

@@ -34,7 +34,7 @@ static void janet_array_impl(JanetArray *array, int32_t capacity) {
     Janet *data = NULL;
     if (capacity > 0) {
         janet_vm.next_collection += capacity * sizeof(Janet);
-        data = (Janet *) janet_malloc(sizeof(Janet) * (size_t) capacity);
+        data = (Janet *) array_allocate(sizeof(Janet), capacity);
         if (NULL == data) {
             JANET_OUT_OF_MEMORY;
         }
@@ -63,7 +63,7 @@ JanetArray *janet_array_n(const Janet *elements, int32_t n) {
     JanetArray *array = janet_gcalloc(JANET_MEMORY_ARRAY, sizeof(JanetArray));
     array->capacity = n;
     array->count = n;
-    array->data = janet_malloc(sizeof(Janet) * (size_t) n);
+    array->data = array_allocate(sizeof(Janet), (size_t) n);
     if (!array->data) {
         JANET_OUT_OF_MEMORY;
     }

src/core/asm.c

@@ -603,7 +603,7 @@ static JanetAssembleResult janet_asm1(JanetAssembler *parent, Janet source, int
     x = janet_get1(s, janet_ckeywordv("constants"));
     if (janet_indexed_view(x, &arr, &count)) {
         def->constants_length = count;
-        def->constants = janet_malloc(sizeof(Janet) * (size_t) count);
+        def->constants = array_allocate(sizeof(Janet), count);
         if (NULL == def->constants) {
             JANET_OUT_OF_MEMORY;
         }
@@ -667,7 +667,7 @@ static JanetAssembleResult janet_asm1(JanetAssembler *parent, Janet source, int
         }
         /* Allocate bytecode array */
         def->bytecode_length = blength;
-        def->bytecode = janet_malloc(sizeof(uint32_t) * (size_t) blength);
+        def->bytecode = array_allocate(sizeof(uint32_t), blength);
         if (NULL == def->bytecode) {
             JANET_OUT_OF_MEMORY;
         }

src/core/buffer.c

@@ -40,7 +40,7 @@ static JanetBuffer *janet_buffer_init_impl(JanetBuffer *buffer, int32_t capacity
     uint8_t *data = NULL;
     if (capacity < 4) capacity = 4;
     janet_gcpressure(capacity);
-    data = janet_malloc(sizeof(uint8_t) * (size_t) capacity);
+    data = array_allocate(sizeof(uint8_t), capacity);
     if (NULL == data) {
         JANET_OUT_OF_MEMORY;
     }

src/core/bytecode.c

@@ -408,6 +408,7 @@ int janet_verify(JanetFuncDef *def) {
     int32_t maxslot = def->arity + vargs;
     int32_t sc = def->slotcount;
 
+    if (def->environments_length > 256) return 15;
     if (def->bytecode_length == 0) return 1;
 
     if (maxslot > sc) return 2;

src/core/fiber.c

@@ -55,7 +55,7 @@ static JanetFiber *fiber_alloc(int32_t capacity) {
         capacity = 32;
     }
     fiber->capacity = capacity;
-    data = janet_malloc(sizeof(Janet) * (size_t) capacity);
+    data = array_allocate(sizeof(Janet), capacity);
     if (NULL == data) {
         JANET_OUT_OF_MEMORY;
     }
@@ -102,7 +102,7 @@ JanetFiber *janet_fiber(JanetFunction *callee, int32_t capacity, int32_t argc, c
 static void janet_fiber_refresh_memory(JanetFiber *fiber) {
     int32_t n = fiber->capacity;
     if (n) {
-        Janet *newData = janet_malloc(sizeof(Janet) * n);
+        Janet *newData = array_allocate(sizeof(Janet), n);
         if (NULL == newData) {
             JANET_OUT_OF_MEMORY;
         }
@@ -256,7 +256,7 @@ static void janet_env_detach(JanetFuncEnv *env) {
     /* Check for closure environment */
     if (env) {
         janet_env_valid(env);
-        int32_t len = env->length;
+        int32_t len = env->length; /* Maximum is 256 */
         size_t s = sizeof(Janet) * (size_t) len;
         Janet *vmem = janet_malloc(s);
         janet_vm.next_collection += (uint32_t) s;

src/core/marsh.c

@@ -79,13 +79,6 @@ enum {
     LB_ARRAY_WEAK, /* 232 */
 } LeadBytes;
 
-/* Avoid overflow and underflow, especially on 32-bit systems */
-static void *array_allocate(size_t element_size, int32_t count) {
-    if (count < 0) return NULL;
-    if ((size_t) count > (SIZE_MAX / element_size)) return NULL;
-    return janet_malloc(element_size * count);
-}
-
 /* Helper to look inside an entry in an environment */
 static Janet entry_getval(Janet env_entry) {
     if (janet_checktype(env_entry, JANET_TABLE)) {

src/core/table.c

@@ -260,7 +260,7 @@ JanetTable *janet_table_clone(JanetTable *table) {
     newTable->capacity = table->capacity;
     newTable->deleted = table->deleted;
     newTable->proto = table->proto;
-    newTable->data = janet_malloc(newTable->capacity * sizeof(JanetKV));
+    newTable->data = array_allocate(sizeof(JanetKV), newTable->capacity);
     if (NULL == newTable->data) {
         JANET_OUT_OF_MEMORY;
     }

src/core/util.c

@@ -1213,3 +1213,10 @@ void *(janet_calloc)(size_t nmemb, size_t size) {
 void *(janet_realloc)(void *ptr, size_t size) {
     return janet_realloc(ptr, size);
 }
+
+/* Avoid overflow and underflow, especially on 32-bit systems */
+void *array_allocate(size_t element_size, int32_t count) {
+    if (count < 0) return NULL;
+    if ((size_t) count > (SIZE_MAX / element_size)) return NULL;
+    return janet_malloc(element_size * count);
+}

src/core/util.h

@@ -172,6 +172,9 @@ int janet_gettime(struct timespec *spec, enum JanetTimeSource source);
 #define strdup(x) _strdup(x)
 #endif
 
+/* Safely allocate arrays */
+void *array_allocate(size_t element_size, int32_t count);
+
 /* Use LoadLibrary on windows or dlopen on posix to load dynamic libraries
  * with native code. */
 #if defined(JANET_NO_DYNAMIC_MODULES)

src/core/wrap.c

@@ -162,7 +162,7 @@ Janet(janet_wrap_number)(double x) {
 
 void *janet_memalloc_empty(int32_t count) {
     int32_t i;
-    void *mem = janet_malloc((size_t) count * sizeof(JanetKV));
+    void *mem = array_allocate(sizeof(JanetKV), count);
     janet_vm.next_collection += (size_t) count * sizeof(JanetKV);
     if (NULL == mem) {
         JANET_OUT_OF_MEMORY;