Address #920 - fiber cancellation can hang event loop.

The main issue was cancellation of fiber using `cancel` rather than
`ev/cancel` could cause issues with the event loop internal ref count.
Since this is almost certainly bad usage (and is not something I want to
encourage or support), we will warn against trying to resume or error
fibers that have already been suspended or scheduled on the event loop.
The distinction between "task" fibers and normal fibers is now kept by a
flag that is set when a fiber is resumed - if it is the outermost fiber
on the stack, it is considered a root fiber. All fibers scheduled with
ev/go or by the event loop are root fibers, and thus cannot be cancelled
or resumed with `cancel` or `resume` - instead, use `ev/cancel` or
`ev/go`.

src/core/fiber.h

@@ -58,6 +58,7 @@
 
 #define JANET_FIBER_EV_FLAG_CANCELED 0x10000
 #define JANET_FIBER_EV_FLAG_SUSPENDED 0x20000
+#define JANET_FIBER_FLAG_ROOT 0x40000
 
 #define janet_fiber_set_status(f, s) do {\
     (f)->flags &= ~JANET_FIBER_STATUS_MASK;\

src/core/vm.c

@@ -315,7 +315,7 @@ static Janet janet_binop_call(const char *lmethod, const char *rmethod, Janet lh
 }
 
 /* Forward declaration */
-static JanetSignal janet_check_can_resume(JanetFiber *fiber, Janet *out);
+static JanetSignal janet_check_can_resume(JanetFiber *fiber, Janet *out, int is_cancel);
 static JanetSignal janet_continue_no_check(JanetFiber *fiber, Janet in, Janet *out);
 
 /* Interpreter main loop */
@@ -1056,7 +1056,7 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in) {
         vm_maybe_auto_suspend(1);
         vm_assert_type(stack[B], JANET_FIBER);
         JanetFiber *child = janet_unwrap_fiber(stack[B]);
-        if (janet_check_can_resume(child, &retreg)) {
+        if (janet_check_can_resume(child, &retreg, 0)) {
             vm_commit();
             janet_panicv(retreg);
         }
@@ -1096,7 +1096,7 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in) {
         Janet retreg;
         vm_assert_type(stack[B], JANET_FIBER);
         JanetFiber *child = janet_unwrap_fiber(stack[B]);
-        if (janet_check_can_resume(child, &retreg)) {
+        if (janet_check_can_resume(child, &retreg, 1)) {
             vm_commit();
             janet_panicv(retreg);
         }
@@ -1330,7 +1330,7 @@ Janet janet_call(JanetFunction *fun, int32_t argc, const Janet *argv) {
     return *janet_vm.return_reg;
 }
 
-static JanetSignal janet_check_can_resume(JanetFiber *fiber, Janet *out) {
+static JanetSignal janet_check_can_resume(JanetFiber *fiber, Janet *out, int is_cancel) {
     /* Check conditions */
     JanetFiberStatus old_status = janet_fiber_status(fiber);
     if (janet_vm.stackn >= JANET_RECURSION_GUARD) {
@@ -1338,6 +1338,24 @@ static JanetSignal janet_check_can_resume(JanetFiber *fiber, Janet *out) {
         *out = janet_cstringv("C stack recursed too deeply");
         return JANET_SIGNAL_ERROR;
     }
+    /* If a "task" fiber is trying to be used as a normal fiber, detect that and fix the reference
+     * count. See bug #920. */
+    if (janet_vm.stackn > 0 && (fiber->gc.flags & JANET_FIBER_FLAG_ROOT)) {
+        *out = janet_cstringv(is_cancel
+                              ? "cannot cancel root fiber"
+#ifdef JANET_EV
+                              ", use ev/cancel"
+#endif
+                              : "cannot resume root fiber"
+#ifdef JANET_EV
+                              ", use ev/go"
+#endif
+                             );
+        return JANET_SIGNAL_ERROR;
+    }
+    if (janet_vm.stackn == 0) {
+        fiber->gc.flags |= JANET_FIBER_FLAG_ROOT;
+    }
     if (old_status == JANET_STATUS_ALIVE ||
             old_status == JANET_STATUS_DEAD ||
             (old_status >= JANET_STATUS_USER0 && old_status <= JANET_STATUS_USER4) ||
@@ -1452,14 +1470,14 @@ static JanetSignal janet_continue_no_check(JanetFiber *fiber, Janet in, Janet *o
 /* Enter the main vm loop */
 JanetSignal janet_continue(JanetFiber *fiber, Janet in, Janet *out) {
     /* Check conditions */
-    JanetSignal tmp_signal = janet_check_can_resume(fiber, out);
+    JanetSignal tmp_signal = janet_check_can_resume(fiber, out, 0);
     if (tmp_signal) return tmp_signal;
     return janet_continue_no_check(fiber, in, out);
 }
 
 /* Enter the main vm loop but immediately raise a signal */
 JanetSignal janet_continue_signal(JanetFiber *fiber, Janet in, Janet *out, JanetSignal sig) {
-    JanetSignal tmp_signal = janet_check_can_resume(fiber, out);
+    JanetSignal tmp_signal = janet_check_can_resume(fiber, out, sig != JANET_SIGNAL_OK);
     if (tmp_signal) return tmp_signal;
     if (sig != JANET_SIGNAL_OK) {
         JanetFiber *child = fiber;