Add matchcap combinator to pegs.

This allows running rules on captures of pegs. This is flexible, but
loses some guarantees about rules only operating on the source string.
This means that line and column information will be incorrect inside the
matchcap combinator.

.builds/linux.yml

@@ -4,12 +4,20 @@ sources:
 packages:
 - meson
 tasks:
-- build: |
+- with-epoll: |
     cd janet
-    meson setup build --buildtype=release
-    cd build
+    meson setup with-epoll --buildtype=release
+    cd with-epoll
+    meson configure -Depoll=true
+    ninja
+    ninja test
+- no-epoll: |
+    cd janet
+    meson setup no-epoll --buildtype=release
+    cd no-epoll
     meson configure -Depoll=false
     ninja
     ninja test
     sudo ninja install
     sudo jpm --verbose install circlet
+    sudo jpm --verbose install spork

.gitignore

@@ -140,3 +140,6 @@ compile_commands.json
 CTestTestfile.cmake
 
 # End of https://www.gitignore.io/api/cmake
+
+# Astyle
+*.orig

CHANGELOG.md

@@ -1,16 +1,60 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
-## Unreleased - ???
+## 1.15.2 - 2021-02-15
+- Fix bug in windows version of `os/spawn` and `os/execute` with setting environment variables.
+- Fix documentation typos.
+- Fix peg integer reading combinators when used with capture tags.
+
+## 1.15.0 - 2021-02-08
+- Fix `gtim` and `ltim` bytecode instructions on non-integer values.
+- Clean up output of flychecking to be the same as the repl.
+- Change behavior of `debug/stacktrace` with a nil error value.
+- Add optional argument to `parser/produce`.
+- Add `no-core` option to creating standalone binaries to make execution faster.
+- Fix bug where a buffer overflow could be confused with an out of memory error.
+- Change error output to `file:line:column: message`. Column is in bytes - tabs
+  are considered to have width 1 (instead of 8).
+
+## 1.14.2 - 2021-01-23
+- Allow `JANET_PROFILE` env variable to load a profile before loading the repl.
+- Update `tracev` macro to allow `def` and `var` inside to work as expected.
+- Use `(dyn :peg-grammar)` for passing a default grammar to `peg/compile` instead of loading
+  `default-peg-grammar` directly from the root environment.
+- Add `ev/thread` for combining threading with the event loop.
+- Add `ev/do-thread` to make `ev/thread` easier to use.
+- Automatically set supervisor channel in `net/accept-loop` and `net/server` correctly.
+
+## 1.14.1 - 2021-01-18
+- Add `doc-of` for reverse documentation lookup.
+- Add `ev/give-supervsior` to send a message to the supervising channel.
+- Add `ev/gather` and `chan` argument to `ev/go`. This new argument allows "supervisor channels"
+  for fibers to enable structured concurrency.
+- Make `-k` flag work on stdin if no files are given.
+- Add `flycheck` function to core.
+- Make `backmatch` and `backref` more expressive in pegs.
+- Fix buggy `string/split`.
+- Add `fiber/last-value` to get the value that was last yielded, errored, or signaled
+  by a fiber.
+- Remove `:generate` verb from `loop` macros. Instead, use the `:in` verb
+  which will now work on fibers as well as other data structures.
+- Define `next`, `get`, and `in` for fibers. This lets
+  `each`, `map`, and similar iteration macros can now iterate over fibers.
+- Remove macro `eachy`, which can be replaced by `each`.
+- Add `dflt` argument to find-index.
 - Deprecate `file/popen` in favor of `os/spawn`.
 - Add `:all` keyword to `ev/read` and `net/read` to make them more like `file/read`. However, we
   do not provide any `:line` option as that requires buffering.
 - Change repl behavior to make Ctrl-C raise SIGINT on posix. The old behavior for Ctrl-C,
   to clear the current line buffer, has been moved to Ctrl-Q.
 - Importing modules that start with `/` is now the only way to import from project root.
-  Before, this would import from / on disk.
+  Before, this would import from / on disk. Previous imports that did not start with `.` or `/`
+  are now unambiguously importing from the syspath, instead of checking both the syspath and
+  the project root. This is backwards incompatible and dependencies should be updated for this.
 - Change hash function for numbers.
 - Improve error handling of `dofile`.
+- Bug fixes in networking and subprocess code.
+- Use markdown formatting in more places for docstrings.
 
 ## 1.13.1 - 2020-12-13
 - Pretty printing a table with a prototype will look for `:_name` instead of `:name`

Makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Calvin Rose
+# Copyright (c) 2021 Calvin Rose
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to
@@ -27,7 +27,7 @@ PREFIX?=/usr/local
 INCLUDEDIR?=$(PREFIX)/include
 BINDIR?=$(PREFIX)/bin
 LIBDIR?=$(PREFIX)/lib
-JANET_BUILD?="\"$(shell git log --pretty=format:'%h' -n 1 || echo local)\""
+JANET_BUILD?="\"$(shell git log --pretty=format:'%h' -n 2> /dev/null || echo local)\""
 CLIBS=-lm -lpthread
 JANET_TARGET=build/janet
 JANET_LIBRARY=build/libjanet.so
@@ -66,7 +66,7 @@ ifeq ($(UNAME), Haiku)
 	LDFLAGS=-Wl,--export-dynamic
 endif
 
-$(shell mkdir -p build/core build/mainclient build/webclient build/boot)
+$(shell mkdir -p build/core build/c build/boot)
 all: $(JANET_TARGET) $(JANET_LIBRARY) $(JANET_STATIC_LIBRARY) build/janet.h
 
 ######################
@@ -149,7 +149,7 @@ build/janet_boot: $(JANET_BOOT_OBJECTS)
 	$(CC) $(BOOT_CFLAGS) -o $@ $(JANET_BOOT_OBJECTS) $(CLIBS)
 
 # Now the reason we bootstrap in the first place
-build/janet.c: build/janet_boot src/boot/boot.janet
+build/c/janet.c: build/janet_boot src/boot/boot.janet
 	build/janet_boot . JANET_PATH '$(JANET_PATH)' > $@
 	cksum $@
 
@@ -157,9 +157,9 @@ build/janet.c: build/janet_boot src/boot/boot.janet
 ##### Amalgamation #####
 ########################
 
-SONAME=libjanet.so.1.13
+SONAME=libjanet.so.1.15
 
-build/shell.c: src/mainclient/shell.c
+build/c/shell.c: src/mainclient/shell.c
 	cp $< $@
 
 build/janet.h: $(JANET_TARGET) src/include/janet.h src/conf/janetconf.h
@@ -168,11 +168,11 @@ build/janet.h: $(JANET_TARGET) src/include/janet.h src/conf/janetconf.h
 build/janetconf.h: src/conf/janetconf.h
 	cp $< $@
 
-build/janet.o: build/janet.c src/include/janet.h src/conf/janetconf.h
-	$(HOSTCC) $(BUILD_CFLAGS) -c $< -o $@ -I build
+build/janet.o: build/c/janet.c src/conf/janetconf.h src/include/janet.h
+	$(HOSTCC) $(BUILD_CFLAGS) -c $< -o $@
 
-build/shell.o: build/shell.c src/include/janet.h src/conf/janetconf.h
-	$(HOSTCC) $(BUILD_CFLAGS) -c $< -o $@ -I build
+build/shell.o: build/c/shell.c src/conf/janetconf.h src/include/janet.h
+	$(HOSTCC) $(BUILD_CFLAGS) -c $< -o $@
 
 $(JANET_TARGET): build/janet.o build/shell.o
 	$(HOSTCC) $(LDFLAGS) $(BUILD_CFLAGS) -o $@ $^ $(CLIBS)
@@ -224,7 +224,7 @@ dist: build/janet-dist.tar.gz
 build/janet-%.tar.gz: $(JANET_TARGET) \
 	build/janet.h \
 	jpm.1 janet.1 LICENSE CONTRIBUTING.md $(JANET_LIBRARY) $(JANET_STATIC_LIBRARY) \
-	build/doc.html README.md build/janet.c build/shell.c jpm
+	build/doc.html README.md build/c/janet.c build/c/shell.c jpm
 	$(eval JANET_DIST_DIR = "janet-$(shell basename $*)")
 	mkdir -p build/$(JANET_DIST_DIR)
 	cp -r $^ build/$(JANET_DIST_DIR)/

README.md

@@ -58,9 +58,9 @@ Documentation is also available locally in the REPL.
 Use the `(doc symbol-name)` macro to get API
 documentation for symbols in the core library. For example,
 ```
-(doc doc)
+(doc apply)
 ```
-Shows documentation for the doc macro.
+Shows documentation for the `apply` function.
 
 To get a list of all bindings in the default
 environment, use the `(all-bindings)` function. You

build_win.bat

@@ -30,7 +30,7 @@ if not "%JANET_BUILD%" == "" (
 
 if not exist build mkdir build
 if not exist build\core mkdir build\core
-if not exist build\mainclient mkdir build\mainclient
+if not exist build\c mkdir build\c
 if not exist build\boot mkdir build\boot
 
 @rem Build the bootstrap interpreter
@@ -44,10 +44,10 @@ for %%f in (src\boot\*.c) do (
 )
 %JANET_LINK% /out:build\janet_boot.exe build\boot\*.obj
 @if errorlevel 1 goto :BUILDFAIL
-build\janet_boot . > build\janet.c
+build\janet_boot . > build\c\janet.c
 
 @rem Build the sources
-%JANET_COMPILE% /Fobuild\janet.obj build\janet.c
+%JANET_COMPILE% /Fobuild\janet.obj build\c\janet.c
 @if errorlevel 1 goto :BUILDFAIL
 %JANET_COMPILE% /Fobuild\shell.obj src\mainclient\shell.c
 @if errorlevel 1 goto :BUILDFAIL
@@ -102,9 +102,9 @@ exit /b 0
 mkdir dist
 janet.exe tools\gendoc.janet > dist\doc.html
 janet.exe tools\removecr.janet dist\doc.html
-janet.exe tools\removecr.janet build\janet.c
+janet.exe tools\removecr.janet build\c\janet.c
 
-copy build\janet.c dist\janet.c
+copy build\c\janet.c dist\janet.c
 copy src\mainclient\shell.c dist\shell.c
 copy janet.exe dist\janet.exe
 copy LICENSE dist\LICENSE
@@ -118,7 +118,6 @@ copy build\janet.h dist\janet.h
 copy build\libjanet.lib dist\libjanet.lib
 
 copy .\jpm dist\jpm
-copy tools\jpm.bat dist\jpm.bat
 
 @rem Create installer
 janet.exe -e "(->> janet/version (peg/match ''(* :d+ `.` :d+ `.` :d+)) first print)" > build\version.txt

examples/iterate-fiber.janet

@@ -0,0 +1,19 @@
+(def f
+  (coro
+    (for i 0 10
+      (yield (string "yield " i))
+      (os/sleep 0))))
+
+(print "simple yielding")
+(each item f (print "got: " item ", now " (fiber/status f)))
+
+(def f
+  (coro
+    (for i 0 10
+      (yield (string "yield " i))
+      (ev/sleep 0))))
+
+(print "complex yielding")
+(each item f (print "got: " item ", now " (fiber/status f)))
+
+(print (fiber/status f))

examples/udpserver.janet

@@ -1,4 +1,4 @@
-(def server (net/server "127.0.0.1" "8009" nil :datagram))
+(def server (net/listen "127.0.0.1" "8009" :datagram))
 (while true
   (def buf @"")
   (def who (:recv-from server 1024 buf))

janet.1

@@ -175,6 +175,10 @@ Disable ANSI colors in the repl. Has no effect if no repl is run.
 Open a REPL (Read Eval Print Loop) after executing all sources. By default, if Janet is called with no
 arguments, a REPL is opened.
 
+.TP
+.BR \-R
+If using the REPL, disable loading the user profile from the JANET_PROFILE environment variable.
+
 .TP
 .BR \-p
 Turn on the persistent flag. By default, when Janet is executing commands from a file and encounters an error,
@@ -221,6 +225,12 @@ find native and source code modules. If no JANET_PATH is set, Janet will look in
 the default location set at compile time.
 .RE
 
+.B JANET_PROFILE
+.RS
+Path to a profile file that the interpreter will load before entering the REPL. This profile file will
+not run for scripts, though. This behavior can be disabled with the -R option.
+.RE
+
 .B JANET_HASHSEED
 .RS
 To disable randomization of Janet's PRF on start up, one can set this variable. This can have the

jpm

@@ -22,6 +22,19 @@
 ###START###
 
 # Overriden on some installs.
+# To configure this script, replace the code between
+# the START and END comments and define a function
+# (install-paths) that gives the the default paths
+# to use. Trailing directory separator not expected.
+#
+# Example.
+#
+#   (defn- install-paths []
+#     {:headerpath "/usr/local/include/janet"
+#      :libpath "/usr/local/lib/janet"
+#      :binpath "/usr/local/bin"
+#
+
 (def- exe-dir
   "Directory containing jpm script"
   (do
@@ -29,9 +42,13 @@
     (def i (last (string/find-all sep exe)))
     (slice exe 0 i)))
 
+(defn- try-real [path]
+  "If os/realpath fails just use normal path."
+  (try (os/realpath) ([_] path)))
+
 (defn- install-paths []
-  {:headerpath (os/realpath (string exe-dir "/../include/janet"))
-   :libpath (os/realpath (string exe-dir "/../lib"))
+  {:headerpath (try-real (string exe-dir "/../include/janet"))
+   :libpath (try-real (string exe-dir "/../lib"))
    :binpath exe-dir})
 
 ###END###
@@ -152,9 +169,7 @@
   [& args]
   (if (dyn :verbose)
     (print ;(interpose " " args)))
-  (def res (os/execute args :p))
-  (unless (zero? res)
-    (error (string "command exited with status " res))))
+  (os/execute args :px))
 
 (defn copy
   "Copy a file or directory recursively from one location to another."
@@ -585,7 +600,7 @@
   (string (string/slice path 0 (- -1 (length modext))) statext))
 
 (defn- make-bin-source
-  [declarations lookup-into-invocations]
+  [declarations lookup-into-invocations no-core]
   (string
     declarations
     ```
@@ -610,15 +625,22 @@ int main(int argc, const char **argv) {
 
     janet_init();
 
+    ```
+    (if no-core
+    ```
+    /* Get core env */
+    JanetTable *env = janet_table(8);
+    JanetTable *lookup = janet_core_lookup_table(NULL);
+    JanetTable *temptab;
+    int handle = janet_gclock();
+    ```
+    ```
     /* Get core env */
     JanetTable *env = janet_core_env(NULL);
     JanetTable *lookup = janet_env_lookup(env);
     JanetTable *temptab;
     int handle = janet_gclock();
-
-    /* Load natives into unmarshalling dictionary */
-
-    ```
+    ```)
     lookup-into-invocations
     ```
     /* Unmarshal bytecode */
@@ -646,7 +668,6 @@ int main(int argc, const char **argv) {
     }
 
     /* Create enviornment */
-    temptab = janet_table(0);
     temptab = env;
     janet_table_put(temptab, janet_ckeywordv("args"), janet_wrap_array(args));
     janet_gcroot(janet_wrap_table(temptab));
@@ -657,6 +678,14 @@ int main(int argc, const char **argv) {
     /* Run everything */
     JanetFiber *fiber = janet_fiber(jfunc, 64, argc, argc ? args->data : NULL);
     fiber->env = temptab;
+#ifdef JANET_EV
+    janet_gcroot(janet_wrap_fiber(fiber));
+    janet_schedule(fiber, janet_wrap_nil());
+    janet_loop();
+    int status = janet_fiber_status(fiber);
+    janet_deinit();
+    return status;
+#else
     Janet out;
     JanetSignal result = janet_continue(fiber, janet_wrap_nil(), &out);
     if (result != JANET_SIGNAL_OK && result != JANET_SIGNAL_EVENT) {
@@ -664,11 +693,9 @@ int main(int argc, const char **argv) {
       janet_deinit();
       return result;
     }
-#ifdef JANET_NET
-    janet_loop();
-#endif
     janet_deinit();
     return 0;
+#endif
 }
 
 ```))
@@ -677,7 +704,7 @@ int main(int argc, const char **argv) {
   "Links an image with libjanet.a (or .lib) to produce an
   executable. Also will try to link native modules into the
   final executable as well."
-  [opts source dest]
+  [opts source dest no-core]
 
   # Create executable's janet image
   (def cimage_dest (string dest ".c"))
@@ -693,7 +720,16 @@ int main(int argc, const char **argv) {
         (def dep-ldflags @[])
 
         # Create marshalling dictionary
-        (def mdict (invert (env-lookup root-env)))
+        (def mdict1 (invert (env-lookup root-env)))
+        (def mdict
+          (if no-core
+            (let [temp @{}]
+              (eachp [k v] mdict1
+                (if (or (cfunction? k) (abstract? k))
+                  (put temp k v)))
+              temp)
+            mdict1))
+
         # Load all native modules
         (def prefixes @{})
         (def static-libs @[])
@@ -738,7 +774,7 @@ int main(int argc, const char **argv) {
         # Make image byte buffer
         (create-buffer-c-impl image cimage_dest "janet_payload_image")
         # Append main function
-        (spit cimage_dest (make-bin-source declarations lookup-into-invocations) :ab)
+        (spit cimage_dest (make-bin-source declarations lookup-into-invocations no-core) :ab)
         (def oimage_dest (out-path cimage_dest ".c" ".o"))
         # Compile and link final exectable
         (unless no-compile
@@ -942,7 +978,7 @@ int main(int argc, const char **argv) {
         (let [op (out-path src ".c" objext)]
           (compile-c opts src op)
           op)
-        (errorf "unknown source file type: %s, expected .c or .cpp"))))
+        (errorf "unknown source file type: %s, expected .c or .cpp" src))))
 
   (when-let [embedded (opts :embedded)]
     (loop [src :in embedded]
@@ -1027,10 +1063,10 @@ int main(int argc, const char **argv) {
   This executable can be installed as well to the --binpath given."
   [&keys {:install install :name name :entry entry :headers headers
           :cflags cflags :lflags lflags :deps deps :ldflags ldflags
-          :no-compile no-compile}]
+          :no-compile no-compile :no-core no-core}]
   (def name (if is-win (string name ".exe") name))
   (def dest (string "build" sep name))
-  (create-executable @{:cflags cflags :lflags lflags :ldflags ldflags :no-compile no-compile} entry dest)
+  (create-executable @{:cflags cflags :lflags lflags :ldflags ldflags :no-compile no-compile} entry dest no-core)
   (if no-compile
     (let [cdest (string dest ".c")]
       (add-dep "build" cdest))
@@ -1337,7 +1373,9 @@ Flags are:
 
 (defn quickbin
   [input output]
-  (create-executable @{} input output)
+  (if (= (os/stat output :mode) :file)
+    (print "output " output " exists."))
+  (create-executable @{:no-compile (dyn :no-compile)} input output (dyn :no-core))
   (do-rule output))
 
 (defn jpm-debug-repl
@@ -1384,26 +1422,30 @@ Flags are:
    "load-lockfile" load-lockfile
    "quickbin" quickbin})
 
-(def- args (tuple/slice (dyn :args) 1))
-(def- len (length args))
-(var i :private 0)
+(defn- main
+  "Script entry."
+  [& argv]
 
-# Get flags
-(while (< i len)
-  (if-let [m (peg/match argpeg (args i))]
-    (if (= 2 (length m))
-      (let [[key value] m]
-        (setdyn (keyword key) value))
-      (setdyn (keyword (m 0)) true))
-    (break))
-  (++ i))
+  (def- args (tuple/slice argv 1))
+  (def- len (length args))
+  (var i :private 0)
 
-# Run subcommand
-(if (= i len)
-  (help)
-  (do
-    (if-let [com (subcommands (args i))]
-      (com ;(tuple/slice args (+ i 1)))
-      (do
-        (print "invalid command " (args i))
-        (help)))))
+  # Get flags
+  (while (< i len)
+    (if-let [m (peg/match argpeg (args i))]
+      (if (= 2 (length m))
+        (let [[key value] m]
+          (setdyn (keyword key) value))
+        (setdyn (keyword (m 0)) true))
+      (break))
+    (++ i))
+
+  # Run subcommand
+  (if (= i len)
+    (help)
+    (do
+      (if-let [com (subcommands (args i))]
+        (com ;(tuple/slice args (+ i 1)))
+        (do
+          (print "invalid command " (args i))
+          (help))))))

meson.build

@@ -20,7 +20,7 @@
 
 project('janet', 'c',
   default_options : ['c_std=c99', 'b_lundef=false', 'default_library=both'],
-  version : '1.13.1')
+  version : '1.15.2')
 
 # Global settings
 janet_path = join_paths(get_option('prefix'), get_option('libdir'), 'janet')

src/conf/janetconf.h

@@ -4,10 +4,10 @@
 #define JANETCONF_H
 
 #define JANET_VERSION_MAJOR 1
-#define JANET_VERSION_MINOR 13
+#define JANET_VERSION_MINOR 15
 #define JANET_VERSION_PATCH 2
-#define JANET_VERSION_EXTRA "-dev"
-#define JANET_VERSION "1.13.2-dev"
+#define JANET_VERSION_EXTRA ""
+#define JANET_VERSION "1.15.2"
 
 /* #define JANET_BUILD "local" */
 

src/core/array.c

@@ -290,6 +290,13 @@ static Janet cfun_array_trim(int32_t argc, Janet *argv) {
     return argv[0];
 }
 
+static Janet cfun_array_clear(int32_t argc, Janet *argv) {
+    janet_fixarity(argc, 1);
+    JanetArray *array = janet_getarray(argv, 0);
+    array->count = 0;
+    return argv[0];
+}
+
 static const JanetReg array_cfuns[] = {
     {
         "array/new", cfun_array_new,
@@ -370,6 +377,12 @@ static const JanetReg array_cfuns[] = {
         JDOC("(array/trim arr)\n\n"
              "Set the backing capacity of an array to its current length. Returns the modified array.")
     },
+    {
+        "array/clear", cfun_array_clear,
+        JDOC("(array/clear arr)\n\n"
+             "Empties an array, setting it's count to 0 but does not free the backing capacity. "
+             "Returns the modified array.")
+    },
     {NULL, NULL, NULL}
 };
 

src/core/asm.c

@@ -992,18 +992,18 @@ static const JanetReg asm_cfuns[] = {
              "func must be a function, not a c function. Will throw on error on a badly\n"
              "typed argument. If given a field name, will only return that part of the function assembly.\n"
              "Possible fields are:\n\n"
-             "\t:arity - number of required and optional arguments.\n"
-             "\t:min-arity - minimum number of arguments function can be called with.\n"
-             "\t:max-arity - maximum number of arguments function can be called with.\n"
-             "\t:vararg - true if function can take a variable number of arguments.\n"
-             "\t:bytecode - array of parsed bytecode instructions. Each instruction is a tuple.\n"
-             "\t:source - name of source file that this function was compiled from.\n"
-             "\t:name - name of function.\n"
-             "\t:slotcount - how many virtual registers, or slots, this function uses. Corresponds to stack space used by function.\n"
-             "\t:constants - an array of constants referenced by this function.\n"
-             "\t:sourcemap - a mapping of each bytecode instruction to a line and column in the source file.\n"
-             "\t:environments - an internal mapping of which enclosing functions are referenced for bindings.\n"
-             "\t:defs - other function definitions that this function may instantiate.\n")
+             "* :arity - number of required and optional arguments.\n\n"
+             "* :min-arity - minimum number of arguments function can be called with.\n\n"
+             "* :max-arity - maximum number of arguments function can be called with.\n\n"
+             "* :vararg - true if function can take a variable number of arguments.\n\n"
+             "* :bytecode - array of parsed bytecode instructions. Each instruction is a tuple.\n\n"
+             "* :source - name of source file that this function was compiled from.\n\n"
+             "* :name - name of function.\n\n"
+             "* :slotcount - how many virtual registers, or slots, this function uses. Corresponds to stack space used by function.\n\n"
+             "* :constants - an array of constants referenced by this function.\n\n"
+             "* :sourcemap - a mapping of each bytecode instruction to a line and column in the source file.\n\n"
+             "* :environments - an internal mapping of which enclosing functions are referenced for bindings.\n\n"
+             "* :defs - other function definitions that this function may instantiate.\n")
     },
     {NULL, NULL, NULL}
 };

src/core/buffer.c

@@ -91,7 +91,7 @@ void janet_buffer_extra(JanetBuffer *buffer, int32_t n) {
     }
     int32_t new_size = buffer->count + n;
     if (new_size > buffer->capacity) {
-        int32_t new_capacity = new_size * 2;
+        int32_t new_capacity = (new_size > (INT32_MAX / 2)) ? INT32_MAX : (new_size * 2);
         uint8_t *new_data = realloc(buffer->data, new_capacity * sizeof(uint8_t));
         janet_gcpressure(new_capacity - buffer->capacity);
         if (NULL == new_data) {

src/core/capi.c

@@ -53,7 +53,9 @@ JANET_NO_RETURN static void janet_top_level_signal(const char *msg) {
 void janet_signalv(JanetSignal sig, Janet message) {
     if (janet_vm_return_reg != NULL) {
         *janet_vm_return_reg = message;
-        janet_vm_fiber->flags |= JANET_FIBER_DID_LONGJUMP;
+        if (NULL != janet_vm_fiber) {
+            janet_vm_fiber->flags |= JANET_FIBER_DID_LONGJUMP;
+        }
 #if defined(JANET_BSD) || defined(JANET_APPLE)
         _longjmp(*janet_vm_jmp_buf, sig);
 #else
@@ -147,6 +149,23 @@ int janet_getmethod(const uint8_t *method, const JanetMethod *methods, Janet *ou
     return 0;
 }
 
+Janet janet_nextmethod(const JanetMethod *methods, Janet key) {
+    if (!janet_checktype(key, JANET_NIL)) {
+        while (methods->name) {
+            if (janet_keyeq(key, methods->name)) {
+                methods++;
+                break;
+            }
+            methods++;
+        }
+    }
+    if (methods->name) {
+        return janet_ckeywordv(methods->name);
+    } else {
+        return janet_wrap_nil();
+    }
+}
+
 DEFINE_GETTER(number, NUMBER, double)
 DEFINE_GETTER(array, ARRAY, JanetArray *)
 DEFINE_GETTER(tuple, TUPLE, const Janet *)

src/core/compile.c

@@ -439,22 +439,22 @@ static JanetSlot janetc_call(JanetFopts opts, JanetSlot *slots, JanetSlot fun) {
                         min_arity = -1 - min_arity;
                         if (min_arity > max && max >= 0) {
                             const uint8_t *es = janet_formatc(
-                                                    "%v expects at most %d argument, got at least %d",
-                                                    fun.constant, max, min_arity);
+                                                    "%v expects at most %d argument%s, got at least %d",
+                                                    fun.constant, max, max == 1 ? "" : "s", min_arity);
                             janetc_error(c, es);
                         }
                     } else {
                         /* Call has no splices */
                         if (min_arity > max && max >= 0) {
                             const uint8_t *es = janet_formatc(
-                                                    "%v expects at most %d argument, got %d",
-                                                    fun.constant, max, min_arity);
+                                                    "%v expects at most %d argument%s, got %d",
+                                                    fun.constant, max, max == 1 ? "" : "s", min_arity);
                             janetc_error(c, es);
                         }
                         if (min_arity < min) {
                             const uint8_t *es = janet_formatc(
-                                                    "%v expects at least %d argument, got %d",
-                                                    fun.constant, min, min_arity);
+                                                    "%v expects at least %d argument%s, got %d",
+                                                    fun.constant, min, min == 1 ? "" : "s", min_arity);
                             janetc_error(c, es);
                         }
                     }
@@ -872,8 +872,12 @@ static Janet cfun(int32_t argc, Janet *argv) {
     } else {
         JanetTable *t = janet_table(4);
         janet_table_put(t, janet_ckeywordv("error"), janet_wrap_string(res.error));
-        janet_table_put(t, janet_ckeywordv("line"), janet_wrap_integer(res.error_mapping.line));
-        janet_table_put(t, janet_ckeywordv("column"), janet_wrap_integer(res.error_mapping.column));
+        if (res.error_mapping.line > 0) {
+            janet_table_put(t, janet_ckeywordv("line"), janet_wrap_integer(res.error_mapping.line));
+        }
+        if (res.error_mapping.column > 0) {
+            janet_table_put(t, janet_ckeywordv("column"), janet_wrap_integer(res.error_mapping.column));
+        }
         if (res.macrofiber) {
             janet_table_put(t, janet_ckeywordv("fiber"), janet_wrap_fiber(res.macrofiber));
         }

src/core/corelib.c

@@ -645,20 +645,21 @@ static const JanetReg corelib_cfuns[] = {
     {
         "type", janet_core_type,
         JDOC("(type x)\n\n"
-             "Returns the type of x as a keyword. x is one of\n"
-             "\t:nil\n"
-             "\t:boolean\n"
-             "\t:number\n"
-             "\t:array\n"
-             "\t:tuple\n"
-             "\t:table\n"
-             "\t:struct\n"
-             "\t:string\n"
-             "\t:buffer\n"
-             "\t:symbol\n"
-             "\t:keyword\n"
-             "\t:function\n"
-             "\t:cfunction\n\n"
+             "Returns the type of `x` as a keyword. `x` is one of:\n\n"
+             "* :nil\n\n"
+             "* :boolean\n\n"
+             "* :number\n\n"
+             "* :array\n\n"
+             "* :tuple\n\n"
+             "* :table\n\n"
+             "* :struct\n\n"
+             "* :string\n\n"
+             "* :buffer\n\n"
+             "* :symbol\n\n"
+             "* :keyword\n\n"
+             "* :function\n\n"
+             "* :cfunction\n\n"
+             "* :fiber\n\n"
              "or another keyword for an abstract type.")
     },
     {
@@ -699,16 +700,16 @@ static const JanetReg corelib_cfuns[] = {
     {
         "module/expand-path", janet_core_expand_path,
         JDOC("(module/expand-path path template)\n\n"
-             "Expands a path template as found in module/paths for module/find. "
-             "This takes in a path (the argument to require) and a template string, template, "
+             "Expands a path template as found in `module/paths` for `module/find`. "
+             "This takes in a path (the argument to require) and a template string, "
              "to expand the path to a path that can be "
              "used for importing files. The replacements are as follows:\n\n"
-             "\t:all:\tthe value of path verbatim\n"
-             "\t:cur:\tthe current file, or (dyn :current-file)\n"
-             "\t:dir:\tthe directory containing the current file\n"
-             "\t:name:\tthe name component of path, with extension if given\n"
-             "\t:native:\tthe extension used to load natives, .so or .dll\n"
-             "\t:sys:\tthe system path, or (dyn :syspath)")
+             "* :all: -- the value of path verbatim\n\n"
+             "* :cur: -- the current file, or (dyn :current-file)\n\n"
+             "* :dir: -- the directory containing the current file\n\n"
+             "* :name: -- the name component of path, with extension if given\n\n"
+             "* :native: -- the extension used to load natives, .so or .dll\n\n"
+             "* :sys: -- the system path, or (dyn :syspath)")
     },
     {
         "int?", janet_core_check_int,
@@ -1205,7 +1206,8 @@ JanetTable *janet_core_env(JanetTable *replacements) {
                    "if native modules are compatible with the host program."));
 
     /* Allow references to the environment */
-    janet_def(env, "_env", janet_wrap_table(env), JDOC("The environment table for the current scope."));
+    janet_def(env, "root-env", janet_wrap_table(env),
+              JDOC("The root environment used to create environments with (make-env)."));
 
     janet_load_libs(env);
     janet_gcroot(janet_wrap_table(env));
@@ -1220,22 +1222,7 @@ JanetTable *janet_core_env(JanetTable *replacements) {
         return janet_vm_core_env;
     }
 
-    /* Load core cfunctions (and some built in janet assembly functions) */
-    JanetTable *dict = janet_table(512);
-    janet_load_libs(dict);
-
-    /* Add replacements */
-    if (replacements != NULL) {
-        for (int32_t i = 0; i < replacements->capacity; i++) {
-            JanetKV kv = replacements->data[i];
-            if (!janet_checktype(kv.key, JANET_NIL)) {
-                janet_table_put(dict, kv.key, kv.value);
-                if (janet_checktype(kv.value, JANET_CFUNCTION)) {
-                    janet_table_put(janet_vm_registry, kv.value, kv.key);
-                }
-            }
-        }
-    }
+    JanetTable *dict = janet_core_lookup_table(replacements);
 
     /* Unmarshal bytecode */
     Janet marsh_out = janet_unmarshal(
@@ -1269,3 +1256,23 @@ JanetTable *janet_core_env(JanetTable *replacements) {
 }
 
 #endif
+
+JanetTable *janet_core_lookup_table(JanetTable *replacements) {
+    JanetTable *dict = janet_table(512);
+    janet_load_libs(dict);
+
+    /* Add replacements */
+    if (replacements != NULL) {
+        for (int32_t i = 0; i < replacements->capacity; i++) {
+            JanetKV kv = replacements->data[i];
+            if (!janet_checktype(kv.key, JANET_NIL)) {
+                janet_table_put(dict, kv.key, kv.value);
+                if (janet_checktype(kv.value, JANET_CFUNCTION)) {
+                    janet_table_put(janet_vm_registry, kv.value, kv.key);
+                }
+            }
+        }
+    }
+
+    return dict;
+}

src/core/debug.c

@@ -102,7 +102,9 @@ void janet_stacktrace(JanetFiber *fiber, Janet err) {
     int32_t fi;
     const char *errstr = (const char *)janet_to_string(err);
     JanetFiber **fibers = NULL;
-    int wrote_error = 0;
+
+    /* Don't print error line if it is nil. */
+    int wrote_error = janet_checktype(err, JANET_NIL);
 
     int print_color = janet_truthy(janet_dyn("err-color"));
     if (print_color) janet_eprintf("\x1b[31m");
@@ -299,9 +301,10 @@ static Janet cfun_debug_stack(int32_t argc, Janet *argv) {
 }
 
 static Janet cfun_debug_stacktrace(int32_t argc, Janet *argv) {
-    janet_fixarity(argc, 2);
+    janet_arity(argc, 1, 2);
     JanetFiber *fiber = janet_getfiber(argv, 0);
-    janet_stacktrace(fiber, argv[1]);
+    Janet x = argc == 1 ? janet_wrap_nil() : argv[1];
+    janet_stacktrace(fiber, x);
     return argv[0];
 }
 
@@ -325,12 +328,12 @@ static Janet cfun_debug_step(int32_t argc, Janet *argv) {
 static const JanetReg debug_cfuns[] = {
     {
         "debug/break", cfun_debug_break,
-        JDOC("(debug/break source byte-offset)\n\n"
-             "Sets a breakpoint with source a key at a given line and column. "
+        JDOC("(debug/break source line col)\n\n"
+             "Sets a breakpoint in `source` at a given line and column. "
              "Will throw an error if the breakpoint location "
              "cannot be found. For example\n\n"
-             "\t(debug/break \"core.janet\" 1000)\n\n"
-             "wil set a breakpoint at the 1000th byte of the file core.janet.")
+             "\t(debug/break \"core.janet\" 10 4)\n\n"
+             "wil set a breakpoint at line 10, 4th column of the file core.janet.")
     },
     {
         "debug/unbreak", cfun_debug_unbreak,
@@ -362,25 +365,25 @@ static const JanetReg debug_cfuns[] = {
         "debug/stack", cfun_debug_stack,
         JDOC("(debug/stack fib)\n\n"
              "Gets information about the stack as an array of tables. Each table "
-             "in the array contains information about a stack frame. The top most, current "
-             "stack frame is the first table in the array, and the bottom most stack frame "
+             "in the array contains information about a stack frame. The top-most, current "
+             "stack frame is the first table in the array, and the bottom-most stack frame "
              "is the last value. Each stack frame contains some of the following attributes:\n\n"
-             "\t:c - true if the stack frame is a c function invocation\n"
-             "\t:column - the current source column of the stack frame\n"
-             "\t:function - the function that the stack frame represents\n"
-             "\t:line - the current source line of the stack frame\n"
-             "\t:name - the human friendly name of the function\n"
-             "\t:pc - integer indicating the location of the program counter\n"
-             "\t:source - string with the file path or other identifier for the source code\n"
-             "\t:slots - array of all values in each slot\n"
-             "\t:tail - boolean indicating a tail call")
+             "* :c - true if the stack frame is a c function invocation\n\n"
+             "* :column - the current source column of the stack frame\n\n"
+             "* :function - the function that the stack frame represents\n\n"
+             "* :line - the current source line of the stack frame\n\n"
+             "* :name - the human-friendly name of the function\n\n"
+             "* :pc - integer indicating the location of the program counter\n\n"
+             "* :source - string with the file path or other identifier for the source code\n\n"
+             "* :slots - array of all values in each slot\n\n"
+             "* :tail - boolean indicating a tail call")
     },
     {
         "debug/stacktrace", cfun_debug_stacktrace,
-        JDOC("(debug/stacktrace fiber err)\n\n"
-             "Prints a nice looking stacktrace for a fiber. The error message "
-             "err must be passed to the function as fiber's do not keep track of "
-             "the last error they have thrown. Returns the fiber.")
+        JDOC("(debug/stacktrace fiber &opt err)\n\n"
+             "Prints a nice looking stacktrace for a fiber. Can optionally provide "
+             "an error value to print the stack trace with. If `err` is nil or not "
+             "provided, will skipp the error line. Returns the fiber.")
     },
     {
         "debug/lineage", cfun_debug_lineage,

src/core/ev.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose and contributors.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -63,6 +63,23 @@ typedef struct {
     void *data;
 } JanetQueue;
 
+typedef struct {
+    JanetFiber *fiber;
+    uint32_t sched_id;
+    enum {
+        JANET_CP_MODE_ITEM,
+        JANET_CP_MODE_CHOICE_READ,
+        JANET_CP_MODE_CHOICE_WRITE
+    } mode;
+} JanetChannelPending;
+
+typedef struct {
+    JanetQueue items;
+    JanetQueue read_pending;
+    JanetQueue write_pending;
+    int32_t limit;
+} JanetChannel;
+
 #define JANET_MAX_Q_CAPACITY 0x7FFFFFF
 
 static void janet_q_init(JanetQueue *q) {
@@ -304,21 +321,6 @@ JanetStream *janet_stream(JanetHandle handle, uint32_t flags, const JanetMethod
     stream->_mask = 0;
     if (methods == NULL) methods = ev_default_stream_methods;
     stream->methods = methods;
-#ifdef JANET_NET
-    if (flags & JANET_STREAM_SOCKET) {
-#ifdef JANET_WINDOWS
-        u_long iMode = 0;
-        ioctlsocket((SOCKET) handle, FIONBIO, &iMode);
-#else
-#if !defined(SOCK_CLOEXEC) && defined(O_CLOEXEC)
-        int extra = O_CLOEXEC;
-#else
-        int extra = 0;
-#endif
-        fcntl(handle, F_SETFL, fcntl(handle, F_GETFL, 0) | O_NONBLOCK | extra);
-#endif
-    }
-#endif
     return stream;
 }
 
@@ -376,7 +378,6 @@ static int janet_stream_getter(void *p, Janet key, Janet *out) {
     if (!janet_checktype(key, JANET_KEYWORD)) return 0;
     const JanetMethod *stream_methods = stream->methods;
     return janet_getmethod(janet_unwrap_keyword(key), stream_methods, out);
-    return 0;
 }
 
 static void janet_stream_marshal(void *p, JanetMarshalContext *ctx) {
@@ -392,7 +393,20 @@ static void janet_stream_marshal(void *p, JanetMarshalContext *ctx) {
      * while in transit, and it's value gets reused. DuplicateHandle does not work
      * for network sockets, and in general for winsock it is better to nipt duplicate
      * unless there is a need to. */
-    janet_marshal_int64(ctx, (int64_t)(s->handle));
+    HANDLE duph = INVALID_HANDLE_VALUE;
+    if (s->flags & JANET_STREAM_SOCKET) {
+        duph = s->handle;
+    } else {
+        DuplicateHandle(
+            GetCurrentProcess(),
+            s->handle,
+            GetCurrentProcess(),
+            &duph,
+            0,
+            FALSE,
+            DUPLICATE_SAME_ACCESS);
+    }
+    janet_marshal_int64(ctx, (int64_t)(duph));
 #else
     /* Marshal after dup becuse it is easier than maintaining our own ref counting. */
     int duph = dup(s->handle);
@@ -419,6 +433,10 @@ static void *janet_stream_unmarshal(JanetMarshalContext *ctx) {
     return p;
 }
 
+static Janet janet_stream_next(void *p, Janet key) {
+    JanetStream *stream = (JanetStream *)p;
+    return janet_nextmethod(stream->methods, key);
+}
 
 const JanetAbstractType janet_stream_type = {
     "core/stream",
@@ -428,7 +446,11 @@ const JanetAbstractType janet_stream_type = {
     NULL,
     janet_stream_marshal,
     janet_stream_unmarshal,
-    JANET_ATEND_UNMARSHAL
+    NULL,
+    NULL,
+    NULL,
+    janet_stream_next,
+    JANET_ATEND_NEXT
 };
 
 /* Register a fiber to resume with value */
@@ -496,13 +518,27 @@ void janet_ev_mark(void) {
     }
 }
 
+static int janet_channel_push(JanetChannel *channel, Janet x, int mode);
+
+static Janet make_supervisor_event(const char *name, JanetFiber *fiber) {
+    Janet tup[2];
+    tup[0] = janet_ckeywordv(name);
+    tup[1] = janet_wrap_fiber(fiber);
+    return janet_wrap_tuple(janet_tuple_n(tup, 2));
+}
+
 /* Run a top level task */
 static void run_one(JanetFiber *fiber, Janet value, JanetSignal sigin) {
     fiber->flags &= ~JANET_FIBER_FLAG_SCHEDULED;
     Janet res;
     JanetSignal sig = janet_continue_signal(fiber, value, &res, sigin);
-    if (sig != JANET_SIGNAL_OK && sig != JANET_SIGNAL_EVENT) {
-        janet_stacktrace(fiber, res);
+    JanetChannel *chan = (JanetChannel *)(fiber->supervisor_channel);
+    if (NULL == chan) {
+        if (sig != JANET_SIGNAL_EVENT && sig != JANET_SIGNAL_YIELD) {
+            janet_stacktrace(fiber, res);
+        }
+    } else if (sig == JANET_SIGNAL_OK || (fiber->flags & (1 << sig))) {
+        janet_channel_push(chan, make_supervisor_event(janet_signal_names[sig], fiber), 2);
     }
 }
 
@@ -553,23 +589,6 @@ void janet_ev_dec_refcount(void) {
 
 /* Channels */
 
-typedef struct {
-    JanetFiber *fiber;
-    uint32_t sched_id;
-    enum {
-        JANET_CP_MODE_ITEM,
-        JANET_CP_MODE_CHOICE_READ,
-        JANET_CP_MODE_CHOICE_WRITE
-    } mode;
-} JanetChannelPending;
-
-typedef struct {
-    JanetQueue items;
-    JanetQueue read_pending;
-    JanetQueue write_pending;
-    int32_t limit;
-} JanetChannel;
-
 #define JANET_MAX_CHANNEL_CAPACITY 0xFFFFFF
 
 static void janet_chan_init(JanetChannel *chan, int32_t limit) {
@@ -589,16 +608,24 @@ static void janet_chan_deinit(JanetChannel *chan) {
  * Janet Channel abstract type
  */
 
-/*static int janet_chanat_get(void *p, Janet key, Janet *out);*/
 static int janet_chanat_mark(void *p, size_t s);
 static int janet_chanat_gc(void *p, size_t s);
+static Janet janet_chanat_next(void *p, Janet key);
+static int janet_chanat_get(void *p, Janet key, Janet *out);
 
 static const JanetAbstractType ChannelAT = {
     "core/channel",
     janet_chanat_gc,
     janet_chanat_mark,
-    NULL, /* janet_chanat_get */
-    JANET_ATEND_GET
+    janet_chanat_get,
+    NULL, /* put */
+    NULL, /* marshal */
+    NULL, /* unmarshal */
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    janet_chanat_next,
+    JANET_ATEND_NEXT
 };
 
 static int janet_chanat_gc(void *p, size_t s) {
@@ -657,7 +684,7 @@ static Janet make_read_result(JanetChannel *channel, Janet x) {
 
 /* Push a value to a channel, and return 1 if channel should block, zero otherwise.
  * If the push would block, will add to the write_pending queue in the channel. */
-static int janet_channel_push(JanetChannel *channel, Janet x, int is_choice) {
+static int janet_channel_push(JanetChannel *channel, Janet x, int mode) {
     JanetChannelPending reader;
     int is_empty;
     do {
@@ -668,11 +695,13 @@ static int janet_channel_push(JanetChannel *channel, Janet x, int is_choice) {
         if (janet_q_push(&channel->items, &x, sizeof(Janet))) {
             janet_panicf("channel overflow: %v", x);
         } else if (janet_q_count(&channel->items) > channel->limit) {
+            /* No root fiber, we are in completion on a root fiber. Don't block. */
+            if (mode == 2) return 0;
             /* Pushed successfully, but should block. */
             JanetChannelPending pending;
             pending.fiber = janet_vm_root_fiber,
             pending.sched_id = janet_vm_root_fiber->sched_id,
-            pending.mode = is_choice ? JANET_CP_MODE_CHOICE_WRITE : JANET_CP_MODE_ITEM;
+            pending.mode = mode ? JANET_CP_MODE_CHOICE_WRITE : JANET_CP_MODE_ITEM;
             janet_q_push(&channel->write_pending, &pending, sizeof(pending));
             return 1;
         }
@@ -816,6 +845,28 @@ static Janet cfun_channel_new(int32_t argc, Janet *argv) {
     return janet_wrap_abstract(channel);
 }
 
+static const JanetMethod ev_chanat_methods[] = {
+    {"select", cfun_channel_choice},
+    {"rselect", cfun_channel_rchoice},
+    {"count", cfun_channel_count},
+    {"take", cfun_channel_pop},
+    {"give", cfun_channel_push},
+    {"capacity", cfun_channel_capacity},
+    {"full", cfun_channel_full},
+    {NULL, NULL}
+};
+
+static int janet_chanat_get(void *p, Janet key, Janet *out) {
+    (void) p;
+    if (!janet_checktype(key, JANET_KEYWORD)) return 0;
+    return janet_getmethod(janet_unwrap_keyword(key), ev_chanat_methods, out);
+}
+
+static Janet janet_chanat_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(ev_chanat_methods, key);
+}
+
 /* Main event loop */
 
 void janet_loop1_impl(int has_timeout, JanetTimestamp timeout);
@@ -910,7 +961,7 @@ typedef struct {
 static JANET_THREAD_LOCAL JanetHandle janet_vm_selfpipe[2];
 
 static void janet_ev_setup_selfpipe(void) {
-    if (janet_make_pipe(janet_vm_selfpipe)) {
+    if (janet_make_pipe(janet_vm_selfpipe, 0)) {
         JANET_EXIT("failed to initialize self pipe in event loop");
     }
 }
@@ -955,7 +1006,7 @@ JanetListenerState *janet_listen(JanetStream *stream, JanetListener behavior, in
     JanetListenerState *state = janet_listen_impl(stream, behavior, mask, size, user);
     if (!(stream->flags & JANET_STREAM_IOCP)) {
         if (NULL == CreateIoCompletionPort(stream->handle, janet_vm_iocp, (ULONG_PTR) stream, 0)) {
-            janet_panic("failed to listen for events");
+            janet_panicf("failed to listen for events: %V", janet_ev_lasterr());
         }
         stream->flags |= JANET_STREAM_IOCP;
     }
@@ -970,7 +1021,7 @@ static void janet_unlisten(JanetListenerState *state) {
 void janet_loop1_impl(int has_timeout, JanetTimestamp to) {
     ULONG_PTR completionKey = 0;
     DWORD num_bytes_transfered = 0;
-    LPOVERLAPPED overlapped;
+    LPOVERLAPPED overlapped = NULL;
 
     /* Calculate how long to wait before timeout */
     uint64_t waittime;
@@ -986,31 +1037,29 @@ void janet_loop1_impl(int has_timeout, JanetTimestamp to) {
     }
     BOOL result = GetQueuedCompletionStatus(janet_vm_iocp, &num_bytes_transfered, &completionKey, &overlapped, (DWORD) waittime);
 
-    if (!result) {
-        if (!has_timeout) {
-            /* queue emptied */
-        }
-    } else if (0 == completionKey) {
-        /* Custom event */
-        JanetSelfPipeEvent *response = (JanetSelfPipeEvent *)(overlapped);
-        response->cb(response->msg);
-        free(response);
-        janet_ev_dec_refcount();
-    } else {
-        /* Normal event */
-        JanetStream *stream = (JanetStream *) completionKey;
-        JanetListenerState *state = stream->state;
-        while (state != NULL) {
-            if (state->tag == overlapped) {
-                state->event = overlapped;
-                state->bytes = num_bytes_transfered;
-                JanetAsyncStatus status = state->machine(state, JANET_ASYNC_EVENT_COMPLETE);
-                if (status == JANET_ASYNC_STATUS_DONE) {
-                    janet_unlisten(state);
+    if (result || overlapped) {
+        if (0 == completionKey) {
+            /* Custom event */
+            JanetSelfPipeEvent *response = (JanetSelfPipeEvent *)(overlapped);
+            response->cb(response->msg);
+            free(response);
+            janet_ev_dec_refcount();
+        } else {
+            /* Normal event */
+            JanetStream *stream = (JanetStream *) completionKey;
+            JanetListenerState *state = stream->state;
+            while (state != NULL) {
+                if (state->tag == overlapped) {
+                    state->event = overlapped;
+                    state->bytes = num_bytes_transfered;
+                    JanetAsyncStatus status = state->machine(state, JANET_ASYNC_EVENT_COMPLETE);
+                    if (status == JANET_ASYNC_STATUS_DONE) {
+                        janet_unlisten(state);
+                    }
+                    break;
+                } else {
+                    state = state->_next;
                 }
-                break;
-            } else {
-                state = state->_next;
             }
         }
     }
@@ -1256,9 +1305,9 @@ void janet_loop1_impl(int has_timeout, JanetTimestamp timeout) {
         if (mask & POLLIN)
             status2 = state->machine(state, JANET_ASYNC_EVENT_READ);
         if (mask & POLLERR)
-            status2 = state->machine(state, JANET_ASYNC_EVENT_ERR);
+            status3 = state->machine(state, JANET_ASYNC_EVENT_ERR);
         if (mask & POLLHUP)
-            status2 = state->machine(state, JANET_ASYNC_EVENT_HUP);
+            status4 = state->machine(state, JANET_ASYNC_EVENT_HUP);
         if (status1 == JANET_ASYNC_STATUS_DONE ||
                 status2 == JANET_ASYNC_STATUS_DONE ||
                 status3 == JANET_ASYNC_STATUS_DONE ||
@@ -1399,12 +1448,16 @@ void janet_ev_default_threaded_callback(JanetEVGenericMessage return_value) {
         case JANET_EV_TCTAG_ERR_KEYWORD:
             janet_cancel(return_value.fiber, janet_ckeywordv((const char *) return_value.argp));
             break;
+        case JANET_EV_TCTAG_BOOLEAN:
+            janet_schedule(return_value.fiber, janet_wrap_boolean(return_value.argi));
+            break;
     }
     janet_gcunroot(janet_wrap_fiber(return_value.fiber));
 }
 
 
 /* Convenience method for common case */
+JANET_NO_RETURN
 void janet_ev_threaded_await(JanetThreadedSubroutine fp, int tag, int argi, void *argp) {
     JanetEVGenericMessage arguments;
     arguments.tag = tag;
@@ -1507,7 +1560,7 @@ JanetAsyncStatus ev_machine_read(JanetListenerState *s, JanetAsyncEvent event) {
             janet_mark(janet_wrap_buffer(state->buf));
             break;
         case JANET_ASYNC_EVENT_CLOSE:
-            janet_cancel(s->fiber, janet_cstringv("stream closed"));
+            janet_schedule(s->fiber, janet_wrap_nil());
             return JANET_ASYNC_STATUS_DONE;
 #ifdef JANET_WINDOWS
         case JANET_ASYNC_EVENT_COMPLETE: {
@@ -1545,16 +1598,11 @@ JanetAsyncStatus ev_machine_read(JanetListenerState *s, JanetAsyncEvent event) {
             memset(&(state->overlapped), 0, sizeof(OVERLAPPED));
             int status;
 #ifdef JANET_NET
-            if (state->mode != JANET_ASYNC_READMODE_READ) {
+            if (state->mode == JANET_ASYNC_READMODE_RECVFROM) {
                 state->wbuf.len = (ULONG) chunk_size;
                 state->wbuf.buf = state->chunk_buf;
-                if (state->mode == JANET_ASYNC_READMODE_RECVFROM) {
-                    status = WSARecvFrom((SOCKET) s->stream->handle, &state->wbuf, 1,
-                                         NULL, &state->flags, &state->from, &state->fromlen, &state->overlapped, NULL);
-                } else {
-                    status = WSARecv((SOCKET) s->stream->handle, &state->wbuf, 1,
-                                     NULL, &state->flags, &state->overlapped, NULL);
-                }
+                status = WSARecvFrom((SOCKET) s->stream->handle, &state->wbuf, 1,
+                                     NULL, &state->flags, &state->from, &state->fromlen, &state->overlapped, NULL);
                 if (status && (WSA_IO_PENDING != WSAGetLastError())) {
                     janet_cancel(s->fiber, janet_ev_lasterr());
                     return JANET_ASYNC_STATUS_DONE;
@@ -1565,7 +1613,11 @@ JanetAsyncStatus ev_machine_read(JanetListenerState *s, JanetAsyncEvent event) {
                 status = ReadFile(s->stream->handle, state->chunk_buf, chunk_size, NULL, &state->overlapped);
                 if (!status && (ERROR_IO_PENDING != WSAGetLastError())) {
                     if (WSAGetLastError() == ERROR_BROKEN_PIPE) {
-                        janet_schedule(s->fiber, janet_wrap_nil());
+                        if (state->bytes_read) {
+                            janet_schedule(s->fiber, janet_wrap_buffer(state->buf));
+                        } else {
+                            janet_schedule(s->fiber, janet_wrap_nil());
+                        }
                     } else {
                         janet_cancel(s->fiber, janet_ev_lasterr());
                     }
@@ -1575,8 +1627,7 @@ JanetAsyncStatus ev_machine_read(JanetListenerState *s, JanetAsyncEvent event) {
         }
         break;
 #else
-        case JANET_ASYNC_EVENT_ERR:
-        case JANET_ASYNC_EVENT_HUP: {
+        case JANET_ASYNC_EVENT_ERR: {
             if (state->bytes_read) {
                 janet_schedule(s->fiber, janet_wrap_buffer(state->buf));
             } else {
@@ -1584,10 +1635,11 @@ JanetAsyncStatus ev_machine_read(JanetListenerState *s, JanetAsyncEvent event) {
             }
             return JANET_ASYNC_STATUS_DONE;
         }
+        case JANET_ASYNC_EVENT_HUP:
         case JANET_ASYNC_EVENT_READ: {
             JanetBuffer *buffer = state->buf;
             int32_t bytes_left = state->bytes_left;
-            int32_t read_limit = bytes_left < 0 ? 4096 : bytes_left;
+            int32_t read_limit = bytes_left > 4096 ? 4096 : bytes_left;
             janet_buffer_extra(buffer, read_limit);
             ssize_t nread;
 #ifdef JANET_NET
@@ -1772,17 +1824,13 @@ JanetAsyncStatus ev_machine_write(JanetListenerState *s, JanetAsyncEvent event)
 
             int status;
 #ifdef JANET_NET
-            if (state->mode != JANET_ASYNC_WRITEMODE_WRITE) {
+            if (state->mode == JANET_ASYNC_WRITEMODE_SENDTO) {
                 SOCKET sock = (SOCKET) s->stream->handle;
                 state->wbuf.buf = (char *) bytes;
                 state->wbuf.len = len;
-                if (state->mode == JANET_ASYNC_WRITEMODE_SENDTO) {
-                    const struct sockaddr *to = state->dest_abst;
-                    int tolen = (int) janet_abstract_size((void *) to);
-                    status = WSASendTo(sock, &state->wbuf, 1, NULL, state->flags, to, tolen, &state->overlapped, NULL);
-                } else {
-                    status = WSASend(sock, &state->wbuf, 1, NULL, state->flags, &state->overlapped, NULL);
-                }
+                const struct sockaddr *to = state->dest_abst;
+                int tolen = (int) janet_abstract_size((void *) to);
+                status = WSASendTo(sock, &state->wbuf, 1, NULL, state->flags, to, tolen, &state->overlapped, NULL);
                 if (status && (WSA_IO_PENDING != WSAGetLastError())) {
                     janet_cancel(s->fiber, janet_ev_lasterr());
                     return JANET_ASYNC_STATUS_DONE;
@@ -1915,44 +1963,65 @@ void janet_ev_sendto_string(JanetStream *stream, JanetString str, void *dest, in
 static volatile long PipeSerialNumber;
 #endif
 
-int janet_make_pipe(JanetHandle handles[2]) {
+int janet_make_pipe(JanetHandle handles[2], int mode) {
 #ifdef JANET_WINDOWS
     /*
      * On windows, the built in CreatePipe function doesn't support overlapped IO
      * so we lift from the windows source code and modify for our own version.
+     *
+     * mode = 0: both sides non-blocking.
+     * mode = 1: only read side non-blocking: write side sent to subprocess
+     * mode = 2: only write side non-blocking: read side sent to subprocess
      */
-    JanetHandle rhandle, whandle;
+    JanetHandle shandle, chandle;
     UCHAR PipeNameBuffer[MAX_PATH];
+    SECURITY_ATTRIBUTES saAttr;
+    memset(&saAttr, 0, sizeof(saAttr));
+    saAttr.nLength = sizeof(saAttr);
+    saAttr.bInheritHandle = TRUE;
     sprintf(PipeNameBuffer,
             "\\\\.\\Pipe\\JanetPipeFile.%08x.%08x",
             GetCurrentProcessId(),
             InterlockedIncrement(&PipeSerialNumber));
-    rhandle = CreateNamedPipeA(
+
+    /* server handle goes to subprocess */
+    shandle = CreateNamedPipeA(
                   PipeNameBuffer,
-                  PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED,
-                  PIPE_TYPE_BYTE | PIPE_NOWAIT,
-                  1,             /* Number of pipes */
+                  (mode == 2 ? PIPE_ACCESS_INBOUND : PIPE_ACCESS_OUTBOUND) | FILE_FLAG_OVERLAPPED,
+                  PIPE_TYPE_BYTE | PIPE_WAIT,
+                  255,           /* Max number of pipes for duplication. */
                   4096,          /* Out buffer size */
                   4096,          /* In buffer size */
                   120 * 1000,    /* Timeout in ms */
-                  NULL);
-    if (!rhandle) return -1;
-    whandle = CreateFileA(
+                  &saAttr);
+    if (shandle == INVALID_HANDLE_VALUE) {
+        return -1;
+    }
+
+    /* we keep client handle */
+    chandle = CreateFileA(
                   PipeNameBuffer,
-                  GENERIC_WRITE,
+                  (mode == 2 ? GENERIC_WRITE : GENERIC_READ),
                   0,
-                  NULL,
+                  &saAttr,
                   OPEN_EXISTING,
                   FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED,
                   NULL);
-    if (whandle == INVALID_HANDLE_VALUE) {
-        CloseHandle(rhandle);
+
+    if (chandle == INVALID_HANDLE_VALUE) {
+        CloseHandle(shandle);
         return -1;
     }
-    handles[0] = rhandle;
-    handles[1] = whandle;
+    if (mode == 2) {
+        handles[0] = shandle;
+        handles[1] = chandle;
+    } else {
+        handles[0] = chandle;
+        handles[1] = shandle;
+    }
     return 0;
 #else
+    (void) mode;
     if (pipe(handles)) return -1;
     if (fcntl(handles[0], F_SETFL, O_NONBLOCK)) goto error;
     if (fcntl(handles[1], F_SETFL, O_NONBLOCK)) goto error;
@@ -1967,22 +2036,83 @@ error:
 /* C functions */
 
 static Janet cfun_ev_go(int32_t argc, Janet *argv) {
-    janet_arity(argc, 1, 2);
+    janet_arity(argc, 1, 3);
     JanetFiber *fiber = janet_getfiber(argv, 0);
     Janet value = argc == 2 ? argv[1] : janet_wrap_nil();
+    JanetChannel *supervisor_channel = janet_optabstract(argv, argc, 2, &ChannelAT,
+                                       janet_vm_root_fiber->supervisor_channel);
+    fiber->supervisor_channel = supervisor_channel;
     janet_schedule(fiber, value);
     return argv[0];
 }
 
-static Janet cfun_ev_call(int32_t argc, Janet *argv) {
+/* For ev/thread - Run an interpreter in the new thread. */
+static JanetEVGenericMessage janet_go_thread_subr(JanetEVGenericMessage args) {
+    JanetBuffer *buffer = (JanetBuffer *) args.argp;
+    const uint8_t *nextbytes = buffer->data;
+    const uint8_t *endbytes = nextbytes + buffer->count;
+    janet_init();
+    JanetTryState tstate;
+    JanetSignal signal = janet_try(&tstate);
+    if (!signal) {
+        Janet aregv = janet_unmarshal(nextbytes, endbytes - nextbytes,
+                                      JANET_MARSHAL_UNSAFE, NULL, &nextbytes);
+        if (!janet_checktype(aregv, JANET_TABLE)) janet_panic("expected table for abstract registry");
+        janet_vm_abstract_registry = janet_unwrap_table(aregv);
+        Janet regv = janet_unmarshal(nextbytes, endbytes - nextbytes,
+                                     JANET_MARSHAL_UNSAFE, NULL, &nextbytes);
+        if (!janet_checktype(regv, JANET_TABLE)) janet_panic("expected table for cfunction registry");
+        janet_vm_registry = janet_unwrap_table(regv);
+        Janet fiberv = janet_unmarshal(nextbytes, endbytes - nextbytes,
+                                       JANET_MARSHAL_UNSAFE, NULL, &nextbytes);
+        Janet value = janet_unmarshal(nextbytes, endbytes - nextbytes,
+                                      JANET_MARSHAL_UNSAFE, NULL, &nextbytes);
+        if (!janet_checktype(fiberv, JANET_FIBER)) janet_panic("expected fiber");
+        JanetFiber *fiber = janet_unwrap_fiber(fiberv);
+        janet_schedule(fiber, value);
+        janet_loop();
+        args.tag = JANET_EV_TCTAG_NIL;
+    } else {
+        if (janet_checktype(tstate.payload, JANET_STRING)) {
+            args.tag = JANET_EV_TCTAG_ERR_STRINGF;
+            args.argp = strdup((const char *) janet_unwrap_string(tstate.payload));
+        } else {
+            args.tag = JANET_EV_TCTAG_ERR_STRING;
+            args.argp = "failed to start thread";
+        }
+    }
+    janet_buffer_deinit(buffer);
+    janet_restore(&tstate);
+    janet_deinit();
+    return args;
+}
+
+static Janet cfun_ev_thread(int32_t argc, Janet *argv) {
+    janet_arity(argc, 1, 3);
+    janet_getfiber(argv, 0);
+    Janet value = argc == 2 ? argv[1] : janet_wrap_nil();
+    /* Marshal arguments for the new thread. */
+    JanetBuffer *buffer = malloc(sizeof(JanetBuffer));
+    if (NULL == buffer) {
+        JANET_OUT_OF_MEMORY;
+    }
+    janet_buffer_init(buffer, 0);
+    janet_marshal(buffer, janet_wrap_table(janet_vm_abstract_registry), NULL, JANET_MARSHAL_UNSAFE);
+    janet_marshal(buffer, janet_wrap_table(janet_vm_registry), NULL, JANET_MARSHAL_UNSAFE);
+    janet_marshal(buffer, argv[0], NULL, JANET_MARSHAL_UNSAFE);
+    janet_marshal(buffer, value, NULL, JANET_MARSHAL_UNSAFE);
+    janet_ev_threaded_await(janet_go_thread_subr, 0, argc, buffer);
+}
+
+static Janet cfun_ev_give_supervisor(int32_t argc, Janet *argv) {
     janet_arity(argc, 1, -1);
-    JanetFunction *fn = janet_getfunction(argv, 0);
-    JanetFiber *fiber = janet_fiber(fn, 64, argc - 1, argv + 1);
-    if (NULL == fiber) janet_panicf("invalid arity to function %v", argv[0]);
-    fiber->env = janet_table(0);
-    fiber->env->proto = janet_current_fiber()->env;
-    janet_schedule(fiber, janet_wrap_nil());
-    return janet_wrap_fiber(fiber);
+    JanetChannel *chan = janet_vm_root_fiber->supervisor_channel;
+    if (NULL != chan) {
+        if (janet_channel_push(chan, janet_wrap_tuple(janet_tuple_n(argv, argc)), 0)) {
+            janet_await();
+        }
+    }
+    return janet_wrap_nil();
 }
 
 JANET_NO_RETURN void janet_sleep_await(double sec) {
@@ -2040,7 +2170,7 @@ Janet janet_cfun_stream_read(int32_t argc, Janet *argv) {
     double to = janet_optnumber(argv, argc, 3, INFINITY);
     if (janet_keyeq(argv[1], "all")) {
         if (to != INFINITY) janet_addtimeout(to);
-        janet_ev_readchunk(stream, buffer, -1);
+        janet_ev_readchunk(stream, buffer, INT32_MAX);
     } else {
         int32_t n = janet_getnat(argv, 1);
         if (to != INFINITY) janet_addtimeout(to);
@@ -2079,16 +2209,28 @@ Janet janet_cfun_stream_write(int32_t argc, Janet *argv) {
 
 static const JanetReg ev_cfuns[] = {
     {
-        "ev/call", cfun_ev_call,
-        JDOC("(ev/call fn & args)\n\n"
-             "Call a function asynchronously. Returns a fiber that is scheduled to "
-             "run the function.")
+        "ev/go", cfun_ev_go,
+        JDOC("(ev/go fiber &opt value supervisor)\n\n"
+             "Put a fiber on the event loop to be resumed later. Optionally pass "
+             "a value to resume with, otherwise resumes with nil. Returns the fiber. "
+             "An optional `core/channel` can be provided as well as a supervisor. When various "
+             "events occur in the newly scheduled fiber, an event will be pushed to the supervisor. "
+             "If not provided, the new fiber will inherit the current supervisor.")
     },
     {
-        "ev/go", cfun_ev_go,
-        JDOC("(ev/go fiber &opt value)\n\n"
-             "Put a fiber on the event loop to be resumed later. Optionally pass "
-             "a value to resume with, otherwise resumes with nil.")
+        "ev/thread", cfun_ev_thread,
+        JDOC("(ev/thread fiber &opt value flags)\n\n"
+             "Resume a (copy of a) `fiber` in a new operating system thread, optionally passing `value` "
+             "to resume with. "
+             "Unlike `ev/go`, this function will suspend the current fiber until the thread is complete. "
+             "The the final result.")
+    },
+    {
+        "ev/give-supervisor", cfun_ev_give_supervisor,
+        JDOC("(ev/give-supervsior tag & payload)\n\n"
+             "Send a message to the current supervior channel if there is one. The message will be a "
+             "tuple of all of the arguments combined into a single message, where the first element is tag. "
+             "By convention, tag should be a keyword indicating the type of message. Returns nil.")
     },
     {
         "ev/sleep", cfun_ev_sleep,
@@ -2150,7 +2292,7 @@ static const JanetReg ev_cfuns[] = {
     {
         "ev/rselect", cfun_channel_rchoice,
         JDOC("(ev/rselect & clauses)\n\n"
-             "Similar to ev/choice, but will try clauses in a random order for fairness.")
+             "Similar to ev/select, but will try clauses in a random order for fairness.")
     },
     {
         "ev/close", janet_cfun_stream_close,
@@ -2185,6 +2327,7 @@ static const JanetReg ev_cfuns[] = {
 
 void janet_lib_ev(JanetTable *env) {
     janet_core_cfuns(env, NULL, ev_cfuns);
+    janet_register_abstract_type(&janet_stream_type);
 }
 
 #endif

src/core/fiber.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -37,9 +37,11 @@ static void fiber_reset(JanetFiber *fiber) {
     fiber->child = NULL;
     fiber->flags = JANET_FIBER_MASK_YIELD | JANET_FIBER_RESUME_NO_USEVAL | JANET_FIBER_RESUME_NO_SKIP;
     fiber->env = NULL;
+    fiber->last_value = janet_wrap_nil();
 #ifdef JANET_EV
     fiber->waiting = NULL;
     fiber->sched_id = 0;
+    fiber->supervisor_channel = NULL;
 #endif
     janet_fiber_set_status(fiber, JANET_STATUS_NEW);
 }
@@ -83,6 +85,7 @@ JanetFiber *janet_fiber_reset(JanetFiber *fiber, JanetFunction *callee, int32_t
     janet_fiber_frame(fiber)->flags |= JANET_STACKFRAME_ENTRANCE;
 #ifdef JANET_EV
     fiber->waiting = NULL;
+    fiber->supervisor_channel = NULL;
 #endif
     return fiber;
 }
@@ -586,6 +589,12 @@ static Janet cfun_fiber_can_resume(int32_t argc, Janet *argv) {
     return janet_wrap_boolean(!isFinished);
 }
 
+static Janet cfun_fiber_last_value(int32_t argc, Janet *argv) {
+    janet_fixarity(argc, 1);
+    JanetFiber *fiber = janet_getfiber(argv, 0);
+    return fiber->last_value;
+}
+
 static const JanetReg fiber_cfuns[] = {
     {
         "fiber/new", cfun_fiber_new,
@@ -593,34 +602,37 @@ static const JanetReg fiber_cfuns[] = {
              "Create a new fiber with function body func. Can optionally "
              "take a set of signals to block from the current parent fiber "
              "when called. The mask is specified as a keyword where each character "
-             "is used to indicate a signal to block. The default sigmask is :y. "
-             "For example, \n\n"
-             "\t(fiber/new myfun :e123)\n\n"
+             "is used to indicate a signal to block. If the ev module is enabled, and "
+             "this fiber is used as an argument to `ev/go`, these \"blocked\" signals "
+             "will result in messages being sent to the supervisor channel. "
+             "The default sigmask is :y. "
+             "For example,\n\n"
+             "    (fiber/new myfun :e123)\n\n"
              "blocks error signals and user signals 1, 2 and 3. The signals are "
-             "as follows: \n\n"
-             "\ta - block all signals\n"
-             "\td - block debug signals\n"
-             "\te - block error signals\n"
-             "\tt - block termination signals: error + user[0-4]\n"
-             "\tu - block user signals\n"
-             "\ty - block yield signals\n"
-             "\t0-9 - block a specific user signal\n\n"
+             "as follows:\n\n"
+             "* :a - block all signals\n"
+             "* :d - block debug signals\n"
+             "* :e - block error signals\n"
+             "* :t - block termination signals: error + user[0-4]\n"
+             "* :u - block user signals\n"
+             "* :y - block yield signals\n"
+             "* :0-9 - block a specific user signal\n\n"
              "The sigmask argument also can take environment flags. If any mutually "
              "exclusive flags are present, the last flag takes precedence.\n\n"
-             "\ti - inherit the environment from the current fiber\n"
-             "\tp - the environment table's prototype is the current environment table")
+             "* :i - inherit the environment from the current fiber\n"
+             "* :p - the environment table's prototype is the current environment table")
     },
     {
         "fiber/status", cfun_fiber_status,
         JDOC("(fiber/status fib)\n\n"
              "Get the status of a fiber. The status will be one of:\n\n"
-             "\t:dead - the fiber has finished\n"
-             "\t:error - the fiber has errored out\n"
-             "\t:debug - the fiber is suspended in debug mode\n"
-             "\t:pending - the fiber has been yielded\n"
-             "\t:user(0-9) - the fiber is suspended by a user signal\n"
-             "\t:alive - the fiber is currently running and cannot be resumed\n"
-             "\t:new - the fiber has just been created and not yet run")
+             "* :dead - the fiber has finished\n"
+             "* :error - the fiber has errored out\n"
+             "* :debug - the fiber is suspended in debug mode\n"
+             "* :pending - the fiber has been yielded\n"
+             "* :user(0-9) - the fiber is suspended by a user signal\n"
+             "* :alive - the fiber is currently running and cannot be resumed\n"
+             "* :new - the fiber has just been created and not yet run")
     },
     {
         "fiber/root", cfun_fiber_root,
@@ -663,6 +675,11 @@ static const JanetReg fiber_cfuns[] = {
         JDOC("(fiber/can-resume? fiber)\n\n"
              "Check if a fiber is finished and cannot be resumed.")
     },
+    {
+        "fiber/last-value", cfun_fiber_last_value,
+        JDOC("(fiber/last-value\n\n"
+             "Get the last value returned or signaled from the fiber.")
+    },
     {NULL, NULL, NULL}
 };
 

src/core/gc.c

@@ -244,6 +244,8 @@ recur:
         return;
     janet_gc_mark(fiber);
 
+    janet_mark(fiber->last_value);
+
     /* Mark values on the argument stack */
     janet_mark_many(fiber->data + fiber->stackstart,
                     fiber->stacktop - fiber->stackstart);
@@ -265,6 +267,12 @@ recur:
     if (fiber->env)
         janet_mark_table(fiber->env);
 
+#ifdef JANET_EV
+    if (fiber->supervisor_channel) {
+        janet_mark_abstract(fiber->supervisor_channel);
+    }
+#endif
+
     /* Explicit tail recursion */
     if (fiber->child) {
         fiber = fiber->child;

src/core/inttypes.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose & contributors
+* Copyright (c) 2021 Calvin Rose & contributors
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -39,6 +39,8 @@
 
 static int it_s64_get(void *p, Janet key, Janet *out);
 static int it_u64_get(void *p, Janet key, Janet *out);
+static Janet janet_int64_next(void *p, Janet key);
+static Janet janet_uint64_next(void *p, Janet key);
 
 static int32_t janet_int64_hash(void *p1, size_t size) {
     (void) size;
@@ -92,7 +94,8 @@ const JanetAbstractType janet_s64_type = {
     it_s64_tostring,
     janet_int64_compare,
     janet_int64_hash,
-    JANET_ATEND_HASH
+    janet_int64_next,
+    JANET_ATEND_NEXT
 };
 
 const JanetAbstractType janet_u64_type = {
@@ -106,7 +109,8 @@ const JanetAbstractType janet_u64_type = {
     it_u64_tostring,
     janet_uint64_compare,
     janet_int64_hash,
-    JANET_ATEND_HASH
+    janet_uint64_next,
+    JANET_ATEND_NEXT
 };
 
 int64_t janet_unwrap_s64(Janet x) {
@@ -134,7 +138,7 @@ int64_t janet_unwrap_s64(Janet x) {
             break;
         }
     }
-    janet_panic("bad s64 initializer");
+    janet_panicf("bad s64 initializer: %t", x);
     return 0;
 }
 
@@ -144,7 +148,9 @@ uint64_t janet_unwrap_u64(Janet x) {
             break;
         case JANET_NUMBER : {
             double dbl = janet_unwrap_number(x);
-            if ((dbl >= 0) && (dbl <= MAX_INT_IN_DBL))
+            /* Allow negative values to be cast to "wrap around".
+             * This let's addition and subtraction work as expected. */
+            if (fabs(dbl) <=  MAX_INT_IN_DBL)
                 return (uint64_t)dbl;
             break;
         }
@@ -163,7 +169,7 @@ uint64_t janet_unwrap_u64(Janet x) {
             break;
         }
     }
-    janet_panic("bad u64 initializer");
+    janet_panicf("bad u64 initializer: %t", x);
     return 0;
 }
 
@@ -197,15 +203,14 @@ static Janet cfun_it_u64_new(int32_t argc, Janet *argv) {
     return janet_wrap_u64(janet_unwrap_u64(argv[0]));
 }
 
-// Code to support polymorphic comparison.
-//
-// int/u64 and int/s64 support a "compare" method that allows
-// comparison to each other, and to Janet numbers, using the
-// "compare" "compare<" ... functions.
-//
-// In the following code explicit casts are sometimes used to help
-// make it clear when int/float conversions are happening.
-//
+/*
+ * Code to support polymorphic comparison.
+ * int/u64 and int/s64 support a "compare" method that allows
+ * comparison to each other, and to Janet numbers, using the
+ * "compare" "compare<" ... functions.
+ * In the following code explicit casts are sometimes used to help
+ * make it clear when int/float conversions are happening.
+ */
 static int compare_double_double(double x, double y) {
     return (x < y) ? -1 : ((x > y) ? 1 : 0);
 }
@@ -242,7 +247,6 @@ static int compare_uint64_double(uint64_t x, double y) {
     }
 }
 
-
 static Janet cfun_it_s64_compare(int32_t argc, Janet *argv) {
     janet_fixarity(argc, 2);
     if (janet_is_int(argv[0]) != JANET_INT_S64)
@@ -383,31 +387,14 @@ static Janet cfun_it_##type##_##name(int32_t argc, Janet *argv) { \
 } \
 
 static Janet cfun_it_s64_mod(int32_t argc, Janet *argv) {
-    janet_arity(argc, 2, -1);
-    int64_t *box = janet_abstract(&janet_s64_type, sizeof(int64_t));
-    *box = janet_unwrap_s64(argv[0]);
-    for (int32_t i = 1; i < argc; i++) {
-        int64_t value = janet_unwrap_s64(argv[i]);
-        if (value == 0) janet_panic("division by zero");
-        int64_t x = *box % value;
-        if (x < 0) {
-            x = (*box < 0) ? x - *box : x + *box;
-        }
-        *box = x;
-    }
-    return janet_wrap_abstract(box);
-}
-
-static Janet cfun_it_s64_modi(int32_t argc, Janet *argv) {
     janet_fixarity(argc, 2);
     int64_t *box = janet_abstract(&janet_s64_type, sizeof(int64_t));
     int64_t op1 = janet_unwrap_s64(argv[0]);
     int64_t op2 = janet_unwrap_s64(argv[1]);
     int64_t x = op1 % op2;
-    if (x < 0) {
-        x = (op1 < 0) ? x - op1 : x + op1;
-    }
-    *box = x;
+    *box = (op1 > 0)
+           ? ((op2 > 0) ? x : (0 == x ? x : x + op2))
+           : ((op2 > 0) ? (0 == x ? x : x + op2) : x);
     return janet_wrap_abstract(box);
 }
 
@@ -418,7 +405,6 @@ OPMETHOD(int64_t, s64, mul, *)
 DIVMETHOD_SIGNED(int64_t, s64, div, /)
 DIVMETHOD_SIGNED(int64_t, s64, rem, %)
 DIVMETHODINVERT_SIGNED(int64_t, s64, divi, /)
-DIVMETHODINVERT_SIGNED(int64_t, s64, remi, %)
 OPMETHOD(int64_t, s64, and, &)
 OPMETHOD(int64_t, s64, or, |)
 OPMETHOD(int64_t, s64, xor, ^)
@@ -431,7 +417,6 @@ OPMETHOD(uint64_t, u64, mul, *)
 DIVMETHOD(uint64_t, u64, div, /)
 DIVMETHOD(uint64_t, u64, mod, %)
 DIVMETHODINVERT(uint64_t, u64, divi, /)
-DIVMETHODINVERT(uint64_t, u64, modi, %)
 OPMETHOD(uint64_t, u64, and, &)
 OPMETHOD(uint64_t, u64, or, |)
 OPMETHOD(uint64_t, u64, xor, ^)
@@ -454,9 +439,9 @@ static JanetMethod it_s64_methods[] = {
     {"/", cfun_it_s64_div},
     {"r/", cfun_it_s64_divi},
     {"mod", cfun_it_s64_mod},
-    {"rmod", cfun_it_s64_modi},
+    {"rmod", cfun_it_s64_mod},
     {"%", cfun_it_s64_rem},
-    {"r%", cfun_it_s64_remi},
+    {"r%", cfun_it_s64_rem},
     {"&", cfun_it_s64_and},
     {"r&", cfun_it_s64_and},
     {"|", cfun_it_s64_or},
@@ -480,9 +465,9 @@ static JanetMethod it_u64_methods[] = {
     {"/", cfun_it_u64_div},
     {"r/", cfun_it_u64_divi},
     {"mod", cfun_it_u64_mod},
-    {"rmod", cfun_it_u64_modi},
+    {"rmod", cfun_it_u64_mod},
     {"%", cfun_it_u64_mod},
-    {"r%", cfun_it_u64_modi},
+    {"r%", cfun_it_u64_mod},
     {"&", cfun_it_u64_and},
     {"r&", cfun_it_u64_and},
     {"|", cfun_it_u64_or},
@@ -496,6 +481,16 @@ static JanetMethod it_u64_methods[] = {
     {NULL, NULL}
 };
 
+static Janet janet_int64_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(it_s64_methods, key);
+}
+
+static Janet janet_uint64_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(it_u64_methods, key);
+}
+
 static int it_s64_get(void *p, Janet key, Janet *out) {
     (void) p;
     if (!janet_checktype(key, JANET_KEYWORD))

src/core/io.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -39,6 +39,7 @@ static int cfun_io_gc(void *p, size_t len);
 static int io_file_get(void *p, Janet key, Janet *out);
 static void io_file_marshal(void *p, JanetMarshalContext *ctx);
 static void *io_file_unmarshal(JanetMarshalContext *ctx);
+static Janet io_file_next(void *p, Janet key);
 
 const JanetAbstractType janet_file_type = {
     "core/file",
@@ -48,7 +49,11 @@ const JanetAbstractType janet_file_type = {
     NULL,
     io_file_marshal,
     io_file_unmarshal,
-    JANET_ATEND_UNMARSHAL
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    io_file_next,
+    JANET_ATEND_NEXT
 };
 
 /* Check arguments to fopen */
@@ -259,20 +264,29 @@ static Janet cfun_io_fflush(int32_t argc, Janet *argv) {
 #define WEXITSTATUS(x) x
 #endif
 
+/* For closing files from C API */
+int janet_file_close(JanetFile *file) {
+    int ret = 0;
+    if (!(file->flags & (JANET_FILE_NOT_CLOSEABLE | JANET_FILE_CLOSED))) {
+#ifndef JANET_NO_PROCESSES
+        if (file->flags & JANET_FILE_PIPED) {
+            ret = pclose(file->file);
+        } else
+#endif
+        {
+            ret = fclose(file->file);
+        }
+        file->flags |= JANET_FILE_CLOSED;
+        return ret;
+    }
+    return 0;
+}
+
 /* Cleanup a file */
 static int cfun_io_gc(void *p, size_t len) {
     (void) len;
     JanetFile *iof = (JanetFile *)p;
-    if (!(iof->flags & (JANET_FILE_NOT_CLOSEABLE | JANET_FILE_CLOSED))) {
-        /* We can't panic inside a gc, so just ignore bad statuses here */
-        if (iof->flags & JANET_FILE_PIPED) {
-#ifndef JANET_NO_PROCESSES
-            pclose(iof->file);
-#endif
-        } else {
-            fclose(iof->file);
-        }
-    }
+    janet_file_close(iof);
     return 0;
 }
 
@@ -346,6 +360,11 @@ static int io_file_get(void *p, Janet key, Janet *out) {
     return janet_getmethod(janet_unwrap_keyword(key), io_file_methods, out);
 }
 
+static Janet io_file_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(io_file_methods, key);
+}
+
 static void io_file_marshal(void *p, JanetMarshalContext *ctx) {
     JanetFile *iof = (JanetFile *)p;
     if (ctx->flags & JANET_MARSHAL_UNSAFE) {
@@ -713,23 +732,24 @@ static const JanetReg io_cfuns[] = {
     {
         "file/temp", cfun_io_temp,
         JDOC("(file/temp)\n\n"
-             "Open an anonymous temporary file that is removed on close."
+             "Open an anonymous temporary file that is removed on close. "
              "Raises an error on failure.")
     },
     {
         "file/open", cfun_io_fopen,
         JDOC("(file/open path &opt mode)\n\n"
-             "Open a file. path is an absolute or relative path, and "
-             "mode is a set of flags indicating the mode to open the file in. "
-             "mode is a keyword where each character represents a flag. If the file "
+             "Open a file. `path` is an absolute or relative path, and "
+             "`mode` is a set of flags indicating the mode to open the file in. "
+             "`mode` is a keyword where each character represents a flag. If the file "
              "cannot be opened, returns nil, otherwise returns the new file handle. "
              "Mode flags:\n\n"
-             "\tr - allow reading from the file\n"
-             "\tw - allow writing to the file\n"
-             "\ta - append to the file\n"
-             "\tb - open the file in binary mode (rather than text mode)\n"
-             "\t+ - append to the file instead of overwriting it\n"
-             "\tn - error if the file cannot be opened instead of returning nil")
+             "* r - allow reading from the file\n\n"
+             "* w - allow writing to the file\n\n"
+             "* a - append to the file\n\n"
+             "Following one of the initial flags, 0 or more of the following flags can be appended:\n\n"
+             "* b - open the file in binary mode (rather than text mode)\n\n"
+             "* + - append to the file instead of overwriting it\n\n"
+             "* n - error if the file cannot be opened instead of returning nil")
     },
     {
         "file/close", cfun_io_fclose,
@@ -742,14 +762,14 @@ static const JanetReg io_cfuns[] = {
     {
         "file/read", cfun_io_fread,
         JDOC("(file/read f what &opt buf)\n\n"
-             "Read a number of bytes from a file into a buffer. A buffer can "
-             "be provided as an optional fourth argument, otherwise a new buffer "
-             "is created. 'what' can either be an integer or a keyword. Returns the "
+             "Read a number of bytes from a file `f` into a buffer. A buffer `buf` can "
+             "be provided as an optional third argument, otherwise a new buffer "
+             "is created. `what` can either be an integer or a keyword. Returns the "
              "buffer with file contents. "
-             "Values for 'what':\n\n"
-             "\t:all - read the whole file\n"
-             "\t:line - read up to and including the next newline character\n"
-             "\tn (integer) - read up to n bytes from the file")
+             "Values for `what`:\n\n"
+             "* :all - read the whole file\n\n"
+             "* :line - read up to and including the next newline character\n\n"
+             "* n (integer) - read up to n bytes from the file")
     },
     {
         "file/write", cfun_io_fwrite,
@@ -766,13 +786,13 @@ static const JanetReg io_cfuns[] = {
     {
         "file/seek", cfun_io_fseek,
         JDOC("(file/seek f &opt whence n)\n\n"
-             "Jump to a relative location in the file. 'whence' must be one of\n\n"
-             "\t:cur - jump relative to the current file location\n"
-             "\t:set - jump relative to the beginning of the file\n"
-             "\t:end - jump relative to the end of the file\n\n"
-             "By default, 'whence' is :cur. Optionally a value n may be passed "
-             "for the relative number of bytes to seek in the file. n may be a real "
-             "number to handle large files of more the 4GB. Returns the file handle.")
+             "Jump to a relative location in the file `f`. `whence` must be one of:\n\n"
+             "* :cur - jump relative to the current file location\n\n"
+             "* :set - jump relative to the beginning of the file\n\n"
+             "* :end - jump relative to the end of the file\n\n"
+             "By default, `whence` is :cur. Optionally a value `n` may be passed "
+             "for the relative number of bytes to seek in the file. `n` may be a real "
+             "number to handle large files of more than 4GB. Returns the file handle.")
     },
 #ifndef JANET_NO_PROCESSES
     {

src/core/marsh.c

@@ -938,6 +938,7 @@ static const uint8_t *unmarshal_one_fiber(
 #ifdef JANET_EV
     fiber->waiting = NULL;
     fiber->sched_id = 0;
+    fiber->supervisor_channel = NULL;
 #endif
 
     /* Push fiber to seen stack */
@@ -1427,9 +1428,9 @@ static const JanetReg marsh_cfuns[] = {
         "marshal", cfun_marshal,
         JDOC("(marshal x &opt reverse-lookup buffer)\n\n"
              "Marshal a value into a buffer and return the buffer. The buffer "
-             "can the later be unmarshalled to reconstruct the initial value. "
+             "can then later be unmarshalled to reconstruct the initial value. "
              "Optionally, one can pass in a reverse lookup table to not marshal "
-             "aliased values that are found in the table. Then a forward"
+             "aliased values that are found in the table. Then a forward "
              "lookup table can be used to recover the original value when "
              "unmarshalling.")
     },

src/core/math.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -31,6 +31,7 @@
 static JANET_THREAD_LOCAL JanetRNG janet_vm_rng = {0, 0, 0, 0, 0};
 
 static int janet_rng_get(void *p, Janet key, Janet *out);
+static Janet janet_rng_next(void *p, Janet key);
 
 static void janet_rng_marshal(void *p, JanetMarshalContext *ctx) {
     JanetRNG *rng = (JanetRNG *)p;
@@ -60,7 +61,11 @@ const JanetAbstractType janet_rng_type = {
     NULL,
     janet_rng_marshal,
     janet_rng_unmarshal,
-    JANET_ATEND_UNMARSHAL
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    janet_rng_next,
+    JANET_ATEND_NEXT
 };
 
 JanetRNG *janet_default_rng(void) {
@@ -203,6 +208,11 @@ static int janet_rng_get(void *p, Janet key, Janet *out) {
     return janet_getmethod(janet_unwrap_keyword(key), rng_methods, out);
 }
 
+static Janet janet_rng_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(rng_methods, key);
+}
+
 /* Get a random number */
 static Janet janet_rand(int32_t argc, Janet *argv) {
     (void) argv;
@@ -500,13 +510,13 @@ void janet_lib_math(JanetTable *env) {
     janet_def(env, "math/-inf", janet_wrap_number(-INFINITY),
               JDOC("The number representing negative infinity"));
     janet_def(env, "math/int32-min", janet_wrap_number(INT32_MIN),
-              JDOC("The maximum contiguous integer representable by a 32 bit signed integer"));
+              JDOC("The minimum contiguous integer representable by a 32 bit signed integer"));
     janet_def(env, "math/int32-max", janet_wrap_number(INT32_MAX),
-              JDOC("The minimum contiguous integer represtenable by a 32 bit signed integer"));
+              JDOC("The maximum contiguous integer represtenable by a 32 bit signed integer"));
     janet_def(env, "math/int-min", janet_wrap_number(JANET_INTMIN_DOUBLE),
-              JDOC("The maximum contiguous integer representable by a double (2^53)"));
+              JDOC("The minimum contiguous integer representable by a double (2^53)"));
     janet_def(env, "math/int-max", janet_wrap_number(JANET_INTMAX_DOUBLE),
-              JDOC("The minimum contiguous integer represtenable by a double (-(2^53))"));
+              JDOC("The maximum contiguous integer represtenable by a double (-(2^53))"));
 #ifdef NAN
     janet_def(env, "math/nan", janet_wrap_number(NAN),
 #else

src/core/net.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose and contributors.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -80,15 +80,22 @@ static JanetStream *make_stream(JSock handle, uint32_t flags);
 #define MSG_NOSIGNAL 0
 #endif
 
-static void nosigpipe(JSock s) {
+/* Make sure a socket doesn't block */
+static void janet_net_socknoblock(JSock s) {
+#ifdef JANET_WINDOWS
+    unsigned long arg = 1;
+    ioctlsocket(s, FIONBIO, &arg);
+#else
+#if !defined(SOCK_CLOEXEC) && defined(O_CLOEXEC)
+    int extra = O_CLOEXEC;
+#else
+    int extra = 0;
+#endif
+    fcntl(s, F_SETFL, fcntl(s, F_GETFL, 0) | O_NONBLOCK | extra);
 #ifdef SO_NOSIGPIPE
     int enable = 1;
-    if (setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, &enable, sizeof(int)) < 0) {
-        JSOCKCLOSE(s);
-        janet_panic("setsockopt(SO_NOSIGPIPE) failed");
-    }
-#else
-    (void) s;
+    setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, &enable, sizeof(int));
+#endif
 #endif
 }
 
@@ -139,6 +146,7 @@ JanetAsyncStatus net_machine_accept(JanetListenerState *s, JanetAsyncEvent event
             if (state->function) {
                 /* Schedule worker */
                 JanetFiber *fiber = janet_fiber(state->function, 64, 1, &streamv);
+                fiber->supervisor_channel = s->fiber->supervisor_channel;
                 janet_schedule(fiber, janet_wrap_nil());
                 /* Now listen again for next connection */
                 Janet err;
@@ -210,11 +218,12 @@ JanetAsyncStatus net_machine_accept(JanetListenerState *s, JanetAsyncEvent event
         case JANET_ASYNC_EVENT_READ: {
             JSock connfd = accept(s->stream->handle, NULL, NULL);
             if (JSOCKVALID(connfd)) {
-                nosigpipe(connfd);
+                janet_net_socknoblock(connfd);
                 JanetStream *stream = make_stream(connfd, JANET_STREAM_READABLE | JANET_STREAM_WRITABLE);
                 Janet streamv = janet_wrap_abstract(stream);
                 if (state->function) {
                     JanetFiber *fiber = janet_fiber(state->function, 64, 1, &streamv);
+                    fiber->supervisor_channel = s->fiber->supervisor_channel;
                     janet_schedule(fiber, janet_wrap_nil());
                 } else {
                     janet_schedule(s->fiber, streamv);
@@ -256,12 +265,11 @@ static struct addrinfo *janet_get_addrinfo(Janet *argv, int32_t offset, int sock
 #ifndef JANET_WINDOWS
     if (janet_keyeq(argv[offset], "unix")) {
         const char *path = janet_getcstring(argv, offset + 1);
-        struct sockaddr_un *saddr = malloc(sizeof(struct sockaddr_un));
+        struct sockaddr_un *saddr = calloc(1, sizeof(struct sockaddr_un));
         if (saddr == NULL) {
             JANET_OUT_OF_MEMORY;
         }
         saddr->sun_family = AF_UNIX;
-        memset(&saddr->sun_path, 0, 108);
 #ifdef JANET_LINUX
         if (path[0] == '@') {
             saddr->sun_path[0] = '\0';
@@ -351,12 +359,12 @@ static Janet cfun_net_connect(int32_t argc, Janet *argv) {
     /* Create socket */
     JSock sock = JSOCKDEFAULT;
     void *addr = NULL;
-    socklen_t addrlen;
+    socklen_t addrlen = 0;
 #ifndef JANET_WINDOWS
     if (is_unix) {
         sock = socket(AF_UNIX, socktype | JSOCKFLAGS, 0);
         if (!JSOCKVALID(sock)) {
-            janet_panic("could not create socket");
+            janet_panicf("could not create socket: %V", janet_ev_lasterr());
         }
         addr = (void *) ai;
         addrlen = sizeof(struct sockaddr_un);
@@ -378,24 +386,30 @@ static Janet cfun_net_connect(int32_t argc, Janet *argv) {
         }
         if (NULL == addr) {
             freeaddrinfo(ai);
-            janet_panic("could not create socket");
+            janet_panicf("could not create socket: %V", janet_ev_lasterr());
         }
     }
 
     /* Connect to socket */
+#ifdef JANET_WINDOWS
+    int status = WSAConnect(sock, addr, addrlen, NULL, NULL, NULL, NULL);
+    freeaddrinfo(ai);
+#else
     int status = connect(sock, addr, addrlen);
     if (is_unix) {
         free(ai);
     } else {
         freeaddrinfo(ai);
     }
+#endif
 
     if (status == -1) {
         JSOCKCLOSE(sock);
-        janet_panic("could not connect to socket");
+        janet_panicf("could not connect to socket: %V", janet_ev_lasterr());
     }
 
-    nosigpipe(sock);
+    /* Set up the socket for non-blocking IO after connect - TODO - non-blocking connect? */
+    janet_net_socknoblock(sock);
 
     /* Wrap socket in abstract type JanetStream */
     JanetStream *stream = make_stream(sock, JANET_STREAM_READABLE | JANET_STREAM_WRITABLE);
@@ -413,6 +427,7 @@ static const char *serverify_socket(JSock sfd) {
         return "setsockopt(SO_REUSEPORT) failed";
     }
 #endif
+    janet_net_socknoblock(sfd);
     return NULL;
 }
 
@@ -430,13 +445,17 @@ static Janet cfun_net_listen(int32_t argc, Janet *argv) {
         sfd = socket(AF_UNIX, socktype | JSOCKFLAGS, 0);
         if (!JSOCKVALID(sfd)) {
             free(ai);
-            janet_panic("could not create socket");
+            janet_panicf("could not create socket: %V", janet_ev_lasterr());
         }
         const char *err = serverify_socket(sfd);
         if (NULL != err || bind(sfd, (struct sockaddr *)ai, sizeof(struct sockaddr_un))) {
             JSOCKCLOSE(sfd);
             free(ai);
-            janet_panic(err ? err : "could not bind socket");
+            if (err) {
+                janet_panic(err);
+            } else {
+                janet_panicf("could not bind socket: %V", janet_ev_lasterr());
+            }
         }
         free(ai);
     } else
@@ -466,8 +485,6 @@ static Janet cfun_net_listen(int32_t argc, Janet *argv) {
         }
     }
 
-    nosigpipe(sfd);
-
     if (socktype == SOCK_DGRAM) {
         /* Datagram server (UDP) */
         JanetStream *stream = make_stream(sfd, JANET_STREAM_UDPSERVER | JANET_STREAM_READABLE);
@@ -479,7 +496,7 @@ static Janet cfun_net_listen(int32_t argc, Janet *argv) {
         int status = listen(sfd, 1024);
         if (status) {
             JSOCKCLOSE(sfd);
-            janet_panic("could not listen on file descriptor");
+            janet_panicf("could not listen on file descriptor: %V", janet_ev_lasterr());
         }
 
         /* Put sfd on our loop */
@@ -513,7 +530,7 @@ static Janet cfun_stream_read(int32_t argc, Janet *argv) {
     double to = janet_optnumber(argv, argc, 3, INFINITY);
     if (janet_keyeq(argv[1], "all")) {
         if (to != INFINITY) janet_addtimeout(to);
-        janet_ev_recvchunk(stream, buffer, -1, MSG_NOSIGNAL);
+        janet_ev_recvchunk(stream, buffer, INT32_MAX, MSG_NOSIGNAL);
     } else {
         int32_t n = janet_getnat(argv, 1);
         if (to != INFINITY) janet_addtimeout(to);
@@ -580,7 +597,7 @@ static Janet cfun_stream_send_to(int32_t argc, Janet *argv) {
 }
 
 static Janet cfun_stream_flush(int32_t argc, Janet *argv) {
-    janet_fixarity(argc, 2);
+    janet_fixarity(argc, 1);
     JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type);
     janet_stream_flags(stream, JANET_STREAM_WRITABLE | JANET_STREAM_SOCKET);
     /* Toggle no delay flag */
@@ -641,7 +658,7 @@ static const JanetReg net_cfuns[] = {
     {
         "net/accept-loop", cfun_stream_accept_loop,
         JDOC("(net/accept-loop stream handler)\n\n"
-             "Shorthand for running a server stream that will continuously accept new connections."
+             "Shorthand for running a server stream that will continuously accept new connections. "
              "Blocks the current fiber until the stream is closed, and will return the stream.")
     },
     {

src/core/os.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2021 Calvin Rose
+* Copyright (c) 2021 Calvin Rose and contributors.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -187,49 +187,71 @@ static Janet os_exit(int32_t argc, Janet *argv) {
 #ifndef JANET_NO_PROCESSES
 
 /* Get env for os_execute */
-static char **os_execute_env(int32_t argc, const Janet *argv) {
-    char **envp = NULL;
-    if (argc > 2) {
-        JanetDictView dict = janet_getdictionary(argv, 2);
-        envp = janet_smalloc(sizeof(char *) * ((size_t)dict.len + 1));
-        int32_t j = 0;
-        for (int32_t i = 0; i < dict.cap; i++) {
-            const JanetKV *kv = dict.kvs + i;
-            if (!janet_checktype(kv->key, JANET_STRING)) continue;
-            if (!janet_checktype(kv->value, JANET_STRING)) continue;
-            const uint8_t *keys = janet_unwrap_string(kv->key);
-            const uint8_t *vals = janet_unwrap_string(kv->value);
-            int32_t klen = janet_string_length(keys);
-            int32_t vlen = janet_string_length(vals);
-            /* Check keys has no embedded 0s or =s. */
-            int skip = 0;
-            for (int32_t k = 0; k < klen; k++) {
-                if (keys[k] == '\0' || keys[k] == '=') {
-                    skip = 1;
-                    break;
-                }
-            }
-            if (skip) continue;
-            char *envitem = janet_smalloc((size_t) klen + (size_t) vlen + 2);
-            memcpy(envitem, keys, klen);
-            envitem[klen] = '=';
-            memcpy(envitem + klen + 1, vals, vlen);
-            envitem[klen + vlen + 1] = 0;
-            envp[j++] = envitem;
-        }
-        envp[j] = NULL;
+#ifdef JANET_WINDOWS
+typedef char *EnvBlock;
+#else
+typedef char **EnvBlock;
+#endif
+
+/* Get env for os_execute */
+static EnvBlock os_execute_env(int32_t argc, const Janet *argv) {
+    if (argc <= 2) return NULL;
+    JanetDictView dict = janet_getdictionary(argv, 2);
+#ifdef JANET_WINDOWS
+    JanetBuffer *temp = janet_buffer(10);
+    for (int32_t i = 0; i < dict.cap; i++) {
+        const JanetKV *kv = dict.kvs + i;
+        if (!janet_checktype(kv->key, JANET_STRING)) continue;
+        if (!janet_checktype(kv->value, JANET_STRING)) continue;
+        const uint8_t *keys = janet_unwrap_string(kv->key);
+        const uint8_t *vals = janet_unwrap_string(kv->value);
+        janet_buffer_push_bytes(temp, keys, janet_string_length(keys));
+        janet_buffer_push_u8(temp, '=');
+        janet_buffer_push_bytes(temp, vals, janet_string_length(vals));
+        janet_buffer_push_u8(temp, '\0');
     }
+    janet_buffer_push_u8(temp, '\0');
+    char *ret = janet_smalloc(temp->count);
+    memcpy(ret, temp->data, temp->count);
+    return ret;
+#else
+    char **envp = janet_smalloc(sizeof(char *) * ((size_t)dict.len + 1));
+    int32_t j = 0;
+    for (int32_t i = 0; i < dict.cap; i++) {
+        const JanetKV *kv = dict.kvs + i;
+        if (!janet_checktype(kv->key, JANET_STRING)) continue;
+        if (!janet_checktype(kv->value, JANET_STRING)) continue;
+        const uint8_t *keys = janet_unwrap_string(kv->key);
+        const uint8_t *vals = janet_unwrap_string(kv->value);
+        int32_t klen = janet_string_length(keys);
+        int32_t vlen = janet_string_length(vals);
+        /* Check keys has no embedded 0s or =s. */
+        int skip = 0;
+        for (int32_t k = 0; k < klen; k++) {
+            if (keys[k] == '\0' || keys[k] == '=') {
+                skip = 1;
+                break;
+            }
+        }
+        if (skip) continue;
+        char *envitem = janet_smalloc((size_t) klen + (size_t) vlen + 2);
+        memcpy(envitem, keys, klen);
+        envitem[klen] = '=';
+        memcpy(envitem + klen + 1, vals, vlen);
+        envitem[klen + vlen + 1] = 0;
+        envp[j++] = envitem;
+    }
+    envp[j] = NULL;
     return envp;
+#endif
 }
 
-/* Free memory from os_execute. Not actually needed, but doesn't pressure the GC
-   in the happy path. */
-static void os_execute_cleanup(char **envp, const char **child_argv) {
+static void os_execute_cleanup(EnvBlock envp, const char **child_argv) {
 #ifdef JANET_WINDOWS
     (void) child_argv;
+    if (NULL != envp) janet_sfree(envp);
 #else
     janet_sfree((void *)child_argv);
-#endif
     if (NULL != envp) {
         char **envitem = envp;
         while (*envitem != NULL) {
@@ -238,6 +260,7 @@ static void os_execute_cleanup(char **envp, const char **child_argv) {
         }
     }
     janet_sfree(envp);
+#endif
 }
 
 #ifdef JANET_WINDOWS
@@ -322,6 +345,9 @@ static const JanetAbstractType ProcAT;
 #define JANET_PROC_WAITED 2
 #define JANET_PROC_WAITING 4
 #define JANET_PROC_ERROR_NONZERO 8
+#define JANET_PROC_OWNS_STDIN 16
+#define JANET_PROC_OWNS_STDOUT 32
+#define JANET_PROC_OWNS_STDERR 64
 typedef struct {
     int flags;
 #ifdef JANET_WINDOWS
@@ -427,9 +453,11 @@ static int janet_proc_mark(void *p, size_t s) {
 }
 
 #ifdef JANET_EV
-JANET_NO_RETURN
+static JANET_NO_RETURN void
+#else
+static Janet
 #endif
-static Janet os_proc_wait_impl(JanetProc *proc) {
+os_proc_wait_impl(JanetProc *proc) {
     if (proc->flags & (JANET_PROC_WAITED | JANET_PROC_WAITING)) {
         janet_panicf("cannot wait twice on a process");
     }
@@ -467,7 +495,12 @@ static Janet os_proc_wait_impl(JanetProc *proc) {
 static Janet os_proc_wait(int32_t argc, Janet *argv) {
     janet_fixarity(argc, 1);
     JanetProc *proc = janet_getabstract(argv, 0, &ProcAT);
+#ifdef JANET_EV
+    os_proc_wait_impl(proc);
+    return janet_wrap_nil();
+#else
     return os_proc_wait_impl(proc);
+#endif
 }
 
 static Janet os_proc_kill(int32_t argc, Janet *argv) {
@@ -491,12 +524,41 @@ static Janet os_proc_kill(int32_t argc, Janet *argv) {
 #endif
     /* After killing process we wait on it. */
     if (argc > 1 && janet_truthy(argv[1])) {
+#ifdef JANET_EV
+        os_proc_wait_impl(proc);
+        return janet_wrap_nil();
+#else
         return os_proc_wait_impl(proc);
+#endif
     } else {
         return argv[0];
     }
 }
 
+static Janet os_proc_close(int32_t argc, Janet *argv) {
+    janet_fixarity(argc, 1);
+    JanetProc *proc = janet_getabstract(argv, 0, &ProcAT);
+#ifdef JANET_EV
+    if (proc->flags & JANET_PROC_OWNS_STDIN) janet_stream_close(proc->in);
+    if (proc->flags & JANET_PROC_OWNS_STDOUT) janet_stream_close(proc->out);
+    if (proc->flags & JANET_PROC_OWNS_STDERR) janet_stream_close(proc->err);
+#else
+    if (proc->flags & JANET_PROC_OWNS_STDIN) janet_file_close(proc->in);
+    if (proc->flags & JANET_PROC_OWNS_STDOUT) janet_file_close(proc->out);
+    if (proc->flags & JANET_PROC_OWNS_STDERR) janet_file_close(proc->err);
+#endif
+    proc->flags &= ~(JANET_PROC_OWNS_STDIN | JANET_PROC_OWNS_STDOUT | JANET_PROC_OWNS_STDERR);
+    if (proc->flags & (JANET_PROC_WAITED | JANET_PROC_WAITING)) {
+        return janet_wrap_nil();
+    }
+#ifdef JANET_EV
+    os_proc_wait_impl(proc);
+    return janet_wrap_nil();
+#else
+    return os_proc_wait_impl(proc);
+#endif
+}
+
 static void swap_handles(JanetHandle *handles) {
     JanetHandle temp = handles[0];
     handles[0] = handles[1];
@@ -514,10 +576,24 @@ static void close_handle(JanetHandle handle) {
 /* Create piped file for os/execute and os/spawn. Need to be careful that we mark
    the error flag if we can't create pipe and don't leak handles. *handle will be cleaned
    up by the calling function. If everything goes well, *handle is owned by the calling function,
-   (if it is set) and the returned JanetFile owns the other end of the pipe, which will be closed
+   (if it is set) and the returned handle owns the other end of the pipe, which will be closed
    on GC or fclose. */
 static JanetHandle make_pipes(JanetHandle *handle, int reverse, int *errflag) {
     JanetHandle handles[2];
+#ifdef JANET_EV
+
+    /* non-blocking pipes */
+    if (janet_make_pipe(handles, reverse ? 2 : 1)) goto error;
+    if (reverse) swap_handles(handles);
+#ifdef JANET_WINDOWS
+    if (!SetHandleInformation(handles[0], HANDLE_FLAG_INHERIT, 0)) goto error;
+#endif
+    *handle = handles[1];
+    return handles[0];
+
+#else
+
+    /* Normal blocking pipes */
 #ifdef JANET_WINDOWS
     SECURITY_ATTRIBUTES saAttr;
     memset(&saAttr, 0, sizeof(saAttr));
@@ -535,6 +611,8 @@ static JanetHandle make_pipes(JanetHandle *handle, int reverse, int *errflag) {
     *handle = handles[1];
     return handles[0];
 #endif
+
+#endif
 error:
     *errflag = 1;
     return JANET_HANDLE_NONE;
@@ -543,6 +621,11 @@ error:
 static const JanetMethod proc_methods[] = {
     {"wait", os_proc_wait},
     {"kill", os_proc_kill},
+    {"close", os_proc_close},
+    /* dud methods for janet_proc_next */
+    {"in", NULL},
+    {"out", NULL},
+    {"err", NULL},
     {NULL, NULL}
 };
 
@@ -568,17 +651,29 @@ static int janet_proc_get(void *p, Janet key, Janet *out) {
     return janet_getmethod(janet_unwrap_keyword(key), proc_methods, out);
 }
 
+static Janet janet_proc_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(proc_methods, key);
+}
+
 static const JanetAbstractType ProcAT = {
     "core/process",
     janet_proc_gc,
     janet_proc_mark,
     janet_proc_get,
-    JANET_ATEND_GET
+    NULL, /* put */
+    NULL, /* marshal */
+    NULL, /* unmarshal */
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    janet_proc_next,
+    JANET_ATEND_NEXT
 };
 
 static JanetHandle janet_getjstream(Janet *argv, int32_t n, void **orig) {
 #ifdef JANET_EV
-    JanetStream *stream = janet_checkabstract(argv[0], &janet_stream_type);
+    JanetStream *stream = janet_checkabstract(argv[n], &janet_stream_type);
     if (stream != NULL) {
         if (stream->flags & JANET_STREAM_CLOSED)
             janet_panic("stream is closed");
@@ -586,7 +681,7 @@ static JanetHandle janet_getjstream(Janet *argv, int32_t n, void **orig) {
         return stream->handle;
     }
 #endif
-    JanetFile *f = janet_checkabstract(argv[0], &janet_file_type);
+    JanetFile *f = janet_checkabstract(argv[n], &janet_file_type);
     if (f != NULL) {
         if (f->flags & JANET_FILE_CLOSED) {
             janet_panic("file is closed");
@@ -662,7 +757,7 @@ static Janet os_execute_impl(int32_t argc, Janet *argv, int is_spawn) {
 
     /* Get environment */
     int use_environ = !janet_flag_at(flags, 0);
-    char **envp = os_execute_env(argc, argv);
+    EnvBlock envp = os_execute_env(argc, argv);
 
     /* Get arguments */
     JanetView exargs = janet_getindexed(argv, 0);
@@ -675,6 +770,7 @@ static Janet os_execute_impl(int32_t argc, Janet *argv, int is_spawn) {
     JanetHandle new_in = JANET_HANDLE_NONE, new_out = JANET_HANDLE_NONE, new_err = JANET_HANDLE_NONE;
     JanetHandle pipe_in = JANET_HANDLE_NONE, pipe_out = JANET_HANDLE_NONE, pipe_err = JANET_HANDLE_NONE;
     int pipe_errflag = 0; /* Track errors setting up pipes */
+    int pipe_owner_flags = 0;
 
     /* Get optional redirections */
     if (argc > 2) {
@@ -684,16 +780,19 @@ static Janet os_execute_impl(int32_t argc, Janet *argv, int is_spawn) {
         Janet maybe_stderr = janet_dictionary_get(tab.kvs, tab.cap, janet_ckeywordv("err"));
         if (janet_keyeq(maybe_stdin, "pipe")) {
             new_in = make_pipes(&pipe_in, 1, &pipe_errflag);
+            pipe_owner_flags |= JANET_PROC_OWNS_STDIN;
         } else if (!janet_checktype(maybe_stdin, JANET_NIL)) {
             new_in = janet_getjstream(&maybe_stdin, 0, &orig_in);
         }
         if (janet_keyeq(maybe_stdout, "pipe")) {
             new_out = make_pipes(&pipe_out, 0, &pipe_errflag);
+            pipe_owner_flags |= JANET_PROC_OWNS_STDOUT;
         } else if (!janet_checktype(maybe_stdout, JANET_NIL)) {
             new_out = janet_getjstream(&maybe_stdout, 0, &orig_out);
         }
-        if (janet_keyeq(maybe_stderr, "err")) {
+        if (janet_keyeq(maybe_stderr, "pipe")) {
             new_err = make_pipes(&pipe_err, 0, &pipe_errflag);
+            pipe_owner_flags |= JANET_PROC_OWNS_STDERR;
         } else if (!janet_checktype(maybe_stderr, JANET_NIL)) {
             new_err = janet_getjstream(&maybe_stderr, 0, &orig_err);
         }
@@ -722,10 +821,12 @@ static Janet os_execute_impl(int32_t argc, Janet *argv, int is_spawn) {
     startupInfo.cb = sizeof(startupInfo);
     startupInfo.dwFlags |= STARTF_USESTDHANDLES;
     saAttr.nLength = sizeof(saAttr);
-    saAttr.bInheritHandle = TRUE;
 
     JanetBuffer *buf = os_exec_escape(exargs);
     if (buf->count > 8191) {
+        if (pipe_in != JANET_HANDLE_NONE) CloseHandle(pipe_in);
+        if (pipe_out != JANET_HANDLE_NONE) CloseHandle(pipe_out);
+        if (pipe_err != JANET_HANDLE_NONE) CloseHandle(pipe_err);
         janet_panic("command line string too long (max 8191 characters)");
     }
     const char *path = (const char *) janet_unwrap_string(exargs.items[0]);
@@ -757,10 +858,6 @@ static Janet os_execute_impl(int32_t argc, Janet *argv, int is_spawn) {
         startupInfo.hStdError = (HANDLE) _get_osfhandle(2);
     }
 
-    /* Use _spawn family of functions. */
-    /* Windows docs say do this before any spawns. */
-    _flushall();
-
     int cp_failed = 0;
     if (!CreateProcess(janet_flag_at(flags, 1) ? NULL : path,
                        (char *) buf->data, /* Single CLI argument */
@@ -788,15 +885,6 @@ static Janet os_execute_impl(int32_t argc, Janet *argv, int is_spawn) {
     pHandle = processInfo.hProcess;
     tHandle = processInfo.hThread;
 
-    /* Wait and cleanup immedaitely */
-    if (!is_spawn) {
-        DWORD code;
-        WaitForSingleObject(pHandle, INFINITE);
-        GetExitCodeProcess(pHandle, &code);
-        status = (int) code;
-        CloseHandle(pHandle);
-        CloseHandle(tHandle);
-    }
 #else
 
     const char **child_argv = janet_smalloc(sizeof(char *) * ((size_t) exargs.len + 1));
@@ -853,16 +941,9 @@ static Janet os_execute_impl(int32_t argc, Janet *argv, int is_spawn) {
         janet_unlock_environ();
     }
 
-    /* Wait for child */
+    os_execute_cleanup(envp, child_argv);
     if (status) {
-        os_execute_cleanup(envp, child_argv);
         janet_panicf("%p: %s", argv[0], strerror(errno));
-    } else if (is_spawn) {
-        /* Get process handle */
-        os_execute_cleanup(envp, child_argv);
-    } else {
-        /* Wait to complete */
-        os_execute_cleanup(envp, child_argv);
     }
 
 #endif
@@ -877,24 +958,24 @@ static Janet os_execute_impl(int32_t argc, Janet *argv, int is_spawn) {
     proc->in = NULL;
     proc->out = NULL;
     proc->err = NULL;
-    if (new_in != JANET_HANDLE_NONE) {
-        proc->in = get_stdio_for_handle(new_in, orig_in, 0);
-        if (NULL == proc->in) janet_panic("failed to construct proc");
-    }
-    if (new_out != JANET_HANDLE_NONE) {
-        proc->out = get_stdio_for_handle(new_out, orig_out, 1);
-        if (NULL == proc->out) janet_panic("failed to construct proc");
-    }
-    if (new_err != JANET_HANDLE_NONE) {
-        proc->err = get_stdio_for_handle(new_err, orig_err, 1);
-        if (NULL == proc->err) janet_panic("failed to construct proc");
-    }
-    proc->flags = 0;
+    proc->flags = pipe_owner_flags;
     if (janet_flag_at(flags, 2)) {
         proc->flags |= JANET_PROC_ERROR_NONZERO;
     }
-
     if (is_spawn) {
+        /* Only set up pointers to stdin, stdout, and stderr if os/spawn. */
+        if (new_in != JANET_HANDLE_NONE) {
+            proc->in = get_stdio_for_handle(new_in, orig_in, 1);
+            if (NULL == proc->in) janet_panic("failed to construct proc");
+        }
+        if (new_out != JANET_HANDLE_NONE) {
+            proc->out = get_stdio_for_handle(new_out, orig_out, 0);
+            if (NULL == proc->out) janet_panic("failed to construct proc");
+        }
+        if (new_err != JANET_HANDLE_NONE) {
+            proc->err = get_stdio_for_handle(new_err, orig_err, 0);
+            if (NULL == proc->err) janet_panic("failed to construct proc");
+        }
         return janet_wrap_abstract(proc);
     } else {
 #ifdef JANET_EV
@@ -913,15 +994,34 @@ static Janet os_spawn(int32_t argc, Janet *argv) {
     return os_execute_impl(argc, argv, 1);
 }
 
+#ifdef JANET_EV
+/* Runs in a separate thread */
+static JanetEVGenericMessage os_shell_subr(JanetEVGenericMessage args) {
+    int stat = system((const char *) args.argp);
+    free(args.argp);
+    if (args.argi) {
+        args.tag = JANET_EV_TCTAG_INTEGER;
+    } else {
+        args.tag = JANET_EV_TCTAG_BOOLEAN;
+    }
+    args.argi = stat;
+    return args;
+}
+#endif
+
 static Janet os_shell(int32_t argc, Janet *argv) {
     janet_arity(argc, 0, 1);
     const char *cmd = argc
                       ? janet_getcstring(argv, 0)
                       : NULL;
+#ifdef JANET_EV
+    janet_ev_threaded_await(os_shell_subr, 0, argc, cmd ? strdup(cmd) : NULL);
+#else
     int stat = system(cmd);
     return argc
            ? janet_wrap_integer(stat)
            : janet_wrap_boolean(stat);
+#endif
 }
 
 #endif /* JANET_NO_PROCESSES */
@@ -1833,7 +1933,7 @@ static Janet os_pipe(int32_t argc, Janet *argv) {
     (void) argv;
     janet_fixarity(argc, 0);
     JanetHandle fds[2];
-    if (janet_make_pipe(fds)) janet_panicv(janet_ev_lasterr());
+    if (janet_make_pipe(fds, 0)) janet_panicv(janet_ev_lasterr());
     JanetStream *reader = janet_stream(fds[0], JANET_STREAM_READABLE, NULL);
     JanetStream *writer = janet_stream(fds[1], JANET_STREAM_WRITABLE, NULL);
     Janet tup[2] = {janet_wrap_abstract(reader), janet_wrap_abstract(writer)};
@@ -1855,27 +1955,27 @@ static const JanetReg os_cfuns[] = {
         "os/which", os_which,
         JDOC("(os/which)\n\n"
              "Check the current operating system. Returns one of:\n\n"
-             "\t:windows\n"
-             "\t:macos\n"
-             "\t:web - Web assembly (emscripten)\n"
-             "\t:linux\n"
-             "\t:freebsd\n"
-             "\t:openbsd\n"
-             "\t:netbsd\n"
-             "\t:posix - A POSIX compatible system (default)\n\n"
+             "* :windows\n\n"
+             "* :macos\n\n"
+             "* :web - Web assembly (emscripten)\n\n"
+             "* :linux\n\n"
+             "* :freebsd\n\n"
+             "* :openbsd\n\n"
+             "* :netbsd\n\n"
+             "* :posix - A POSIX compatible system (default)\n\n"
              "May also return a custom keyword specified at build time.")
     },
     {
         "os/arch", os_arch,
         JDOC("(os/arch)\n\n"
              "Check the ISA that janet was compiled for. Returns one of:\n\n"
-             "\t:x86\n"
-             "\t:x86-64\n"
-             "\t:arm\n"
-             "\t:aarch64\n"
-             "\t:sparc\n"
-             "\t:wasm\n"
-             "\t:unknown\n")
+             "* :x86\n\n"
+             "* :x86-64\n\n"
+             "* :arm\n\n"
+             "* :aarch64\n\n"
+             "* :sparc\n\n"
+             "* :wasm\n\n"
+             "* :unknown\n")
     },
 #ifndef JANET_REDUCED_OS
     {
@@ -1897,22 +1997,22 @@ static const JanetReg os_cfuns[] = {
     {
         "os/stat", os_stat,
         JDOC("(os/stat path &opt tab|key)\n\n"
-             "Gets information about a file or directory. Returns a table If the third argument is a keyword, returns "
-             " only that information from stat. If the file or directory does not exist, returns nil. The keys are\n\n"
-             "\t:dev - the device that the file is on\n"
-             "\t:mode - the type of file, one of :file, :directory, :block, :character, :fifo, :socket, :link, or :other\n"
-             "\t:int-permissions - A Unix permission integer like 8r744\n"
-             "\t:permissions - A Unix permission string like \"rwxr--r--\"\n"
-             "\t:uid - File uid\n"
-             "\t:gid - File gid\n"
-             "\t:nlink - number of links to file\n"
-             "\t:rdev - Real device of file. 0 on windows.\n"
-             "\t:size - size of file in bytes\n"
-             "\t:blocks - number of blocks in file. 0 on windows\n"
-             "\t:blocksize - size of blocks in file. 0 on windows\n"
-             "\t:accessed - timestamp when file last accessed\n"
-             "\t:changed - timestamp when file last changed (permissions changed)\n"
-             "\t:modified - timestamp when file last modified (content changed)\n")
+             "Gets information about a file or directory. Returns a table if the second argument is a keyword, returns "
+             " only that information from stat. If the file or directory does not exist, returns nil. The keys are:\n\n"
+             "* :dev - the device that the file is on\n\n"
+             "* :mode - the type of file, one of :file, :directory, :block, :character, :fifo, :socket, :link, or :other\n\n"
+             "* :int-permissions - A Unix permission integer like 8r744\n\n"
+             "* :permissions - A Unix permission string like \"rwxr--r--\"\n\n"
+             "* :uid - File uid\n\n"
+             "* :gid - File gid\n\n"
+             "* :nlink - number of links to file\n\n"
+             "* :rdev - Real device of file. 0 on windows.\n\n"
+             "* :size - size of file in bytes\n\n"
+             "* :blocks - number of blocks in file. 0 on windows\n\n"
+             "* :blocksize - size of blocks in file. 0 on windows\n\n"
+             "* :accessed - timestamp when file last accessed\n\n"
+             "* :changed - timestamp when file last changed (permissions changed)\n\n"
+             "* :modified - timestamp when file last modified (content changed)\n")
     },
     {
         "os/lstat", os_lstat,
@@ -1986,20 +2086,20 @@ static const JanetReg os_cfuns[] = {
     {
         "os/execute", os_execute,
         JDOC("(os/execute args &opts flags env)\n\n"
-             "Execute a program on the system and pass it string arguments. Flags "
+             "Execute a program on the system and pass it string arguments. `flags` "
              "is a keyword that modifies how the program will execute.\n\n"
-             "\t:e - enables passing an environment to the program. Without :e, the "
-             "current environment is inherited.\n"
-             "\t:p - allows searching the current PATH for the binary to execute. "
-             "Without this flag, binaries must use absolute paths.\n"
-             "\t:x - raise error if exit code is non-zero.\n"
-             "env is a table or struct mapping environment variables to values. It can also "
+             "* :e - enables passing an environment to the program. Without :e, the "
+             "current environment is inherited.\n\n"
+             "* :p - allows searching the current PATH for the binary to execute. "
+             "Without this flag, binaries must use absolute paths.\n\n"
+             "* :x - raise error if exit code is non-zero.\n\n"
+             "`env` is a table or struct mapping environment variables to values. It can also "
              "contain the keys :in, :out, and :err, which allow redirecting stdio in the subprocess. "
              "These arguments should be core/file values. "
              "One can also pass in the :pipe keyword "
              "for these arguments to create files that will read (for :err and :out) or write (for :in) "
-             "to the file descriptor of the subprocess. This is only useful in os/spawn, which takes "
-             "the same parameters as os/execute, but will return an object that contains references to these "
+             "to the file descriptor of the subprocess. This is only useful in `os/spawn`, which takes "
+             "the same parameters as `os/execute`, but will return an object that contains references to these "
              "files via (return-value :in), (return-value :out), and (return-value :err). "
              "Returns the exit status of the program.")
     },
@@ -2026,6 +2126,12 @@ static const JanetReg os_cfuns[] = {
              "handle on windows. If wait is truthy, will wait for the process to finsih and "
              "returns the exit code. Otherwise, returns proc.")
     },
+    {
+        "os/proc-close", os_proc_close,
+        JDOC("(os/proc-close proc)\n\n"
+             "Wait on a process if it has not been waited on, and close pipes created by `os/spawn` "
+             "if they have not been closed. Returns nil.")
+    },
 #endif
     {
         "os/setenv", os_setenv,
@@ -2073,19 +2179,19 @@ static const JanetReg os_cfuns[] = {
     {
         "os/date", os_date,
         JDOC("(os/date &opt time local)\n\n"
-             "Returns the given time as a date struct, or the current time if no time is given. "
+             "Returns the given time as a date struct, or the current time if `time` is not given. "
              "Returns a struct with following key values. Note that all numbers are 0-indexed. "
-             "Date is given in UTC unless local is truthy, in which case the date is formatted for "
+             "Date is given in UTC unless `local` is truthy, in which case the date is formatted for "
              "the local timezone.\n\n"
-             "\t:seconds - number of seconds [0-61]\n"
-             "\t:minutes - number of minutes [0-59]\n"
-             "\t:hours - number of hours [0-23]\n"
-             "\t:month-day - day of month [0-30]\n"
-             "\t:month - month of year [0, 11]\n"
-             "\t:year - years since year 0 (e.g. 2019)\n"
-             "\t:week-day - day of the week [0-6]\n"
-             "\t:year-day - day of the year [0-365]\n"
-             "\t:dst - If Day Light Savings is in effect")
+             "* :seconds - number of seconds [0-61]\n\n"
+             "* :minutes - number of minutes [0-59]\n\n"
+             "* :hours - number of hours [0-23]\n\n"
+             "* :month-day - day of month [0-30]\n\n"
+             "* :month - month of year [0, 11]\n\n"
+             "* :year - years since year 0 (e.g. 2019)\n\n"
+             "* :week-day - day of the week [0-6]\n\n"
+             "* :year-day - day of the year [0-365]\n\n"
+             "* :dst - if Day Light Savings is in effect")
     },
     {
         "os/rename", os_rename,

src/core/parse.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -175,7 +175,14 @@ static void popstate(JanetParser *p, Janet val) {
         if (newtop->flags & PFLAG_CONTAINER) {
             newtop->argn++;
             /* Keep track of number of values in the root state */
-            if (p->statecount == 1) p->pending++;
+            if (p->statecount == 1) {
+                p->pending++;
+                /* Root items are always wrapped in a tuple for source map info. */
+                const Janet *tup = janet_tuple_n(&val, 1);
+                janet_tuple_sm_line(tup) = (int32_t) top.line;
+                janet_tuple_sm_column(tup) = (int32_t) top.column;
+                val = janet_wrap_tuple(tup);
+            }
             push_arg(p, val);
             return;
         } else if (newtop->flags & PFLAG_READERMAC) {
@@ -730,6 +737,19 @@ const char *janet_parser_error(JanetParser *parser) {
 }
 
 Janet janet_parser_produce(JanetParser *parser) {
+    Janet ret;
+    size_t i;
+    if (parser->pending == 0) return janet_wrap_nil();
+    ret = janet_unwrap_tuple(parser->args[0])[0];
+    for (i = 1; i < parser->argcount; i++) {
+        parser->args[i - 1] = parser->args[i];
+    }
+    parser->pending--;
+    parser->argcount--;
+    return ret;
+}
+
+Janet janet_parser_produce_wrapped(JanetParser *parser) {
     Janet ret;
     size_t i;
     if (parser->pending == 0) return janet_wrap_nil();
@@ -840,13 +860,21 @@ static int parsergc(void *p, size_t size) {
 }
 
 static int parserget(void *p, Janet key, Janet *out);
+static Janet parsernext(void *p, Janet key);
 
 const JanetAbstractType janet_parser_type = {
     "core/parser",
     parsergc,
     parsermark,
     parserget,
-    JANET_ATEND_GET
+    NULL, /* put */
+    NULL, /* marshal */
+    NULL, /* unmarshal */
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    parsernext,
+    JANET_ATEND_NEXT
 };
 
 /* C Function parser */
@@ -902,8 +930,13 @@ static Janet cfun_parse_insert(int32_t argc, Janet *argv) {
     if (s->flags & PFLAG_COMMENT) s--;
     if (s->flags & PFLAG_CONTAINER) {
         s->argn++;
-        if (p->statecount == 1) p->pending++;
-        push_arg(p, argv[1]);
+        if (p->statecount == 1) {
+            p->pending++;
+            Janet tup = janet_wrap_tuple(janet_tuple_n(argv + 1, 1));
+            push_arg(p, tup);
+        } else {
+            push_arg(p, argv[1]);
+        }
     } else if (s->flags & (PFLAG_STRING | PFLAG_LONGSTRING)) {
         const uint8_t *str = janet_to_string(argv[1]);
         int32_t slen = janet_string_length(str);
@@ -972,9 +1005,13 @@ static Janet cfun_parse_error(int32_t argc, Janet *argv) {
 }
 
 static Janet cfun_parse_produce(int32_t argc, Janet *argv) {
-    janet_fixarity(argc, 1);
+    janet_arity(argc, 1, 2);
     JanetParser *p = janet_getabstract(argv, 0, &janet_parser_type);
-    return janet_parser_produce(p);
+    if (argc == 2 && janet_truthy(argv[1])) {
+        return janet_parser_produce_wrapped(p);
+    } else {
+        return janet_parser_produce(p);
+    }
 }
 
 static Janet cfun_parse_flush(int32_t argc, Janet *argv) {
@@ -1183,6 +1220,11 @@ static int parserget(void *p, Janet key, Janet *out) {
     return janet_getmethod(janet_unwrap_keyword(key), parser_methods, out);
 }
 
+static Janet parsernext(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(parser_methods, key);
+}
+
 static const JanetReg parse_cfuns[] = {
     {
         "parser/new", cfun_parse_parser,
@@ -1204,10 +1246,12 @@ static const JanetReg parse_cfuns[] = {
     },
     {
         "parser/produce", cfun_parse_produce,
-        JDOC("(parser/produce parser)\n\n"
+        JDOC("(parser/produce parser &opt wrap)\n\n"
              "Dequeue the next value in the parse queue. Will return nil if "
              "no parsed values are in the queue, otherwise will dequeue the "
-             "next value.")
+             "next value. If `wrap` is truthy, will return a 1-element tuple that "
+             "wraps the result. This tuple can be used for source-mapping "
+             "purposes.")
     },
     {
         "parser/consume", cfun_parse_consume,
@@ -1234,9 +1278,9 @@ static const JanetReg parse_cfuns[] = {
         JDOC("(parser/status parser)\n\n"
              "Gets the current status of the parser state machine. The status will "
              "be one of:\n\n"
-             "\t:pending - a value is being parsed.\n"
-             "\t:error - a parsing error was encountered.\n"
-             "\t:root - the parser can either read more values or safely terminate.")
+             "* :pending - a value is being parsed.\n\n"
+             "* :error - a parsing error was encountered.\n\n"
+             "* :root - the parser can either read more values or safely terminate.")
     },
     {
         "parser/flush", cfun_parse_flush,
@@ -1250,10 +1294,10 @@ static const JanetReg parse_cfuns[] = {
         JDOC("(parser/state parser &opt key)\n\n"
              "Returns a representation of the internal state of the parser. If a key is passed, "
              "only that information about the state is returned. Allowed keys are:\n\n"
-             "\t:delimiters - Each byte in the string represents a nested data structure. For example, "
+             "* :delimiters - Each byte in the string represents a nested data structure. For example, "
              "if the parser state is '([\"', then the parser is in the middle of parsing a "
-             "string inside of square brackets inside parentheses. Can be used to augment a REPL prompt."
-             "\t:frames - Each table in the array represents a 'frame' in the parser state. Frames "
+             "string inside of square brackets inside parentheses. Can be used to augment a REPL prompt.\n\n"
+             "* :frames - Each table in the array represents a 'frame' in the parser state. Frames "
              "contain information about the start of the expression being parsed as well as the "
              "type of that expression and some type-specific information.")
     },

src/core/peg.c

@@ -44,11 +44,13 @@ typedef struct {
     JanetArray *captures;
     JanetBuffer *scratch;
     JanetBuffer *tags;
+    JanetArray *tagged_captures;
     const Janet *extrav;
     int32_t *linemap;
     int32_t extrac;
     int32_t depth;
     int32_t linemaplen;
+    int32_t has_backref;
     enum {
         PEG_MODE_NORMAL,
         PEG_MODE_ACCUMULATE
@@ -60,6 +62,7 @@ typedef struct {
  * if one branch fails and try a new branch. */
 typedef struct {
     int32_t cap;
+    int32_t tcap;
     int32_t scratch;
 } CapState;
 
@@ -68,6 +71,7 @@ static CapState cap_save(PegState *s) {
     CapState cs;
     cs.scratch = s->scratch->count;
     cs.cap = s->captures->count;
+    cs.tcap = s->tagged_captures->count;
     return cs;
 }
 
@@ -75,7 +79,15 @@ static CapState cap_save(PegState *s) {
 static void cap_load(PegState *s, CapState cs) {
     s->scratch->count = cs.scratch;
     s->captures->count = cs.cap;
-    s->tags->count = cs.cap;
+    s->tags->count = cs.tcap;
+    s->tagged_captures->count = cs.tcap;
+}
+
+/* Load a saved capture state in the case of success. Keeps
+ * tagged captures around for backref. */
+static void cap_load_keept(PegState *s, CapState cs) {
+    s->scratch->count = cs.scratch;
+    s->captures->count = cs.cap;
 }
 
 /* Add a capture */
@@ -83,8 +95,11 @@ static void pushcap(PegState *s, Janet capture, uint32_t tag) {
     if (s->mode == PEG_MODE_ACCUMULATE) {
         janet_to_string_b(s->scratch, capture);
     }
-    if (tag || s->mode == PEG_MODE_NORMAL) {
+    if (s->mode == PEG_MODE_NORMAL) {
         janet_array_push(s->captures, capture);
+    }
+    if (s->has_backref) {
+        janet_array_push(s->tagged_captures, capture);
         janet_buffer_push_u8(s->tags, tag);
     }
 }
@@ -271,7 +286,7 @@ tail:
             const uint8_t *next_text;
             CapState cs = cap_save(s);
             down1(s);
-            while (text < s->text_end) {
+            while (text <= s->text_end) {
                 CapState cs2 = cap_save(s);
                 next_text = peg_rule(s, rule_a, text);
                 if (next_text) {
@@ -281,7 +296,7 @@ tail:
                 text++;
             }
             up1(s);
-            if (text >= s->text_end) {
+            if (text > s->text_end) {
                 cap_load(s, cs);
                 return NULL;
             }
@@ -321,7 +336,7 @@ tail:
             uint32_t tag = rule[2];
             for (int32_t i = s->tags->count - 1; i >= 0; i--) {
                 if (s->tags->data[i] == search) {
-                    pushcap(s, s->captures->data[i], tag);
+                    pushcap(s, s->tagged_captures->data[i], tag);
                     return text;
                 }
             }
@@ -358,15 +373,15 @@ tail:
         }
 
         case RULE_CAPTURE: {
-            uint32_t tag = rule[2];
             down1(s);
             const uint8_t *result = peg_rule(s, s->bytecode + rule[1], text);
             up1(s);
             if (!result) return NULL;
             /* Specialized pushcap - avoid intermediate string creation */
-            if (!tag && s->mode == PEG_MODE_ACCUMULATE) {
+            if (!s->has_backref && s->mode == PEG_MODE_ACCUMULATE) {
                 janet_buffer_push_bytes(s->scratch, text, (int32_t)(result - text));
             } else {
+                uint32_t tag = rule[2];
                 pushcap(s, janet_stringv(text, (int32_t)(result - text)), tag);
             }
             return result;
@@ -388,7 +403,7 @@ tail:
             if (!result) return NULL;
             Janet cap = janet_stringv(s->scratch->data + cs.scratch,
                                       s->scratch->count - cs.scratch);
-            cap_load(s, cs);
+            cap_load_keept(s, cs);
             pushcap(s, cap, tag);
             return result;
         }
@@ -419,7 +434,7 @@ tail:
                         s->captures->data + cs.cap,
                         sizeof(Janet) * num_sub_captures);
             sub_captures->count = num_sub_captures;
-            cap_load(s, cs);
+            cap_load_keept(s, cs);
             pushcap(s, janet_wrap_array(sub_captures), tag);
             return result;
         }
@@ -464,7 +479,7 @@ tail:
                                      s->captures->data + cs.cap);
                     break;
             }
-            cap_load(s, cs);
+            cap_load_keept(s, cs);
             if (rule[0] == RULE_MATCHTIME && !janet_truthy(cap)) return NULL;
             pushcap(s, cap, tag);
             return result;
@@ -495,7 +510,7 @@ tail:
             uint32_t search = rule[1];
             for (int32_t i = s->tags->count - 1; i >= 0; i--) {
                 if (s->tags->data[i] == search) {
-                    Janet capture = s->captures->data[i];
+                    Janet capture = s->tagged_captures->data[i];
                     if (!janet_checktype(capture, JANET_STRING))
                         return NULL;
                     const uint8_t *bytes = janet_unwrap_string(capture);
@@ -581,6 +596,67 @@ tail:
             return text + width;
         }
 
+        case RULE_MATCHCAP: {
+            const uint32_t *rule_a = s->bytecode + rule[1];
+            const uint32_t *rule_b = s->bytecode + rule[2];
+
+            int oldmode = s->mode;
+            CapState cs = cap_save(s);
+            s->mode = PEG_MODE_NORMAL;
+            int32_t old_cap = s->captures->count;
+            down1(s);
+            const uint8_t *b_result = peg_rule(s, rule_b, text);
+            up1(s);
+            s->mode = oldmode;
+            if (!b_result) return NULL;
+            int32_t new_cap = s->captures->count;
+
+            /* Check for bad captures */
+            for (int32_t i = old_cap; i < new_cap; i++) {
+                Janet capture = s->captures->data[i];
+                if (!janet_checktype(capture, JANET_STRING)) {
+                    return NULL;
+                }
+            }
+
+            /* Save captures to temporary buffer */
+            Janet *temp_mem = janet_smalloc(sizeof(Janet) * (new_cap - old_cap));
+            for (int32_t i = old_cap; i < new_cap; i++) {
+                temp_mem[i - old_cap] = s->captures->data[i];
+            }
+            cap_load(s, cs);
+
+            for (int32_t i = old_cap; i < new_cap; i++) {
+                Janet capture = temp_mem[i - old_cap];
+                const uint8_t *str = janet_unwrap_string(capture);
+                PegState subs;
+                subs.mode = PEG_MODE_NORMAL;
+                subs.text_start = str;
+                subs.text_end = str + janet_string_length(str);
+                subs.depth = s->depth - 1;
+                subs.captures = s->captures;
+                subs.tagged_captures = s->tagged_captures;
+                subs.scratch = janet_buffer(10);
+                subs.tags = s->tags;
+                subs.constants = s->constants;
+                subs.bytecode = s->bytecode;
+                subs.linemap = NULL;
+                subs.linemaplen = -1;
+                subs.has_backref = s->has_backref;
+                subs.extrac = s->extrac;
+                subs.extrav = s->extrav;
+
+                const uint8_t *a_result = peg_rule(&subs, rule_a, str);
+                if (NULL == a_result) {
+                    janet_sfree(temp_mem);
+                    return NULL;
+                }
+            }
+
+            janet_sfree(temp_mem);
+            return b_result;
+        }
+
     }
 }
 
@@ -597,6 +673,7 @@ typedef struct {
     Janet form;
     int depth;
     uint32_t nexttag;
+    int has_backref;
 } Builder;
 
 /* Forward declaration to allow recursion */
@@ -828,6 +905,9 @@ static void spec_ifnot(Builder *b, int32_t argc, const Janet *argv) {
 static void spec_lenprefix(Builder *b, int32_t argc, const Janet *argv) {
     spec_branch(b, argc, argv, RULE_LENPREFIX);
 }
+static void spec_matchcap(Builder *b, int32_t argc, const Janet *argv) {
+    spec_branch(b, argc, argv, RULE_MATCHCAP);
+}
 
 static void spec_between(Builder *b, int32_t argc, const Janet *argv) {
     peg_fixarity(b, argc, 3);
@@ -937,6 +1017,7 @@ static void spec_reference(Builder *b, int32_t argc, const Janet *argv) {
     Reserve r = reserve(b, 3);
     uint32_t search = emit_tag(b, argv[0]);
     uint32_t tag = (argc == 2) ? emit_tag(b, argv[1]) : 0;
+    b->has_backref = 1;
     emit_2(r, RULE_GETTAG, search, tag);
 }
 
@@ -959,6 +1040,7 @@ static void spec_column(Builder *b, int32_t argc, const Janet *argv) {
 }
 
 static void spec_backmatch(Builder *b, int32_t argc, const Janet *argv) {
+    b->has_backref = 1;
     spec_tag1(b, argc, argv, RULE_BACKMATCH);
 }
 
@@ -1009,7 +1091,7 @@ static void spec_matchtime(Builder *b, int32_t argc, const Janet *argv) {
 static void spec_readint(Builder *b, int32_t argc, const Janet *argv, uint32_t mask) {
     peg_arity(b, argc, 1, 2);
     Reserve r = reserve(b, 3);
-    uint32_t tag = (argc == 2) ? emit_tag(b, argv[3]) : 0;
+    uint32_t tag = (argc == 2) ? emit_tag(b, argv[1]) : 0;
     int32_t width = peg_getnat(b, argv[0]);
     if ((width < 0) || (width > JANET_MAX_READINT_WIDTH)) {
         peg_panicf(b, "width must be between 0 and %d, got %d", JANET_MAX_READINT_WIDTH, width);
@@ -1072,6 +1154,7 @@ static const SpecialPair peg_specials[] = {
     {"lenprefix", spec_lenprefix},
     {"line", spec_line},
     {"look", spec_look},
+    {"matchcap", spec_matchcap},
     {"not", spec_not},
     {"opt", spec_opt},
     {"position", spec_position},
@@ -1295,6 +1378,7 @@ static void *peg_unmarshal(JanetMarshalContext *ctx) {
     }
 
     /* verify peg bytecode */
+    int32_t has_backref = 0;
     uint32_t i = 0;
     while (i < blen) {
         uint32_t instr = bytecode[i];
@@ -1310,9 +1394,13 @@ static void *peg_unmarshal(JanetMarshalContext *ctx) {
             case RULE_POSITION:
             case RULE_LINE:
             case RULE_COLUMN:
+                /* [1 word] */
+                i += 2;
+                break;
             case RULE_BACKMATCH:
                 /* [1 word] */
                 i += 2;
+                has_backref = 1;
                 break;
             case RULE_SET:
                 /* [8 words] */
@@ -1339,6 +1427,7 @@ static void *peg_unmarshal(JanetMarshalContext *ctx) {
             case RULE_IF:
             case RULE_IFNOT:
             case RULE_LENPREFIX:
+            case RULE_MATCHCAP:
                 /* [rule_a, rule_b (b if not a)] */
                 if (rule[1] >= blen) goto bad;
                 if (rule[2] >= blen) goto bad;
@@ -1353,9 +1442,13 @@ static void *peg_unmarshal(JanetMarshalContext *ctx) {
                 i += 4;
                 break;
             case RULE_ARGUMENT:
+                /* [searchtag, tag] */
+                i += 3;
+                break;
             case RULE_GETTAG:
                 /* [searchtag, tag] */
                 i += 3;
+                has_backref = 1;
                 break;
             case RULE_CONSTANT:
                 /* [constant, tag] */
@@ -1409,6 +1502,7 @@ static void *peg_unmarshal(JanetMarshalContext *ctx) {
     /* Good return */
     peg->bytecode = bytecode;
     peg->constants = constants;
+    peg->has_backref = has_backref;
     free(op_flags);
     return peg;
 
@@ -1418,16 +1512,21 @@ bad:
 }
 
 static int cfun_peg_getter(JanetAbstract a, Janet key, Janet *out);
+static Janet peg_next(void *p, Janet key);
 
 const JanetAbstractType janet_peg_type = {
     "core/peg",
     NULL,
     peg_mark,
     cfun_peg_getter,
-    NULL,
+    NULL, /* put */
     peg_marshal,
     peg_unmarshal,
-    JANET_ATEND_UNMARSHAL
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    peg_next,
+    JANET_ATEND_NEXT
 };
 
 /* Convert Builder to JanetPeg (Janet Abstract Value) */
@@ -1445,6 +1544,7 @@ static JanetPeg *make_peg(Builder *b) {
     safe_memcpy(peg->bytecode, b->bytecode, bytecode_size);
     safe_memcpy(peg->constants, b->constants, constants_size);
     peg->bytecode_len = janet_v_count(b->bytecode);
+    peg->has_backref = b->has_backref;
     return peg;
 }
 
@@ -1452,13 +1552,20 @@ static JanetPeg *make_peg(Builder *b) {
 static JanetPeg *compile_peg(Janet x) {
     Builder builder;
     builder.grammar = janet_table(0);
-    builder.default_grammar = janet_get_core_table("default-peg-grammar");
+    builder.default_grammar = NULL;
+    {
+        Janet default_grammarv = janet_dyn("peg-grammar");
+        if (janet_checktype(default_grammarv, JANET_TABLE)) {
+            builder.default_grammar = janet_unwrap_table(default_grammarv);
+        }
+    }
     builder.tags = janet_table(0);
     builder.constants = NULL;
     builder.bytecode = NULL;
     builder.nexttag = 1;
     builder.form = x;
     builder.depth = JANET_RECURSION_GUARD;
+    builder.has_backref = 0;
     peg_compile1(&builder, x);
     JanetPeg *peg = make_peg(&builder);
     builder_cleanup(&builder);
@@ -1515,12 +1622,14 @@ static PegCall peg_cfun_init(int32_t argc, Janet *argv, int get_replace) {
     ret.s.text_end = ret.bytes.bytes + ret.bytes.len;
     ret.s.depth = JANET_RECURSION_GUARD;
     ret.s.captures = janet_array(0);
+    ret.s.tagged_captures = janet_array(0);
     ret.s.scratch = janet_buffer(10);
     ret.s.tags = janet_buffer(10);
     ret.s.constants = ret.peg->constants;
     ret.s.bytecode = ret.peg->bytecode;
     ret.s.linemap = NULL;
     ret.s.linemaplen = -1;
+    ret.s.has_backref = ret.peg->has_backref;
     return ret;
 }
 
@@ -1609,12 +1718,18 @@ static int cfun_peg_getter(JanetAbstract a, Janet key, Janet *out) {
     return janet_getmethod(janet_unwrap_keyword(key), peg_methods, out);
 }
 
+static Janet peg_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(peg_methods, key);
+}
+
 static const JanetReg peg_cfuns[] = {
     {
         "peg/compile", cfun_peg_compile,
         JDOC("(peg/compile peg)\n\n"
              "Compiles a peg source data structure into a <core/peg>. This will speed up matching "
-             "if the same peg will be used multiple times.")
+             "if the same peg will be used multiple times. Will also use `(dyn :peg-grammar)` to suppliment "
+             "the grammar of the peg for otherwise undefined peg keywords.")
     },
     {
         "peg/match", cfun_peg_match,

src/core/pp.c

@@ -42,7 +42,14 @@ static void number_to_string_b(JanetBuffer *buffer, double x) {
     const char *fmt = (x == floor(x) &&
                        x <= JANET_INTMAX_DOUBLE &&
                        x >= JANET_INTMIN_DOUBLE) ? "%.0f" : "%g";
-    int count = snprintf((char *) buffer->data + buffer->count, BUFSIZE, fmt, x);
+    int count;
+    if (x == 0.0) {
+        /* Prevent printing of '-0' */
+        count = 1;
+        buffer->data[buffer->count] = '0';
+    } else {
+        count = snprintf((char *) buffer->data + buffer->count, BUFSIZE, fmt, x);
+    }
     buffer->count += count;
 }
 

src/core/string.c

@@ -398,6 +398,7 @@ static Janet cfun_string_split(int32_t argc, Janet *argv) {
         const uint8_t *slice = janet_string(state.text + lastindex, result - lastindex);
         janet_array_push(array, janet_wrap_string(slice));
         lastindex = result + state.patlen;
+        kmp_seti(&state, lastindex);
     }
     const uint8_t *slice = janet_string(state.text + lastindex, state.textlen - lastindex);
     janet_array_push(array, janet_wrap_string(slice));
@@ -598,9 +599,8 @@ static const JanetReg string_cfuns[] = {
         JDOC("(string/find-all patt str)\n\n"
              "Searches for all instances of pattern patt in string "
              "str. Returns an array of all indices of found patterns. Overlapping "
-             "instances of the pattern are not counted, meaning a byte in string "
-             "will only contribute to finding at most on occurrence of pattern. If no "
-             "occurrences are found, will return an empty array.")
+             "instances of the pattern are counted individually, meaning a byte in str "
+             "may contribute to multiple found patterns.")
     },
     {
         "string/has-prefix?", cfun_string_hasprefix,
@@ -621,7 +621,8 @@ static const JanetReg string_cfuns[] = {
     {
         "string/replace-all", cfun_string_replaceall,
         JDOC("(string/replace-all patt subst str)\n\n"
-             "Replace all instances of patt with subst in the string str. "
+             "Replace all instances of patt with subst in the string str. Overlapping "
+             "matches will not be counted, only the first match in such a span will be replaced. "
              "Will return the new string if patt is found, otherwise returns str.")
     },
     {

src/core/thread.c

@@ -84,6 +84,9 @@ static JANET_THREAD_LOCAL JanetTable *janet_vm_thread_decode = NULL;
 static JanetTable *janet_thread_get_decode(void) {
     if (janet_vm_thread_decode == NULL) {
         janet_vm_thread_decode = janet_get_core_table("load-image-dict");
+        if (NULL == janet_vm_thread_decode) {
+            janet_vm_thread_decode = janet_table(0);
+        }
         janet_gcroot(janet_wrap_table(janet_vm_thread_decode));
     }
     return janet_vm_thread_decode;
@@ -418,13 +421,21 @@ int janet_thread_receive(Janet *msg_out, double timeout) {
 }
 
 static int janet_thread_getter(void *p, Janet key, Janet *out);
+static Janet janet_thread_next(void *p, Janet key);
 
 const JanetAbstractType janet_thread_type = {
     "core/thread",
     thread_gc,
     thread_mark,
     janet_thread_getter,
-    JANET_ATEND_GET
+    NULL, /* put */
+    NULL, /* marshal */
+    NULL, /* unmarshal */
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    janet_thread_next,
+    JANET_ATEND_NEXT
 };
 
 static JanetThread *janet_make_thread(JanetMailbox *mailbox, JanetTable *encode) {
@@ -512,7 +523,7 @@ static int thread_worker(JanetMailboxPair *pair) {
         janet_stacktrace(fiber, out);
     }
 
-#ifdef JANET_NET
+#ifdef JANET_EV
     janet_loop();
 #endif
 
@@ -708,6 +719,11 @@ static int janet_thread_getter(void *p, Janet key, Janet *out) {
     return janet_getmethod(janet_unwrap_keyword(key), janet_thread_methods, out);
 }
 
+static Janet janet_thread_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(janet_thread_methods, key);
+}
+
 static const JanetReg threadlib_cfuns[] = {
     {
         "thread/current", cfun_thread_current,
@@ -720,10 +736,10 @@ static const JanetReg threadlib_cfuns[] = {
              "Start a new thread that will start immediately. "
              "If capacity is provided, that is how many messages can be stored in the thread's mailbox before blocking senders. "
              "The capacity must be between 1 and 65535 inclusive, and defaults to 10. "
-             "Can optionally provide flags to the new thread - supported flags are:\n"
-             "\t:h - Start a heavyweight thread. This loads the core environment by default, so may use more memory initially. Messages may compress better, though.\n"
-             "\t:a - Allow sending over registered abstract types to the new thread\n"
-             "\t:c - Send over cfunction information to the new thread.\n"
+             "Can optionally provide flags to the new thread - supported flags are:\n\n"
+             "* :h - Start a heavyweight thread. This loads the core environment by default, so may use more memory initially. Messages may compress better, though.\n\n"
+             "* :a - Allow sending over registered abstract types to the new thread\n\n"
+             "* :c - Send over cfunction information to the new thread.\n\n"
              "Returns a handle to the new thread.")
     },
     {

src/core/util.h

@@ -91,6 +91,7 @@ void janet_buffer_format(
     int32_t argstart,
     int32_t argc,
     Janet *argv);
+Janet janet_next_impl(Janet ds, Janet key, int is_interpreter);
 
 /* Inside the janet core, defining globals is different
  * at bootstrap time and normal runtime */
@@ -107,6 +108,11 @@ void janet_core_cfuns(JanetTable *env, const char *regprefix, const JanetReg *cf
 int janet_gettime(struct timespec *spec);
 #endif
 
+/* strdup */
+#ifdef JANET_WINDOWS
+#define strdup(x) _strdup(x)
+#endif
+
 #define RETRY_EINTR(RC, CALL) do { (RC) = CALL; } while((RC) < 0 && errno == EINTR)
 
 /* Initialize builtin libraries */
@@ -145,7 +151,7 @@ extern const JanetAbstractType janet_address_type;
 #ifdef JANET_EV
 void janet_lib_ev(JanetTable *env);
 void janet_ev_mark(void);
-int janet_make_pipe(JanetHandle handles[2]);
+int janet_make_pipe(JanetHandle handles[2], int mode);
 #endif
 
 #endif

src/core/value.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -25,6 +25,7 @@
 #include "util.h"
 #include "state.h"
 #include "gc.h"
+#include "fiber.h"
 #include <janet.h>
 #endif
 
@@ -115,6 +116,10 @@ static int traversal_next(Janet *x, Janet *y) {
  */
 
 Janet janet_next(Janet ds, Janet key) {
+    return janet_next_impl(ds, key, 0);
+}
+
+Janet janet_next_impl(Janet ds, Janet key, int is_interpreter) {
     JanetType t = janet_type(ds);
     switch (t) {
         default:
@@ -177,6 +182,44 @@ Janet janet_next(Janet ds, Janet key) {
             if (NULL == at->next) break;
             return at->next(abst, key);
         }
+        case JANET_FIBER: {
+            JanetFiber *child = janet_unwrap_fiber(ds);
+            Janet retreg;
+            JanetFiberStatus status = janet_fiber_status(child);
+            if (status == JANET_STATUS_ALIVE ||
+                    status == JANET_STATUS_DEAD ||
+                    status == JANET_STATUS_ERROR ||
+                    status == JANET_STATUS_USER0 ||
+                    status == JANET_STATUS_USER1 ||
+                    status == JANET_STATUS_USER2 ||
+                    status == JANET_STATUS_USER3 ||
+                    status == JANET_STATUS_USER4) {
+                return janet_wrap_nil();
+            }
+            janet_vm_fiber->child = child;
+            JanetSignal sig = janet_continue(child, janet_wrap_nil(), &retreg);
+            if (sig != JANET_SIGNAL_OK && !(child->flags & (1 << sig))) {
+                if (is_interpreter) {
+                    janet_signalv(sig, retreg);
+                } else {
+                    janet_vm_fiber->child = NULL;
+                    janet_panicv(retreg);
+                }
+            }
+            janet_vm_fiber->child = NULL;
+            if (sig == JANET_SIGNAL_OK ||
+                    sig == JANET_SIGNAL_ERROR ||
+                    sig == JANET_SIGNAL_USER0 ||
+                    sig == JANET_SIGNAL_USER1 ||
+                    sig == JANET_SIGNAL_USER2 ||
+                    sig == JANET_SIGNAL_USER3 ||
+                    sig == JANET_SIGNAL_USER4) {
+                /* Fiber cannot be resumed, so discard last value. */
+                return janet_wrap_nil();
+            } else {
+                return janet_wrap_integer(0);
+            }
+        }
     }
     return janet_wrap_nil();
 }
@@ -264,18 +307,14 @@ int32_t janet_hash(Janet x) {
             hash = janet_struct_hash(janet_unwrap_struct(x));
             break;
         case JANET_NUMBER: {
-            double num = janet_unwrap_number(x);
-            if (isnan(num) || isinf(num) || num == 0) {
-                hash = 0;
-            } else {
-                hash = (int32_t)num;
-                hash = ((hash >> 16) ^ hash) * 0x45d9f3b;
-                hash = ((hash >> 16) ^ hash) * 0x45d9f3b;
-                hash = (hash >> 16) ^ hash;
-
-                uint32_t lo = (uint32_t)(janet_u64(x) & 0xFFFFFFFF);
-                hash ^= lo + 0x9e3779b9 + (hash << 6) + (hash >> 2);
-            }
+            union {
+                double d;
+                uint64_t u;
+            } as;
+            as.d = janet_unwrap_number(x);
+            uint32_t lo = (uint32_t)(as.u & 0xFFFFFFFF);
+            uint32_t hi = (uint32_t)(as.u >> 32);
+            hash = (int32_t)(hi ^ (lo >> 3));
             break;
         }
         case JANET_ABSTRACT: {
@@ -434,6 +473,14 @@ Janet janet_in(Janet ds, Janet key) {
             }
             break;
         }
+        case JANET_FIBER: {
+            /* Bit of a hack to allow iterating over fibers. */
+            if (janet_equals(key, janet_wrap_integer(0))) {
+                return janet_unwrap_fiber(ds)->last_value;
+            } else {
+                janet_panicf("expected key 0, got %v", key);
+            }
+        }
     }
     return value;
 }
@@ -489,6 +536,14 @@ Janet janet_get(Janet ds, Janet key) {
             const JanetKV *st = janet_unwrap_struct(ds);
             return janet_struct_get(st, key);
         }
+        case JANET_FIBER: {
+            /* Bit of a hack to allow iterating over fibers. */
+            if (janet_equals(key, janet_wrap_integer(0))) {
+                return janet_unwrap_fiber(ds)->last_value;
+            } else {
+                return janet_wrap_nil();
+            }
+        }
     }
 }
 
@@ -545,6 +600,14 @@ Janet janet_getindex(Janet ds, int32_t index) {
             }
             break;
         }
+        case JANET_FIBER: {
+            if (index == 0) {
+                value = janet_unwrap_fiber(ds)->last_value;
+            } else {
+                value = janet_wrap_nil();
+            }
+            break;
+        }
     }
     return value;
 }

src/core/vm.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -202,6 +202,20 @@ JANET_THREAD_LOCAL jmp_buf *janet_vm_jmp_buf = NULL;
             vm_checkgc_pcnext();\
         }\
     }
+#define vm_compop_imm(op) \
+    {\
+        Janet op1 = stack[B];\
+        if (janet_checktype(op1, JANET_NUMBER)) {\
+            double x1 = janet_unwrap_number(op1);\
+            double x2 = (double) CS; \
+            stack[A] = janet_wrap_boolean(x1 op x2);\
+            vm_pcnext();\
+        } else {\
+            vm_commit();\
+            stack[A] = janet_wrap_boolean(janet_compare(op1, janet_wrap_integer(CS)) op 0);\
+            vm_checkgc_pcnext();\
+        }\
+    }
 
 /* Trace a function call */
 static void vm_do_trace(JanetFunction *func, int32_t argc, const Janet *argv) {
@@ -780,8 +794,7 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in) {
     vm_compop( <=);
 
     VM_OP(JOP_LESS_THAN_IMMEDIATE)
-    stack[A] = janet_wrap_boolean(janet_unwrap_integer(stack[B]) < CS);
-    vm_pcnext();
+    vm_compop_imm( <);
 
     VM_OP(JOP_GREATER_THAN)
     vm_compop( >);
@@ -790,15 +803,14 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in) {
     vm_compop( >=);
 
     VM_OP(JOP_GREATER_THAN_IMMEDIATE)
-    stack[A] = janet_wrap_boolean(janet_unwrap_integer(stack[B]) > CS);
-    vm_pcnext();
+    vm_compop_imm( >);
 
     VM_OP(JOP_EQUALS)
     stack[A] = janet_wrap_boolean(janet_equals(stack[B], stack[C]));
     vm_pcnext();
 
     VM_OP(JOP_EQUALS_IMMEDIATE)
-    stack[A] = janet_wrap_boolean(janet_unwrap_integer(stack[B]) == CS);
+    stack[A] = janet_wrap_boolean(janet_unwrap_number(stack[B]) == (double) CS);
     vm_pcnext();
 
     VM_OP(JOP_NOT_EQUALS)
@@ -806,7 +818,7 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in) {
     vm_pcnext();
 
     VM_OP(JOP_NOT_EQUALS_IMMEDIATE)
-    stack[A] = janet_wrap_boolean(janet_unwrap_integer(stack[B]) != CS);
+    stack[A] = janet_wrap_boolean(janet_unwrap_number(stack[B]) != (double) CS);
     vm_pcnext();
 
     VM_OP(JOP_COMPARE)
@@ -815,7 +827,11 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in) {
 
     VM_OP(JOP_NEXT)
     vm_commit();
-    stack[A] = janet_next(stack[B], stack[C]);
+    {
+        Janet temp = janet_next_impl(stack[B], stack[C], 1);
+        vm_restore();
+        stack[A] = temp;
+    }
     vm_pcnext();
 
     VM_OP(JOP_LOAD_NIL)
@@ -1272,7 +1288,14 @@ Janet janet_call(JanetFunction *fun, int32_t argc, const Janet *argv) {
     /* Push frame */
     janet_fiber_pushn(janet_vm_fiber, argv, argc);
     if (janet_fiber_funcframe(janet_vm_fiber, fun)) {
-        janet_panicf("arity mismatch in %v", janet_wrap_function(fun));
+        int32_t min = fun->def->min_arity;
+        int32_t max = fun->def->max_arity;
+        Janet funv = janet_wrap_function(fun);
+        if (min == max && min != argc)
+            janet_panicf("arity mismatch in %v, expected %d, got %d", funv, min, argc);
+        if (min >= 0 && argc < min)
+            janet_panicf("arity mismatch in %v, expected at least %d, got %d", funv, min, argc);
+        janet_panicf("arity mismatch in %v, expected at most %d, got %d", funv, max, argc);
     }
     janet_fiber_frame(janet_vm_fiber)->flags |= JANET_STACKFRAME_ENTRANCE;
 
@@ -1341,18 +1364,42 @@ static JanetSignal janet_continue_no_check(JanetFiber *fiber, Janet in, Janet *o
     janet_fiber_did_resume(fiber);
 #endif
 
+    /* Clear last value */
+    fiber->last_value = janet_wrap_nil();
+
     /* Continue child fiber if it exists */
     if (fiber->child) {
         if (janet_vm_root_fiber == NULL) janet_vm_root_fiber = fiber;
         JanetFiber *child = fiber->child;
+        uint32_t instr = (janet_stack_frame(fiber->data + fiber->frame)->pc)[0];
         janet_vm_stackn++;
         JanetSignal sig = janet_continue(child, in, &in);
         janet_vm_stackn--;
         if (janet_vm_root_fiber == fiber) janet_vm_root_fiber = NULL;
         if (sig != JANET_SIGNAL_OK && !(child->flags & (1 << sig))) {
             *out = in;
+            janet_fiber_set_status(fiber, sig);
             return sig;
         }
+        /* Check if we need any special handling for certain opcodes */
+        switch (instr & 0x7F) {
+            default:
+                break;
+            case JOP_NEXT: {
+                if (sig == JANET_SIGNAL_OK ||
+                        sig == JANET_SIGNAL_ERROR ||
+                        sig == JANET_SIGNAL_USER0 ||
+                        sig == JANET_SIGNAL_USER1 ||
+                        sig == JANET_SIGNAL_USER2 ||
+                        sig == JANET_SIGNAL_USER3 ||
+                        sig == JANET_SIGNAL_USER4) {
+                    in = janet_wrap_nil();
+                } else {
+                    in = janet_wrap_integer(0);
+                }
+                break;
+            }
+        }
         fiber->child = NULL;
     }
 
@@ -1371,22 +1418,23 @@ static JanetSignal janet_continue_no_check(JanetFiber *fiber, Janet in, Janet *o
 
     /* Save global state */
     JanetTryState tstate;
-    JanetSignal signal = janet_try(&tstate);
-    if (!signal) {
+    JanetSignal sig = janet_try(&tstate);
+    if (!sig) {
         /* Normal setup */
         if (janet_vm_root_fiber == NULL) janet_vm_root_fiber = fiber;
         janet_vm_fiber = fiber;
         janet_fiber_set_status(fiber, JANET_STATUS_ALIVE);
-        signal = run_vm(fiber, in);
+        sig = run_vm(fiber, in);
     }
 
     /* Restore */
     if (janet_vm_root_fiber == fiber) janet_vm_root_fiber = NULL;
-    janet_fiber_set_status(fiber, signal);
+    janet_fiber_set_status(fiber, sig);
     janet_restore(&tstate);
+    fiber->last_value = tstate.payload;
     *out = tstate.payload;
 
-    return signal;
+    return sig;
 }
 
 /* Enter the main vm loop */

src/include/janet.h

@@ -838,9 +838,15 @@ struct JanetFiber {
     JanetTable *env; /* Dynamic bindings table (usually current environment). */
     Janet *data; /* Dynamically resized stack memory */
     JanetFiber *child; /* Keep linked list of fibers for restarting pending fibers */
+    Janet last_value; /* Last returned value from a fiber */
 #ifdef JANET_EV
+    /* These fields are only relevant for fibers that are used as "root fibers" -
+     * that is, fibers that are scheduled on the event loop and behave much like threads
+     * in a multi-tasking system. It would be possible to move these fields to a new
+     * type, say "JanetTask", that as separate from fibers to save a bit of space. */
     JanetListenerState *waiting;
     uint32_t sched_id; /* Increment everytime fiber is scheduled by event loop */
+    void *supervisor_channel; /* Channel to push self to when complete */
 #endif
 };
 
@@ -1318,6 +1324,7 @@ typedef struct {
 #define JANET_EV_TCTAG_ERR_STRING 5   /* cancel with janet_cstringv((const char *) argp) */
 #define JANET_EV_TCTAG_ERR_STRINGF 6  /* cancel with janet_cstringv((const char *) argp), then call free on argp. */
 #define JANET_EV_TCTAG_ERR_KEYWORD 7  /* cancel with janet_ckeywordv((const char *) argp) */
+#define JANET_EV_TCTAG_BOOLEAN 8      /* resume with janet_wrap_boolean(argi) */
 
 /* Function pointer that is run in the thread pool */
 typedef JanetEVGenericMessage(*JanetThreadedSubroutine)(JanetEVGenericMessage arguments);
@@ -1327,7 +1334,7 @@ typedef void (*JanetThreadedCallback)(JanetEVGenericMessage return_value);
 
 /* API calls for quickly offloading some work in C to a new thread or thread pool. */
 JANET_API void janet_ev_threaded_call(JanetThreadedSubroutine fp, JanetEVGenericMessage arguments, JanetThreadedCallback cb);
-JANET_API void janet_ev_threaded_await(JanetThreadedSubroutine fp, int tag, int argi, void *argp);
+JANET_NO_RETURN JANET_API void janet_ev_threaded_await(JanetThreadedSubroutine fp, int tag, int argi, void *argp);
 
 /* Callback used by janet_ev_threaded_await */
 JANET_API void janet_ev_default_threaded_callback(JanetEVGenericMessage return_value);
@@ -1360,6 +1367,7 @@ JANET_API void janet_parser_deinit(JanetParser *parser);
 JANET_API void janet_parser_consume(JanetParser *parser, uint8_t c);
 JANET_API enum JanetParserStatus janet_parser_status(JanetParser *parser);
 JANET_API Janet janet_parser_produce(JanetParser *parser);
+JANET_API Janet janet_parser_produce_wrapped(JanetParser *parser);
 JANET_API const char *janet_parser_error(JanetParser *parser);
 JANET_API void janet_parser_flush(JanetParser *parser);
 JANET_API void janet_parser_eof(JanetParser *parser);
@@ -1399,6 +1407,7 @@ JANET_API JanetCompileResult janet_compile(Janet source, JanetTable *env, JanetS
 
 /* Get the default environment for janet */
 JANET_API JanetTable *janet_core_env(JanetTable *replacements);
+JANET_API JanetTable *janet_core_lookup_table(JanetTable *replacements);
 
 JANET_API int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char *sourcePath, Janet *out);
 JANET_API int janet_dostring(JanetTable *env, const char *str, const char *sourcePath, Janet *out);
@@ -1694,6 +1703,7 @@ JANET_API void janet_arity(int32_t arity, int32_t min, int32_t max);
 JANET_API void janet_fixarity(int32_t arity, int32_t fix);
 
 JANET_API int janet_getmethod(JanetKeyword method, const JanetMethod *methods, Janet *out);
+JANET_API Janet janet_nextmethod(const JanetMethod *methods, Janet key);
 
 JANET_API double janet_getnumber(const Janet *argv, int32_t n);
 JANET_API JanetArray *janet_getarray(const Janet *argv, int32_t n);
@@ -1771,6 +1781,7 @@ JANET_API FILE *janet_dynfile(const char *name, FILE *def);
 JANET_API JanetFile *janet_getjfile(const Janet *argv, int32_t n);
 JANET_API JanetAbstract janet_checkfile(Janet j);
 JANET_API FILE *janet_unwrapfile(Janet j, int32_t *flags);
+JANET_API int janet_file_close(JanetFile *file);
 
 JANET_API int janet_cryptorand(uint8_t *out, size_t n);
 
@@ -1830,7 +1841,8 @@ typedef enum {
     RULE_LENPREFIX,    /* [rule_a, rule_b (repeat rule_b rule_a times)] */
     RULE_READINT,      /* [(signedness << 4) | (endianess << 5) | bytewidth, tag] */
     RULE_LINE,         /* [tag] */
-    RULE_COLUMN        /* [tag] */
+    RULE_COLUMN,       /* [tag] */
+    RULE_MATCHCAP      /* [rule_a, rule_b] */
 } JanetPegOpcod;
 
 typedef struct {
@@ -1838,6 +1850,7 @@ typedef struct {
     Janet *constants;
     size_t bytecode_len;
     uint32_t num_constants;
+    int has_backref;
 } JanetPeg;
 
 #endif

src/mainclient/shell.c

@@ -1042,19 +1042,23 @@ int main(int argc, char **argv) {
     janet_table_put(env, janet_ckeywordv("executable"), janet_cstringv(argv[0]));
 
     /* Run startup script */
-    Janet mainfun, out;
+    Janet mainfun;
     janet_resolve(env, janet_csymbol("cli-main"), &mainfun);
     Janet mainargs[1] = { janet_wrap_array(args) };
     JanetFiber *fiber = janet_fiber(janet_unwrap_function(mainfun), 64, 1, mainargs);
     fiber->env = env;
+
+#ifdef JANET_EV
+    janet_gcroot(janet_wrap_fiber(fiber));
+    janet_schedule(fiber, janet_wrap_nil());
+    janet_loop();
+    status = janet_fiber_status(fiber);
+#else
+    Janet out;
     status = janet_continue(fiber, janet_wrap_nil(), &out);
     if (status != JANET_SIGNAL_OK && status != JANET_SIGNAL_EVENT) {
         janet_stacktrace(fiber, out);
     }
-
-#ifdef JANET_EV
-    status = JANET_SIGNAL_OK;
-    janet_loop();
 #endif
 
     /* Deinitialize vm */

test/suite0001.janet

@@ -281,4 +281,38 @@
 (assert (not (even? -10.1)) "even? 8")
 (assert (not (even? -10.6)) "even? 9")
 
+# Map arities
+(assert (deep= (map inc [1 2 3]) @[2 3 4]))
+(assert (deep= (map + [1 2 3] [10 20 30]) @[11 22 33]))
+(assert (deep= (map + [1 2 3] [10 20 30] [100 200 300]) @[111 222 333]))
+(assert (deep= (map + [1 2 3] [10 20 30] [100 200 300] [1000 2000 3000]) @[1111 2222 3333]))
+(assert (deep= (map + [1 2 3] [10 20 30] [100 200 300] [1000 2000 3000] [10000 20000 30000]) @[11111 22222 33333]))
+
+# Sort function
+(assert (deep=
+          (range 99)
+          (sort (mapcat (fn [[x y z]] [z y x]) (partition 3 (range 99))))) "sort 5")
+(assert (<= ;(sort (map (fn [x] (math/random)) (range 1000)))) "sort 6")
+
+# And and or
+
+(assert (= (and true true) true) "and true true")
+(assert (= (and true false) false) "and true false")
+(assert (= (and false true) false) "and false true")
+(assert (= (and true true true) true) "and true true true")
+(assert (= (and 0 1 2) 2) "and 0 1 2")
+(assert (= (and 0 1 nil) nil) "and 0 1 nil")
+(assert (= (and 1) 1) "and 1")
+(assert (= (and) true) "and with no arguments")
+
+(assert (= (or true true) true) "or true true")
+(assert (= (or true false) true) "or true false")
+(assert (= (or false true) true) "or false true")
+(assert (= (or false false) false) "or false true")
+(assert (= (or true true false) true) "or true true false")
+(assert (= (or 0 1 2) 0) "or 0 1 2")
+(assert (= (or nil 1 2) 1) "or nil 1 2")
+(assert (= (or 1) 1) "or 1")
+(assert (= (or) nil) "or with no arguments")
+
 (end-suite)

test/suite0002.janet

@@ -49,7 +49,7 @@
 # Generators
 (def gen (generate [x :range [0 100] :when (pos? (% x 4))] x))
 (var gencount 0)
-(loop [x :generate gen]
+(loop [x :in gen]
   (++ gencount)
   (assert (pos? (% x 4)) "generate in loop"))
 (assert (= gencount 75) "generate loop count")

test/suite0003.janet

@@ -473,4 +473,21 @@
 
 (check-deep '(* (int 2) -1) "123" nil)
 
+# to/thru bug
+(check-deep '(to -1) "aaaa" @[])
+(check-deep '(thru -1) "aaaa" @[])
+(check-deep ''(to -1) "aaaa" @["aaaa"])
+(check-deep ''(thru -1) "aaaa" @["aaaa"])
+(check-deep '(to "b") "aaaa" nil)
+(check-deep '(thru "b") "aaaa" nil)
+
+# matchcap
+(def matchcap-test
+  ~{:span (* "{" '(to "}") "}")
+    :parse-span (* '3 3)
+    :main (matchcap :parse-span (any :span ))})
+
+(check-deep matchcap-test "{bigrig}{fatman}{catdog}" @["big" "fat" "cat"])
+(check-deep matchcap-test "{bigrig}{fatman}{catdoggy}" @["big" "fat" "cat"])
+
 (end-suite)

test/suite0006.janet

@@ -190,4 +190,39 @@
 (assert (= (test-expand "../def.txt" ":all:") "../def.txt") "module/expand-path 7")
 (assert (= (test-expand "../././././abcd/../def.txt" ":all:") "../def.txt") "module/expand-path 8")
 
+# Integer type checks
+(assert (compare= 0 (- (int/u64 "1000") 1000)) "subtract from int/u64")
+
+(assert (odd? (int/u64 "1001")) "odd? 1")
+(assert (not (odd? (int/u64 "1000"))) "odd? 2")
+(assert (odd? (int/s64 "1001")) "odd? 3")
+(assert (not (odd? (int/s64 "1000"))) "odd? 4")
+(assert (odd? (int/s64 "-1001")) "odd? 5")
+(assert (not (odd? (int/s64 "-1000"))) "odd? 6")
+
+(assert (even? (int/u64 "1000")) "even? 1")
+(assert (not (even? (int/u64 "1001"))) "even? 2")
+(assert (even? (int/s64 "1000")) "even? 3")
+(assert (not (even? (int/s64 "1001"))) "even? 4")
+(assert (even? (int/s64 "-1000")) "even? 5")
+(assert (not (even? (int/s64 "-1001"))) "even? 6")
+
+# integer type operations
+(defn modcheck [x y]
+  (assert (= (string (mod x y)) (string (mod (int/s64 x) y)))
+          (string "int/s64 (mod " x " " y ") expected " (mod x y) ", got "
+                  (mod (int/s64 x) y)))
+  (assert (= (string (% x y)) (string (% (int/s64 x) y)))
+          (string "int/s64 (% " x " " y ") expected " (% x y) ", got "
+                  (% (int/s64 x) y))))
+
+(modcheck 1 2)
+(modcheck 1 3)
+(modcheck 4 2)
+(modcheck 4 1)
+(modcheck 10 3)
+(modcheck 10 -3)
+(modcheck -10 3)
+(modcheck -10 -3)
+
 (end-suite)

test/suite0007.janet

@@ -320,4 +320,8 @@
   (array/push a x))
 (assert (deep= (range 4) a) "eachk 1")
 
+
+(tracev (def my-unique-var-name true))
+(assert my-unique-var-name "tracev upscopes")
+
 (end-suite)

test/suite0008.janet

@@ -309,7 +309,7 @@ neldb\0\0\0\xD8\x05printG\x01\0\xDE\xDE\xDE'\x03\0marshal_tes/\x02
 # Issue 428
 (var result nil)
 (defn f [] (yield {:a :ok}))
-(assert-no-error "issue 428 1" (loop [{:a x} :generate (fiber/new f)] (set result x)))
+(assert-no-error "issue 428 1" (loop [{:a x} :in (fiber/new f)] (set result x)))
 (assert (= result :ok) "issue 428 2")
 
 # Inline 3 argument get

test/suite0009.janet

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Calvin Rose & contributors
+# Copyright (c) 2021 Calvin Rose & contributors
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to
@@ -21,45 +21,125 @@
 (import ./helper :prefix "" :exit true)
 (start-suite 9)
 
+# Subprocess
+
+(def janet (dyn :executable))
+
+(repeat 10
+
+  (let [p (os/spawn [janet "-e" `(print "hello")`] :p {:out :pipe})]
+    (os/proc-wait p)
+    (def x (:read (p :out) :all))
+    (assert (deep= "hello" (string/trim x)) "capture stdout from os/spawn pre close."))
+
+  (let [p (os/spawn [janet "-e" `(print "hello")`] :p {:out :pipe})]
+    (def x (:read (p :out) 1024))
+    (os/proc-wait p)
+    (assert (deep= "hello" (string/trim x)) "capture stdout from os/spawn post close."))
+
+  (let [p (os/spawn [janet "-e" `(file/read stdin :line)`] :px {:in :pipe})]
+    (:write (p :in) "hello!\n")
+    (assert-no-error "pipe stdin to process" (os/proc-wait p))))
+
+(let [p (os/spawn [janet "-e" `(print (file/read stdin :line))`] :px {:in :pipe :out :pipe})]
+  (:write (p :in) "hello!\n")
+  (def x (:read (p :out) 1024))
+  (assert-no-error "pipe stdin to process 2" (os/proc-wait p))
+  (assert (= "hello!" (string/trim x)) "round trip pipeline in process"))
+
+# Parallel subprocesses
+
+(defn calc-1
+  "Run subprocess, read from stdout, then wait on subprocess."
+  [code]
+  (let [p (os/spawn [janet "-e" (string `(printf "%j" ` code `)`)] :px {:out :pipe})]
+    (os/proc-wait p)
+    (def output (:read (p :out) :all))
+    (parse output)))
+
+(assert
+  (deep=
+    (ev/gather
+      (calc-1 "(+ 1 2 3 4)")
+      (calc-1 "(+ 5 6 7 8)")
+      (calc-1 "(+ 9 10 11 12)"))
+    @[10 26 42]) "parallel subprocesses 1")
+
+(defn calc-2
+  "Run subprocess, wait on subprocess, then read from stdout. Read only up to 10 bytes instead of :all"
+  [code]
+  (let [p (os/spawn [janet "-e" (string `(printf "%j" ` code `)`)] :px {:out :pipe})]
+    (def output (:read (p :out) 10))
+    (os/proc-wait p)
+    (parse output)))
+
+(assert
+  (deep=
+    (ev/gather
+      (calc-2 "(+ 1 2 3 4)")
+      (calc-2 "(+ 5 6 7 8)")
+      (calc-2 "(+ 9 10 11 12)"))
+    @[10 26 42]) "parallel subprocesses 2")
+
+# File piping
+
+(assert-no-error "file writing 1"
+  (with [f (file/temp)]
+    (os/execute [janet "-e" `(repeat 20 (print :hello))`] :p {:out f})))
+
+(assert-no-error "file writing 2"
+  (with [f (file/open "unique.txt" :w)]
+    (os/execute [janet "-e" `(repeat 20 (print :hello))`] :p {:out f})
+    (file/flush f)))
+
+# Issue #593
+(assert-no-error "file writing 3"
+  (def outfile (file/open "unique.txt" :w))
+  (os/execute [janet "-e" "(pp (seq [i :range (1 10)] i))"] :p {:out outfile})
+  (file/flush outfile)
+  (file/close outfile)
+  (os/rm "unique.txt"))
+
+# ev/gather
+
+(assert (deep= @[1 2 3] (ev/gather 1 2 3)) "ev/gather 1")
+(assert (deep= @[] (ev/gather)) "ev/gather 2")
+(assert-error "ev/gather 3" (ev/gather 1 2 (error 3)))
+
 # Net testing
 
-(defn handler
-  "Simple handler for connections."
-  [stream]
-  (defer (:close stream)
-    (def id (gensym))
-    (def b @"")
-    (:read stream 1024 b)
-    (:write stream b)
-    (buffer/clear b)))
+(repeat 10
 
-(def s (net/server "127.0.0.1" "8000" handler))
-(assert s "made server 1")
+  (defn handler
+    "Simple handler for connections."
+    [stream]
+    (defer (:close stream)
+      (def id (gensym))
+      (def b @"")
+      (net/read stream 1024 b)
+      (net/write stream b)
+      (buffer/clear b)))
 
-# We need some sleep for windows to let the server stabilize
-# or else the first read can fail. Might be a strange windows
-# "bug", but needs further investigating. Otherwise, `build_win test`
-# can sometimes fail on windows, leading to flaky testing.
-(ev/sleep 0.3)
+  (def s (net/server "127.0.0.1" "8000" handler))
+  (assert s "made server 1")
 
-(defn test-echo [msg]
-  (with [conn (net/connect "127.0.0.1" "8000")]
-    (:write conn msg)
-    (def res (:read conn 1024))
-    (assert (= (string res) msg) (string "echo " msg))))
+  (defn test-echo [msg]
+    (with [conn (net/connect "127.0.0.1" "8000")]
+      (net/write conn msg)
+      (def res (net/read conn 1024))
+      (assert (= (string res) msg) (string "echo " msg))))
 
-(test-echo "hello")
-(test-echo "world")
-(test-echo (string/repeat "abcd" 200))
+  (test-echo "hello")
+  (test-echo "world")
+  (test-echo (string/repeat "abcd" 200))
 
-(:close s)
+  (:close s))
 
 # Create pipe
 
 (var pipe-counter 0)
 (def chan (ev/chan 10))
 (let [[reader writer] (os/pipe)]
-  (ev/sleep 0.3)
   (ev/spawn
     (while (ev/read reader 3)
       (++ pipe-counter))
@@ -73,14 +153,14 @@
   (ev/take chan))
 
 (var result nil)
-(def fiber
-  (ev/spawn 
-    (set result (protect (ev/sleep 0.4)))
+(var fiber nil)
+(set fiber
+  (ev/spawn
+    (set result (protect (ev/sleep 10)))
     (assert (= result '(false "boop")) "ev/cancel 1")))
-(ev/sleep 0.1)
+(ev/sleep 0)
 (ev/cancel fiber "boop")
-(ev/sleep 0.1)
 
-(assert-error "bad arity to ev/call" (ev/call inc 1 2 3))
+(assert (os/execute [janet "-e" `(+ 1 2 3)`] :xp) "os/execute self")
 
 (end-suite)

test/suite0010.janet

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Calvin Rose & contributors
+#- Copyright (c) 2020 Calvin Rose & contributors
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to
@@ -131,5 +131,34 @@
 (check-indent "\n    Hello, world!\n   dedented text\n    " 4)
 (check-indent "\n    Hello, world!\n    indented text\n    " 4)
 
+# String bugs
+(assert (deep= (string/find-all "qq" "qqq") @[0 1]) "string/find-all 1")
+(assert (deep= (string/find-all "q" "qqq") @[0 1 2]) "string/find-all 2")
+(assert (deep= (string/split "qq" "1qqqqz") @["1" "" "z"]) "string/split 1")
+(assert (deep= (string/split "aa" "aaa") @["" "a"]) "string/split 2")
+
+# Comparisons
+(assert (> 1e23 100) "less than immediate 1")
+(assert (> 1e23 1000) "less than immediate 2")
+(assert (< 100 1e23) "greater than immediate 1")
+(assert (< 1000 1e23) "greater than immediate 2")
+
+# os/execute with environment variables
+(assert (= 0 (os/execute [(dyn :executable) "-e" "(+ 1 2 3)"] :pe {"HELLO" "WORLD"})) "os/execute with env")
+
+# Regression #638
+(compwhen
+  (dyn 'ev/go)
+  (assert
+    (= [true :caught]
+       (protect
+         (try
+           (do
+             (ev/sleep 0)
+             (with-dyns []
+               (ev/sleep 0)
+               (error "oops")))
+           ([err] :caught))))
+    "regression #638"))
 
 (end-suite)