Update some sr.ht configs.

Previous, the instructions were defined only for values that
fit into 32 bit integers for legacy reasons.

.builds/linux.yml

@@ -0,0 +1,23 @@
+image: archlinux
+sources:
+- https://git.sr.ht/~bakpakin/janet
+packages:
+- meson
+tasks:
+- with-epoll: |
+    cd janet
+    meson setup with-epoll --buildtype=release
+    cd with-epoll
+    meson configure -Depoll=true
+    ninja
+    ninja test
+- no-epoll: |
+    cd janet
+    meson setup no-epoll --buildtype=release
+    cd no-epoll
+    meson configure -Depoll=false
+    ninja
+    ninja test
+    sudo ninja install
+    sudo jpm --verbose install circlet
+    sudo jpm --verbose install spork

.builds/meson.yml

@@ -1,14 +0,0 @@
-image: openbsd/latest
-sources:
-- https://git.sr.ht/~bakpakin/janet
-packages:
-- meson
-tasks:
-- build: |
-    cd janet
-    meson setup build --buildtype=release
-    cd build
-    ninja
-    ninja test
-    doas ninja install
-    doas jpm --verbose install circlet

.builds/meson2.yml

@@ -1,15 +0,0 @@
-image: openbsd/latest
-sources:
-- https://git.sr.ht/~bakpakin/janet
-packages:
-- meson
-tasks:
-- build: |
-    cd janet
-    meson setup build --buildtype=release
-    cd build
-    meson configure -Dprf=true
-    ninja
-    ninja test
-    doas ninja install
-    doas jpm --verbose install circlet

.builds/meson_min.yml

@@ -1,22 +0,0 @@
-image: openbsd/latest
-sources:
-- https://git.sr.ht/~bakpakin/janet
-packages:
-- meson
-tasks:
-- build: |
-    cd janet
-    meson setup build --buildtype=release
-    cd build
-    meson configure -Dsingle_threaded=true
-    meson configure -Dnanbox=false
-    meson configure -Ddynamic_modules=false
-    meson configure -Ddocstrings=false
-    meson configure -Dnet=false
-    meson configure -Dsourcemaps=false
-    meson configure -Dpeg=false
-    meson configure -Dassembler=false
-    meson configure -Dint_types=false
-    meson configure -Dtyped_array=false
-    meson configure -Dreduced_os=true
-    ninja # will not pass tests but should build

.builds/openbsd.yml

@@ -3,10 +3,31 @@ sources:
 - https://git.sr.ht/~bakpakin/janet
 packages:
 - gmake
+- meson
 tasks:
-- build: |
+- gmake: |
     cd janet
     gmake
     gmake test
     doas gmake install
     gmake test-install
+- meson_min: |
+    cd janet
+    meson setup build_meson_min --buildtype=release -Dsingle_threaded=true -Dnanbox=false -Ddynamic_modules=false -Ddocstrings=false -Dnet=false -Dsourcemaps=false -Dpeg=false -Dassembler=false -Dint_types=false -Dtyped_array=false -Dreduced_os=true
+    cd build_meson_min
+    ninja
+- meson_prf: |
+    cd janet
+    meson setup build_meson_prf --buildtype=release -Dprf=true
+    cd build_meson_prf
+    ninja
+    ninja test
+- meson_default: |
+    cd janet
+    meson setup build_meson_default --buildtype=release
+    cd build_meson_default
+    ninja
+    ninja test
+    doas ninja install
+    doas jpm --verbose install circlet
+

.gitignore

@@ -140,3 +140,6 @@ compile_commands.json
 CTestTestfile.cmake
 
 # End of https://www.gitignore.io/api/cmake
+
+# Astyle
+*.orig

CHANGELOG.md

@@ -1,8 +1,83 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
-## Unreleased - ???
+## 1.15.0 - 2021-02-08
+- Fix `gtim` and `ltim` bytecode instructions on non-integer values.
+- Clean up output of flychecking to be the same as the repl.
+- Change behavior of `debug/stacktrace` with a nil error value.
+- Add optional argument to `parser/produce`.
+- Add `no-core` option to creating standalone binaries to make execution faster.
+- Fix bug where a buffer overflow could be confused with an out of memory error.
+- Change error output to `file:line:column: message`. Column is in bytes - tabs
+  are considered to have width 1 (instead of 8).
+
+## 1.14.2 - 2021-01-23
+- Allow `JANET_PROFILE` env variable to load a profile before loading the repl.
+- Update `tracev` macro to allow `def` and `var` inside to work as expected.
+- Use `(dyn :peg-grammar)` for passing a default grammar to `peg/compile` instead of loading
+  `default-peg-grammar` directly from the root environment.
+- Add `ev/thread` for combining threading with the event loop.
+- Add `ev/do-thread` to make `ev/thread` easier to use.
+- Automatically set supervisor channel in `net/accept-loop` and `net/server` correctly.
+
+## 1.14.1 - 2021-01-18
+- Add `doc-of` for reverse documentation lookup.
+- Add `ev/give-supervsior` to send a message to the supervising channel.
+- Add `ev/gather` and `chan` argument to `ev/go`. This new argument allows "supervisor channels"
+  for fibers to enable structured concurrency.
+- Make `-k` flag work on stdin if no files are given.
+- Add `flycheck` function to core.
+- Make `backmatch` and `backref` more expressive in pegs.
+- Fix buggy `string/split`.
+- Add `fiber/last-value` to get the value that was last yielded, errored, or signaled
+  by a fiber.
+- Remove `:generate` verb from `loop` macros. Instead, use the `:in` verb
+  which will now work on fibers as well as other data structures.
+- Define `next`, `get`, and `in` for fibers. This lets
+  `each`, `map`, and similar iteration macros can now iterate over fibers.
+- Remove macro `eachy`, which can be replaced by `each`.
+- Add `dflt` argument to find-index.
+- Deprecate `file/popen` in favor of `os/spawn`.
+- Add `:all` keyword to `ev/read` and `net/read` to make them more like `file/read`. However, we
+  do not provide any `:line` option as that requires buffering.
+- Change repl behavior to make Ctrl-C raise SIGINT on posix. The old behavior for Ctrl-C,
+  to clear the current line buffer, has been moved to Ctrl-Q.
+- Importing modules that start with `/` is now the only way to import from project root.
+  Before, this would import from / on disk. Previous imports that did not start with `.` or `/`
+  are now unambiguously importing from the syspath, instead of checking both the syspath and
+  the project root. This is backwards incompatible and dependencies should be updated for this.
+- Change hash function for numbers.
+- Improve error handling of `dofile`.
+- Bug fixes in networking and subprocess code.
+- Use markdown formatting in more places for docstrings.
+
+## 1.13.1 - 2020-12-13
+- Pretty printing a table with a prototype will look for `:_name` instead of `:name`
+  in the prototype table to tag the output.
+- `match` macro implementation changed to be tail recursive.
+- Adds a :preload loader which allows one to manually put things into `module/cache`.
+- Add `buffer/push` function.
+- Backtick delimited strings and buffers are now reindented based on the column of the
+  opening delimiter. Whitespace in columns to the left of the starting column is ignored unless
+  there are non-space/non-newline characters in that region, in which case the old behavior is preserved.
+- Argument to `(error)` combinator in PEGs is now optional.
+- Add `(line)` and `(column)` combinators to PEGs to capture source line and column.
+  This should make error reporting a bit easier.
+- Add `merge-module` to core.
+- During installation and release, merge janetconf.h into janet.h for easier install.
+- Add `upscope` special form.
+- `os/execute` and `os/spawn` can take streams for redirecting IO.
+- Add `:parser` and `:read` parameters to `run-context`.
+- Add `os/open` if ev is enabled.
+- Add `os/pipe` if ev is enabled.
+- Add `janet_thread_current(void)` to C API
 - Add integer parsing forms to pegs. This makes parsing many binary protocols easier.
+- Lots of updates to networking code - now can use epoll (or poll) on linux and IOCP on windows.
+- Add `ev/` module. This exposes a fiber scheduler, queues, timeouts, and other functionality to users
+  for single threaded cooperative scheduling and asynchronous IO.
+- Add `net/accept-loop` and `net/listen`. These functions break down `net/server` into it's essential parts
+  and are more flexible. They also allow further improvements to these utility functions.
+- Various small bug fixes.
 
 ## 1.12.2 - 2020-09-20
 - Add janet\_try and janet\_restore to C API.

Makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Calvin Rose
+# Copyright (c) 2021 Calvin Rose
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to
@@ -27,7 +27,7 @@ PREFIX?=/usr/local
 INCLUDEDIR?=$(PREFIX)/include
 BINDIR?=$(PREFIX)/bin
 LIBDIR?=$(PREFIX)/lib
-JANET_BUILD?="\"$(shell git log --pretty=format:'%h' -n 1 || echo local)\""
+JANET_BUILD?="\"$(shell git log --pretty=format:'%h' -n 2> /dev/null || echo local)\""
 CLIBS=-lm -lpthread
 JANET_TARGET=build/janet
 JANET_LIBRARY=build/libjanet.so
@@ -66,8 +66,8 @@ ifeq ($(UNAME), Haiku)
 	LDFLAGS=-Wl,--export-dynamic
 endif
 
-$(shell mkdir -p build/core build/mainclient build/webclient build/boot)
-all: $(JANET_TARGET) $(JANET_LIBRARY) $(JANET_STATIC_LIBRARY)
+$(shell mkdir -p build/core build/c build/boot)
+all: $(JANET_TARGET) $(JANET_LIBRARY) $(JANET_STATIC_LIBRARY) build/janet.h
 
 ######################
 ##### Name Files #####
@@ -149,7 +149,7 @@ build/janet_boot: $(JANET_BOOT_OBJECTS)
 	$(CC) $(BOOT_CFLAGS) -o $@ $(JANET_BOOT_OBJECTS) $(CLIBS)
 
 # Now the reason we bootstrap in the first place
-build/janet.c: build/janet_boot src/boot/boot.janet
+build/c/janet.c: build/janet_boot src/boot/boot.janet
 	build/janet_boot . JANET_PATH '$(JANET_PATH)' > $@
 	cksum $@
 
@@ -157,22 +157,22 @@ build/janet.c: build/janet_boot src/boot/boot.janet
 ##### Amalgamation #####
 ########################
 
-SONAME=libjanet.so.1.12
+SONAME=libjanet.so.1.15
 
-build/shell.c: src/mainclient/shell.c
+build/c/shell.c: src/mainclient/shell.c
 	cp $< $@
 
-build/janet.h: src/include/janet.h
-	cp $< $@
+build/janet.h: $(JANET_TARGET) src/include/janet.h src/conf/janetconf.h
+	./$(JANET_TARGET) tools/patch-header.janet src/include/janet.h src/conf/janetconf.h $@
 
 build/janetconf.h: src/conf/janetconf.h
 	cp $< $@
 
-build/janet.o: build/janet.c build/janet.h build/janetconf.h
-	$(HOSTCC) $(BUILD_CFLAGS) -c $< -o $@ -I build
+build/janet.o: build/c/janet.c src/conf/janetconf.h src/include/janet.h
+	$(HOSTCC) $(BUILD_CFLAGS) -c $< -o $@
 
-build/shell.o: build/shell.c build/janet.h build/janetconf.h
-	$(HOSTCC) $(BUILD_CFLAGS) -c $< -o $@ -I build
+build/shell.o: build/c/shell.c src/conf/janetconf.h src/include/janet.h
+	$(HOSTCC) $(BUILD_CFLAGS) -c $< -o $@
 
 $(JANET_TARGET): build/janet.o build/shell.o
 	$(HOSTCC) $(LDFLAGS) $(BUILD_CFLAGS) -o $@ $^ $(CLIBS)
@@ -222,9 +222,9 @@ callgrind: $(JANET_TARGET)
 dist: build/janet-dist.tar.gz
 
 build/janet-%.tar.gz: $(JANET_TARGET) \
-	src/include/janet.h src/conf/janetconf.h \
+	build/janet.h \
 	jpm.1 janet.1 LICENSE CONTRIBUTING.md $(JANET_LIBRARY) $(JANET_STATIC_LIBRARY) \
-	build/doc.html README.md build/janet.c build/shell.c jpm
+	build/doc.html README.md build/c/janet.c build/c/shell.c jpm
 	$(eval JANET_DIST_DIR = "janet-$(shell basename $*)")
 	mkdir -p build/$(JANET_DIST_DIR)
 	cp -r $^ build/$(JANET_DIST_DIR)/
@@ -262,11 +262,11 @@ build/janet.pc: $(JANET_TARGET)
 	echo 'Libs: -L$${libdir} -ljanet' >> $@
 	echo 'Libs.private: $(CLIBS)' >> $@
 
-install: $(JANET_TARGET) $(JANET_LIBRARY) $(JANET_STATIC_LIBRARY) build/janet.pc build/jpm
+install: $(JANET_TARGET) $(JANET_LIBRARY) $(JANET_STATIC_LIBRARY) build/janet.pc build/jpm build/janet.h
 	mkdir -p '$(DESTDIR)$(BINDIR)'
 	cp $(JANET_TARGET) '$(DESTDIR)$(BINDIR)/janet'
 	mkdir -p '$(DESTDIR)$(INCLUDEDIR)/janet'
-	cp -rf $(JANET_HEADERS) '$(DESTDIR)$(INCLUDEDIR)/janet'
+	cp -r build/janet.h '$(DESTDIR)$(INCLUDEDIR)/janet'
 	mkdir -p '$(DESTDIR)$(JANET_PATH)'
 	mkdir -p '$(DESTDIR)$(LIBDIR)'
 	cp $(JANET_LIBRARY) '$(DESTDIR)$(LIBDIR)/libjanet.so.$(shell $(JANET_TARGET) -e '(print janet/version)')'

README.md

@@ -4,8 +4,6 @@
 [![Build Status](https://travis-ci.org/janet-lang/janet.svg?branch=master)](https://travis-ci.org/janet-lang/janet)
 [![builds.sr.ht status](https://builds.sr.ht/~bakpakin/janet/commits/freebsd.yml.svg)](https://builds.sr.ht/~bakpakin/janet/commits/freebsd.yml?)
 [![builds.sr.ht status](https://builds.sr.ht/~bakpakin/janet/commits/openbsd.yml.svg)](https://builds.sr.ht/~bakpakin/janet/commits/openbsd.yml?)
-[![builds.sr.ht status](https://builds.sr.ht/~bakpakin/janet/commits/meson.yml.svg)](https://builds.sr.ht/~bakpakin/janet/commits/meson.yml?)
-[![builds.sr.ht status](https://builds.sr.ht/~bakpakin/janet/commits/meson_min.yml.svg)](https://builds.sr.ht/~bakpakin/janet/commits/meson_min.yml?)
 
 <img src="https://raw.githubusercontent.com/janet-lang/janet/master/assets/janet-w200.png" alt="Janet logo" width=200 align="left">
 
@@ -60,9 +58,9 @@ Documentation is also available locally in the REPL.
 Use the `(doc symbol-name)` macro to get API
 documentation for symbols in the core library. For example,
 ```
-(doc doc)
+(doc apply)
 ```
-Shows documentation for the doc macro.
+Shows documentation for the `apply` function.
 
 To get a list of all bindings in the default
 environment, use the `(all-bindings)` function. You

appveyor.yml

@@ -35,9 +35,6 @@ artifacts:
     - name: janet.h
       path: dist\janet.h
       type: File
-    - name: janetconf.h
-      path: dist\janetconf.h
-      type: File
     - name: shell.c
       path: dist\shell.c
       type: File

build_win.bat

@@ -28,10 +28,10 @@ if not "%JANET_BUILD%" == "" (
     @set JANET_COMPILE=%JANET_COMPILE% /DJANET_BUILD="\"%JANET_BUILD%\""
 )
 
-mkdir build
-mkdir build\core
-mkdir build\mainclient
-mkdir build\boot
+if not exist build mkdir build
+if not exist build\core mkdir build\core
+if not exist build\c mkdir build\c
+if not exist build\boot mkdir build\boot
 
 @rem Build the bootstrap interpreter
 for %%f in (src\core\*.c) do (
@@ -44,10 +44,10 @@ for %%f in (src\boot\*.c) do (
 )
 %JANET_LINK% /out:build\janet_boot.exe build\boot\*.obj
 @if errorlevel 1 goto :BUILDFAIL
-build\janet_boot . > build\janet.c
+build\janet_boot . > build\c\janet.c
 
 @rem Build the sources
-%JANET_COMPILE% /Fobuild\janet.obj build\janet.c
+%JANET_COMPILE% /Fobuild\janet.obj build\c\janet.c
 @if errorlevel 1 goto :BUILDFAIL
 %JANET_COMPILE% /Fobuild\shell.obj src\mainclient\shell.c
 @if errorlevel 1 goto :BUILDFAIL
@@ -102,9 +102,9 @@ exit /b 0
 mkdir dist
 janet.exe tools\gendoc.janet > dist\doc.html
 janet.exe tools\removecr.janet dist\doc.html
-janet.exe tools\removecr.janet build\janet.c
+janet.exe tools\removecr.janet build\c\janet.c
 
-copy build\janet.c dist\janet.c
+copy build\c\janet.c dist\janet.c
 copy src\mainclient\shell.c dist\shell.c
 copy janet.exe dist\janet.exe
 copy LICENSE dist\LICENSE
@@ -113,12 +113,11 @@ copy README.md dist\README.md
 copy janet.lib dist\janet.lib
 copy janet.exp dist\janet.exp
 
-copy src\include\janet.h dist\janet.h
-copy src\conf\janetconf.h dist\janetconf.h
+janet.exe tools\patch-header.janet src\include\janet.h src\conf\janetconf.h build\janet.h
+copy build\janet.h dist\janet.h
 copy build\libjanet.lib dist\libjanet.lib
 
 copy .\jpm dist\jpm
-copy tools\jpm.bat dist\jpm.bat
 
 @rem Create installer
 janet.exe -e "(->> janet/version (peg/match ''(* :d+ `.` :d+ `.` :d+)) first print)" > build\version.txt

examples/async-execute.janet

@@ -0,0 +1,22 @@
+(defn dowork [name n]
+  (print name " starting work...")
+  (os/execute [(dyn :executable) "-e" (string "(os/sleep " n ")")])
+  (print name " finished work!"))
+
+# Will be done in parallel
+(print "starting group A")
+(ev/call dowork "A 2" 2)
+(ev/call dowork "A 1" 1)
+(ev/call dowork "A 3" 3)
+
+(ev/sleep 4)
+
+# Will also be done in parallel
+(print "starting group B")
+(ev/call dowork "B 2" 2)
+(ev/call dowork "B 1" 1)
+(ev/call dowork "B 3" 3)
+
+(ev/sleep 4)
+
+(print "all work done")

examples/iterate-fiber.janet

@@ -0,0 +1,19 @@
+(def f
+  (coro
+    (for i 0 10
+      (yield (string "yield " i))
+      (os/sleep 0))))
+
+(print "simple yielding")
+(each item f (print "got: " item ", now " (fiber/status f)))
+
+(def f
+  (coro
+    (for i 0 10
+      (yield (string "yield " i))
+      (ev/sleep 0))))
+
+(print "complex yielding")
+(each item f (print "got: " item ", now " (fiber/status f)))
+
+(print (fiber/status f))

examples/udpserver.janet

@@ -1,4 +1,4 @@
-(def server (net/server "127.0.0.1" "8009" nil :datagram))
+(def server (net/listen "127.0.0.1" "8009" :datagram))
 (while true
   (def buf @"")
   (def who (:recv-from server 1024 buf))

janet.1

@@ -64,6 +64,10 @@ Move cursor to the beginning of input line.
 .BR Ctrl\-B
 Move cursor one character to the left.
 
+.TP 16
+.BR Ctrl\-D
+If on a newline, indicate end of stream and exit the repl.
+
 .TP 16
 .BR Ctrl\-E
 Move cursor to the end of input line.
@@ -100,6 +104,10 @@ Delete one word before the cursor.
 .BR Ctrl\-G
 Show documentation for the current symbol under the cursor.
 
+.TP 16
+.BR Ctrl\-Q
+Clear the current command, including already typed lines.
+
 .TP 16
 .BR Alt\-B/Alt\-F
 Move cursor backwards and forwards one word.
@@ -167,6 +175,10 @@ Disable ANSI colors in the repl. Has no effect if no repl is run.
 Open a REPL (Read Eval Print Loop) after executing all sources. By default, if Janet is called with no
 arguments, a REPL is opened.
 
+.TP
+.BR \-R
+If using the REPL, disable loading the user profile from the JANET_PROFILE environment variable.
+
 .TP
 .BR \-p
 Turn on the persistent flag. By default, when Janet is executing commands from a file and encounters an error,
@@ -213,6 +225,12 @@ find native and source code modules. If no JANET_PATH is set, Janet will look in
 the default location set at compile time.
 .RE
 
+.B JANET_PROFILE
+.RS
+Path to a profile file that the interpreter will load before entering the REPL. This profile file will
+not run for scripts, though. This behavior can be disabled with the -R option.
+.RE
+
 .B JANET_HASHSEED
 .RS
 To disable randomization of Janet's PRF on start up, one can set this variable. This can have the

jpm

@@ -22,6 +22,19 @@
 ###START###
 
 # Overriden on some installs.
+# To configure this script, replace the code between
+# the START and END comments and define a function 
+# (install-paths) that gives the the default paths
+# to use. Trailing directory separator not expected.
+#
+# Example.
+# 
+#   (defn- install-paths []
+#     {:headerpath "/usr/local/include/janet"
+#      :libpath "/usr/local/lib/janet"
+#      :binpath "/usr/local/bin"
+#
+
 (def- exe-dir
   "Directory containing jpm script"
   (do
@@ -29,9 +42,13 @@
     (def i (last (string/find-all sep exe)))
     (slice exe 0 i)))
 
+(defn- try-real [path]
+  "If os/realpath fails just use normal path."
+  (try (os/realpath) ([_] path)))
+
 (defn- install-paths []
-  {:headerpath (os/realpath (string exe-dir "/../include/janet"))
-   :libpath (os/realpath (string exe-dir "/../lib"))
+  {:headerpath (try-real (string exe-dir "/../include/janet"))
+   :libpath (try-real (string exe-dir "/../lib"))
    :binpath exe-dir})
 
 ###END###
@@ -585,7 +602,7 @@
   (string (string/slice path 0 (- -1 (length modext))) statext))
 
 (defn- make-bin-source
-  [declarations lookup-into-invocations]
+  [declarations lookup-into-invocations no-core]
   (string
     declarations
     ```
@@ -610,15 +627,22 @@ int main(int argc, const char **argv) {
 
     janet_init();
 
+    ```
+    (if no-core
+    ```
+    /* Get core env */
+    JanetTable *env = janet_table(8);
+    JanetTable *lookup = janet_core_lookup_table(NULL);
+    JanetTable *temptab;
+    int handle = janet_gclock();
+    ```
+    ```
     /* Get core env */
     JanetTable *env = janet_core_env(NULL);
     JanetTable *lookup = janet_env_lookup(env);
     JanetTable *temptab;
     int handle = janet_gclock();
-
-    /* Load natives into unmarshalling dictionary */
-
-    ```
+    ```)
     lookup-into-invocations
     ```
     /* Unmarshal bytecode */
@@ -646,7 +670,6 @@ int main(int argc, const char **argv) {
     }
 
     /* Create enviornment */
-    temptab = janet_table(0);
     temptab = env;
     janet_table_put(temptab, janet_ckeywordv("args"), janet_wrap_array(args));
     janet_gcroot(janet_wrap_table(temptab));
@@ -657,6 +680,14 @@ int main(int argc, const char **argv) {
     /* Run everything */
     JanetFiber *fiber = janet_fiber(jfunc, 64, argc, argc ? args->data : NULL);
     fiber->env = temptab;
+#ifdef JANET_EV
+    janet_gcroot(janet_wrap_fiber(fiber));
+    janet_schedule(fiber, janet_wrap_nil());
+    janet_loop();
+    int status = janet_fiber_status(fiber);
+    janet_deinit();
+    return status;
+#else
     Janet out;
     JanetSignal result = janet_continue(fiber, janet_wrap_nil(), &out);
     if (result != JANET_SIGNAL_OK && result != JANET_SIGNAL_EVENT) {
@@ -664,11 +695,9 @@ int main(int argc, const char **argv) {
       janet_deinit();
       return result;
     }
-#ifdef JANET_NET
-    janet_loop();
-#endif
     janet_deinit();
     return 0;
+#endif
 }
 
 ```))
@@ -677,7 +706,7 @@ int main(int argc, const char **argv) {
   "Links an image with libjanet.a (or .lib) to produce an
   executable. Also will try to link native modules into the
   final executable as well."
-  [opts source dest]
+  [opts source dest no-core]
 
   # Create executable's janet image
   (def cimage_dest (string dest ".c"))
@@ -693,7 +722,16 @@ int main(int argc, const char **argv) {
         (def dep-ldflags @[])
 
         # Create marshalling dictionary
-        (def mdict (invert (env-lookup root-env)))
+        (def mdict1 (invert (env-lookup root-env)))
+        (def mdict
+          (if no-core
+            (let [temp @{}]
+              (eachp [k v] mdict1
+                (if (or (cfunction? k) (abstract? k))
+                  (put temp k v)))
+              temp)
+            mdict1))
+
         # Load all native modules
         (def prefixes @{})
         (def static-libs @[])
@@ -738,7 +776,7 @@ int main(int argc, const char **argv) {
         # Make image byte buffer
         (create-buffer-c-impl image cimage_dest "janet_payload_image")
         # Append main function
-        (spit cimage_dest (make-bin-source declarations lookup-into-invocations) :ab)
+        (spit cimage_dest (make-bin-source declarations lookup-into-invocations no-core) :ab)
         (def oimage_dest (out-path cimage_dest ".c" ".o"))
         # Compile and link final exectable
         (unless no-compile
@@ -1027,10 +1065,10 @@ int main(int argc, const char **argv) {
   This executable can be installed as well to the --binpath given."
   [&keys {:install install :name name :entry entry :headers headers
           :cflags cflags :lflags lflags :deps deps :ldflags ldflags
-          :no-compile no-compile}]
+          :no-compile no-compile :no-core no-core}]
   (def name (if is-win (string name ".exe") name))
   (def dest (string "build" sep name))
-  (create-executable @{:cflags cflags :lflags lflags :ldflags ldflags :no-compile no-compile} entry dest)
+  (create-executable @{:cflags cflags :lflags lflags :ldflags ldflags :no-compile no-compile} entry dest no-core)
   (if no-compile
     (let [cdest (string dest ".c")]
       (add-dep "build" cdest))
@@ -1337,7 +1375,9 @@ Flags are:
 
 (defn quickbin
   [input output]
-  (create-executable @{} input output)
+  (if (= (os/stat output :mode) :file)
+    (print "output " output " exists."))
+  (create-executable @{:no-compile (dyn :no-compile)} input output (dyn :no-core))
   (do-rule output))
 
 (defn jpm-debug-repl

meson.build

@@ -20,7 +20,7 @@
 
 project('janet', 'c',
   default_options : ['c_std=c99', 'b_lundef=false', 'default_library=both'],
-  version : '1.13.0')
+  version : '1.15.0')
 
 # Global settings
 janet_path = join_paths(get_option('prefix'), get_option('libdir'), 'janet')
@@ -73,6 +73,7 @@ conf.set('JANET_NO_UMASK', not get_option('umask'))
 conf.set('JANET_NO_REALPATH', not get_option('realpath'))
 conf.set('JANET_NO_PROCESSES', not get_option('processes'))
 conf.set('JANET_SIMPLE_GETLINE', get_option('simple_getline'))
+conf.set('JANET_EV_EPOLL', get_option('epoll'))
 if get_option('os_name') != ''
   conf.set('JANET_OS_NAME', get_option('os_name'))
 endif
@@ -247,8 +248,14 @@ pkg.generate(libjanet,
 
 # Installation
 install_man('janet.1')
-install_headers(['src/include/janet.h', jconf], subdir: 'janet')
 install_data(sources : ['tools/.keep'], install_dir : join_paths(get_option('libdir'), 'janet'))
+patched_janet = custom_target('patched-janeth',
+  input : ['tools/patch-header.janet', 'src/include/janet.h', jconf],
+  install : true,
+  install_dir : join_paths(get_option('includedir'), 'janet'),
+  build_by_default : true,
+  output : ['janet.h'],
+  command : [janet_nativeclient, '@INPUT@', '@OUTPUT@'])
 if get_option('peg') and not get_option('reduced_os') and get_option('processes')
   install_man('jpm.1')
   patched_jpm = custom_target('patched-jpm',

meson_options.txt

@@ -17,6 +17,7 @@ option('processes', type : 'boolean', value : true)
 option('umask', type : 'boolean', value : true)
 option('realpath', type : 'boolean', value : true)
 option('simple_getline', type : 'boolean', value : false)
+option('epoll', type : 'boolean', value : false)
 
 option('recursion_guard', type : 'integer', min : 10, max : 8000, value : 1024)
 option('max_proto_depth', type : 'integer', min : 10, max : 8000, value : 200)

src/conf/janetconf.h

@@ -1,36 +1,13 @@
-/*
-* Copyright (c) 2020 Calvin Rose
-*
-* Permission is hereby granted, free of charge, to any person obtaining a copy
-* of this software and associated documentation files (the "Software"), to
-* deal in the Software without restriction, including without limitation the
-* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-* sell copies of the Software, and to permit persons to whom the Software is
-* furnished to do so, subject to the following conditions:
-*
-* The above copyright notice and this permission notice shall be included in
-* all copies or substantial portions of the Software.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-* IN THE SOFTWARE.
-*/
-
-/* This is an example janetconf.h file. This will be usually generated
- * by the build system. */
+/* This will be generated by the build system if this file is not used */
 
 #ifndef JANETCONF_H
 #define JANETCONF_H
 
 #define JANET_VERSION_MAJOR 1
-#define JANET_VERSION_MINOR 13
+#define JANET_VERSION_MINOR 15
 #define JANET_VERSION_PATCH 0
-#define JANET_VERSION_EXTRA "-dev"
-#define JANET_VERSION "1.13.0-dev"
+#define JANET_VERSION_EXTRA ""
+#define JANET_VERSION "1.15.0"
 
 /* #define JANET_BUILD "local" */
 

src/core/array.c

@@ -290,6 +290,13 @@ static Janet cfun_array_trim(int32_t argc, Janet *argv) {
     return argv[0];
 }
 
+static Janet cfun_array_clear(int32_t argc, Janet *argv) {
+    janet_fixarity(argc, 1);
+    JanetArray *array = janet_getarray(argv, 0);
+    array->count = 0;
+    return argv[0];
+}
+
 static const JanetReg array_cfuns[] = {
     {
         "array/new", cfun_array_new,
@@ -370,6 +377,12 @@ static const JanetReg array_cfuns[] = {
         JDOC("(array/trim arr)\n\n"
              "Set the backing capacity of an array to its current length. Returns the modified array.")
     },
+    {
+        "array/clear", cfun_array_clear,
+        JDOC("(array/clear arr)\n\n"
+             "Empties an array, setting it's count to 0 but does not free the backing capacity. "
+             "Returns the modified array.")
+    },
     {NULL, NULL, NULL}
 };
 

src/core/asm.c

@@ -992,18 +992,18 @@ static const JanetReg asm_cfuns[] = {
              "func must be a function, not a c function. Will throw on error on a badly\n"
              "typed argument. If given a field name, will only return that part of the function assembly.\n"
              "Possible fields are:\n\n"
-             "\t:arity - number of required and optional arguments.\n"
-             "\t:min-arity - minimum number of arguments function can be called with.\n"
-             "\t:max-arity - maximum number of arguments function can be called with.\n"
-             "\t:vararg - true if function can take a variable number of arguments.\n"
-             "\t:bytecode - array of parsed bytecode instructions. Each instruction is a tuple.\n"
-             "\t:source - name of source file that this function was compiled from.\n"
-             "\t:name - name of function.\n"
-             "\t:slotcount - how many virtual registers, or slots, this function uses. Corresponds to stack space used by function.\n"
-             "\t:constants - an array of constants referenced by this function.\n"
-             "\t:sourcemap - a mapping of each bytecode instruction to a line and column in the source file.\n"
-             "\t:environments - an internal mapping of which enclosing functions are referenced for bindings.\n"
-             "\t:defs - other function definitions that this function may instantiate.\n")
+             "* :arity - number of required and optional arguments.\n\n"
+             "* :min-arity - minimum number of arguments function can be called with.\n\n"
+             "* :max-arity - maximum number of arguments function can be called with.\n\n"
+             "* :vararg - true if function can take a variable number of arguments.\n\n"
+             "* :bytecode - array of parsed bytecode instructions. Each instruction is a tuple.\n\n"
+             "* :source - name of source file that this function was compiled from.\n\n"
+             "* :name - name of function.\n\n"
+             "* :slotcount - how many virtual registers, or slots, this function uses. Corresponds to stack space used by function.\n\n"
+             "* :constants - an array of constants referenced by this function.\n\n"
+             "* :sourcemap - a mapping of each bytecode instruction to a line and column in the source file.\n\n"
+             "* :environments - an internal mapping of which enclosing functions are referenced for bindings.\n\n"
+             "* :defs - other function definitions that this function may instantiate.\n")
     },
     {NULL, NULL, NULL}
 };

src/core/buffer.c

@@ -31,12 +31,11 @@
 /* Initialize a buffer */
 JanetBuffer *janet_buffer_init(JanetBuffer *buffer, int32_t capacity) {
     uint8_t *data = NULL;
-    if (capacity > 0) {
-        janet_gcpressure(capacity);
-        data = malloc(sizeof(uint8_t) * (size_t) capacity);
-        if (NULL == data) {
-            JANET_OUT_OF_MEMORY;
-        }
+    if (capacity < 4) capacity = 4;
+    janet_gcpressure(capacity);
+    data = malloc(sizeof(uint8_t) * (size_t) capacity);
+    if (NULL == data) {
+        JANET_OUT_OF_MEMORY;
     }
     buffer->count = 0;
     buffer->capacity = capacity;
@@ -92,7 +91,7 @@ void janet_buffer_extra(JanetBuffer *buffer, int32_t n) {
     }
     int32_t new_size = buffer->count + n;
     if (new_size > buffer->capacity) {
-        int32_t new_capacity = new_size * 2;
+        int32_t new_capacity = (new_size > (INT32_MAX / 2)) ? INT32_MAX : (new_size * 2);
         uint8_t *new_data = realloc(buffer->data, new_capacity * sizeof(uint8_t));
         janet_gcpressure(new_capacity - buffer->capacity);
         if (NULL == new_data) {
@@ -200,19 +199,14 @@ static Janet cfun_buffer_fill(int32_t argc, Janet *argv) {
 static Janet cfun_buffer_trim(int32_t argc, Janet *argv) {
     janet_fixarity(argc, 1);
     JanetBuffer *buffer = janet_getbuffer(argv, 0);
-    if (buffer->count) {
-        if (buffer->count < buffer->capacity) {
-            uint8_t *newData = realloc(buffer->data, buffer->count);
-            if (NULL == newData) {
-                JANET_OUT_OF_MEMORY;
-            }
-            buffer->data = newData;
-            buffer->capacity = buffer->count;
+    if (buffer->count < buffer->capacity) {
+        int32_t newcap = buffer->count > 4 ? buffer->count : 4;
+        uint8_t *newData = realloc(buffer->data, newcap);
+        if (NULL == newData) {
+            JANET_OUT_OF_MEMORY;
         }
-    } else {
-        buffer->capacity = 0;
-        free(buffer->data);
-        buffer->data = NULL;
+        buffer->data = newData;
+        buffer->capacity = newcap;
     }
     return argv[0];
 }
@@ -256,6 +250,26 @@ static Janet cfun_buffer_chars(int32_t argc, Janet *argv) {
     return argv[0];
 }
 
+static Janet cfun_buffer_push(int32_t argc, Janet *argv) {
+    int32_t i;
+    janet_arity(argc, 1, -1);
+    JanetBuffer *buffer = janet_getbuffer(argv, 0);
+    for (i = 1; i < argc; i++) {
+        if (janet_checktype(argv[i], JANET_NUMBER)) {
+            janet_buffer_push_u8(buffer, (uint8_t)(janet_getinteger(argv, i) & 0xFF));
+        } else {
+            JanetByteView view = janet_getbytes(argv, i);
+            if (view.bytes == buffer->data) {
+                janet_buffer_ensure(buffer, buffer->count + view.len, 2);
+                view.bytes = buffer->data;
+            }
+            janet_buffer_push_bytes(buffer, view.bytes, view.len);
+        }
+    }
+    return argv[0];
+}
+
+
 static Janet cfun_buffer_clear(int32_t argc, Janet *argv) {
     janet_fixarity(argc, 1);
     JanetBuffer *buffer = janet_getbuffer(argv, 0);
@@ -407,22 +421,32 @@ static const JanetReg buffer_cfuns[] = {
     },
     {
         "buffer/push-byte", cfun_buffer_u8,
-        JDOC("(buffer/push-byte buffer x)\n\n"
-             "Append a byte to a buffer. Will expand the buffer as necessary. "
+        JDOC("(buffer/push-byte buffer & xs)\n\n"
+             "Append bytes to a buffer. Will expand the buffer as necessary. "
              "Returns the modified buffer. Will throw an error if the buffer overflows.")
     },
     {
         "buffer/push-word", cfun_buffer_word,
-        JDOC("(buffer/push-word buffer x)\n\n"
-             "Append a machine word to a buffer. The 4 bytes of the integer are appended "
-             "in twos complement, little endian order, unsigned. Returns the modified buffer. Will "
+        JDOC("(buffer/push-word buffer & xs)\n\n"
+             "Append machine words to a buffer. The 4 bytes of the integer are appended "
+             "in twos complement, little endian order, unsigned for all x. Returns the modified buffer. Will "
              "throw an error if the buffer overflows.")
     },
     {
         "buffer/push-string", cfun_buffer_chars,
-        JDOC("(buffer/push-string buffer str)\n\n"
-             "Push a string onto the end of a buffer. Non string values will be converted "
-             "to strings before being pushed. Returns the modified buffer. "
+        JDOC("(buffer/push-string buffer & xs)\n\n"
+             "Push byte sequences onto the end of a buffer. "
+             "Will accept any of strings, keywords, symbols, and buffers. "
+             "Returns the modified buffer. "
+             "Will throw an error if the buffer overflows.")
+    },
+    {
+        "buffer/push", cfun_buffer_push,
+        JDOC("(buffer/push buffer & xs)\n\n"
+             "Push both individual bytes and byte sequences to a buffer. For each x in xs, "
+             "push the byte if x is an integer, otherwise push the bytesequence to the buffer. "
+             "Thus, this function behaves like both `buffer/push-string` and `buffer/push-byte`. "
+             "Returns the modified buffer. "
              "Will throw an error if the buffer overflows.")
     },
     {

src/core/capi.c

@@ -53,7 +53,9 @@ JANET_NO_RETURN static void janet_top_level_signal(const char *msg) {
 void janet_signalv(JanetSignal sig, Janet message) {
     if (janet_vm_return_reg != NULL) {
         *janet_vm_return_reg = message;
-        janet_vm_fiber->flags |= JANET_FIBER_DID_LONGJUMP;
+        if (NULL != janet_vm_fiber) {
+            janet_vm_fiber->flags |= JANET_FIBER_DID_LONGJUMP;
+        }
 #if defined(JANET_BSD) || defined(JANET_APPLE)
         _longjmp(*janet_vm_jmp_buf, sig);
 #else
@@ -147,6 +149,23 @@ int janet_getmethod(const uint8_t *method, const JanetMethod *methods, Janet *ou
     return 0;
 }
 
+Janet janet_nextmethod(const JanetMethod *methods, Janet key) {
+    if (!janet_checktype(key, JANET_NIL)) {
+        while (methods->name) {
+            if (janet_keyeq(key, methods->name)) {
+                methods++;
+                break;
+            }
+            methods++;
+        }
+    }
+    if (methods->name) {
+        return janet_ckeywordv(methods->name);
+    } else {
+        return janet_wrap_nil();
+    }
+}
+
 DEFINE_GETTER(number, NUMBER, double)
 DEFINE_GETTER(array, ARRAY, JanetArray *)
 DEFINE_GETTER(tuple, TUPLE, const Janet *)

src/core/compile.c

@@ -439,22 +439,22 @@ static JanetSlot janetc_call(JanetFopts opts, JanetSlot *slots, JanetSlot fun) {
                         min_arity = -1 - min_arity;
                         if (min_arity > max && max >= 0) {
                             const uint8_t *es = janet_formatc(
-                                                    "%v expects at most %d argument, got at least %d",
-                                                    fun.constant, max, min_arity);
+                                                    "%v expects at most %d argument%s, got at least %d",
+                                                    fun.constant, max, max == 1 ? "" : "s", min_arity);
                             janetc_error(c, es);
                         }
                     } else {
                         /* Call has no splices */
                         if (min_arity > max && max >= 0) {
                             const uint8_t *es = janet_formatc(
-                                                    "%v expects at most %d argument, got %d",
-                                                    fun.constant, max, min_arity);
+                                                    "%v expects at most %d argument%s, got %d",
+                                                    fun.constant, max, max == 1 ? "" : "s", min_arity);
                             janetc_error(c, es);
                         }
                         if (min_arity < min) {
                             const uint8_t *es = janet_formatc(
-                                                    "%v expects at least %d argument, got %d",
-                                                    fun.constant, min, min_arity);
+                                                    "%v expects at least %d argument%s, got %d",
+                                                    fun.constant, min, min == 1 ? "" : "s", min_arity);
                             janetc_error(c, es);
                         }
                     }
@@ -872,8 +872,12 @@ static Janet cfun(int32_t argc, Janet *argv) {
     } else {
         JanetTable *t = janet_table(4);
         janet_table_put(t, janet_ckeywordv("error"), janet_wrap_string(res.error));
-        janet_table_put(t, janet_ckeywordv("line"), janet_wrap_integer(res.error_mapping.line));
-        janet_table_put(t, janet_ckeywordv("column"), janet_wrap_integer(res.error_mapping.column));
+        if (res.error_mapping.line > 0) {
+            janet_table_put(t, janet_ckeywordv("line"), janet_wrap_integer(res.error_mapping.line));
+        }
+        if (res.error_mapping.column > 0) {
+            janet_table_put(t, janet_ckeywordv("column"), janet_wrap_integer(res.error_mapping.column));
+        }
         if (res.macrofiber) {
             janet_table_put(t, janet_ckeywordv("fiber"), janet_wrap_fiber(res.macrofiber));
         }

src/core/corelib.c

@@ -548,35 +548,35 @@ static const JanetReg corelib_cfuns[] = {
     {
         "describe", janet_core_describe,
         JDOC("(describe x)\n\n"
-             "Returns a string that is a human readable description of a value x.")
+             "Returns a string that is a human-readable description of a value x.")
     },
     {
         "string", janet_core_string,
-        JDOC("(string & parts)\n\n"
-             "Creates a string by concatenating values together. Values are "
-             "converted to bytes via describe if they are not byte sequences. "
+        JDOC("(string & xs)\n\n"
+             "Creates a string by concatenating the elements of `xs` together. If an "
+             "element is not a byte sequence, it is converted to bytes via `describe`. "
              "Returns the new string.")
     },
     {
         "symbol", janet_core_symbol,
         JDOC("(symbol & xs)\n\n"
-             "Creates a symbol by concatenating values together. Values are "
-             "converted to bytes via describe if they are not byte sequences. Returns "
-             "the new symbol.")
+             "Creates a symbol by concatenating the elements of `xs` together. If an "
+             "element is not a byte sequence, it is converted to bytes via `describe`. "
+             "Returns the new symbol.")
     },
     {
         "keyword", janet_core_keyword,
         JDOC("(keyword & xs)\n\n"
-             "Creates a keyword by concatenating values together. Values are "
-             "converted to bytes via describe if they are not byte sequences. Returns "
-             "the new keyword.")
+             "Creates a keyword by concatenating the elements of `xs` together. If an "
+             "element is not a byte sequence, it is converted to bytes via `describe`. "
+             "Returns the new keyword.")
     },
     {
         "buffer", janet_core_buffer,
         JDOC("(buffer & xs)\n\n"
-             "Creates a new buffer by concatenating values together. Values are "
-             "converted to bytes via describe if they are not byte sequences. Returns "
-             "the new buffer.")
+             "Creates a buffer by concatenating the elements of `xs` together. If an "
+             "element is not a byte sequence, it is converted to bytes via `describe`. "
+             "Returns the new buffer.")
     },
     {
         "abstract?", janet_core_is_abstract,
@@ -645,20 +645,21 @@ static const JanetReg corelib_cfuns[] = {
     {
         "type", janet_core_type,
         JDOC("(type x)\n\n"
-             "Returns the type of x as a keyword. x is one of\n"
-             "\t:nil\n"
-             "\t:boolean\n"
-             "\t:number\n"
-             "\t:array\n"
-             "\t:tuple\n"
-             "\t:table\n"
-             "\t:struct\n"
-             "\t:string\n"
-             "\t:buffer\n"
-             "\t:symbol\n"
-             "\t:keyword\n"
-             "\t:function\n"
-             "\t:cfunction\n\n"
+             "Returns the type of `x` as a keyword. `x` is one of:\n\n"
+             "* :nil\n\n"
+             "* :boolean\n\n"
+             "* :number\n\n"
+             "* :array\n\n"
+             "* :tuple\n\n"
+             "* :table\n\n"
+             "* :struct\n\n"
+             "* :string\n\n"
+             "* :buffer\n\n"
+             "* :symbol\n\n"
+             "* :keyword\n\n"
+             "* :function\n\n"
+             "* :cfunction\n\n"
+             "* :fiber\n\n"
              "or another keyword for an abstract type.")
     },
     {
@@ -699,16 +700,16 @@ static const JanetReg corelib_cfuns[] = {
     {
         "module/expand-path", janet_core_expand_path,
         JDOC("(module/expand-path path template)\n\n"
-             "Expands a path template as found in module/paths for module/find. "
-             "This takes in a path (the argument to require) and a template string, template, "
+             "Expands a path template as found in `module/paths` for `module/find`. "
+             "This takes in a path (the argument to require) and a template string, "
              "to expand the path to a path that can be "
              "used for importing files. The replacements are as follows:\n\n"
-             "\t:all:\tthe value of path verbatim\n"
-             "\t:cur:\tthe current file, or (dyn :current-file)\n"
-             "\t:dir:\tthe directory containing the current file\n"
-             "\t:name:\tthe name component of path, with extension if given\n"
-             "\t:native:\tthe extension used to load natives, .so or .dll\n"
-             "\t:sys:\tthe system path, or (dyn :syspath)")
+             "* :all: -- the value of path verbatim\n\n"
+             "* :cur: -- the current file, or (dyn :current-file)\n\n"
+             "* :dir: -- the directory containing the current file\n\n"
+             "* :name: -- the name component of path, with extension if given\n\n"
+             "* :native: -- the extension used to load natives, .so or .dll\n\n"
+             "* :sys: -- the system path, or (dyn :syspath)")
     },
     {
         "int?", janet_core_check_int,
@@ -1205,7 +1206,8 @@ JanetTable *janet_core_env(JanetTable *replacements) {
                    "if native modules are compatible with the host program."));
 
     /* Allow references to the environment */
-    janet_def(env, "_env", janet_wrap_table(env), JDOC("The environment table for the current scope."));
+    janet_def(env, "root-env", janet_wrap_table(env),
+              JDOC("The root environment used to create environments with (make-env)."));
 
     janet_load_libs(env);
     janet_gcroot(janet_wrap_table(env));
@@ -1220,7 +1222,42 @@ JanetTable *janet_core_env(JanetTable *replacements) {
         return janet_vm_core_env;
     }
 
-    /* Load core cfunctions (and some built in janet assembly functions) */
+    JanetTable *dict = janet_core_lookup_table(replacements);
+
+    /* Unmarshal bytecode */
+    Janet marsh_out = janet_unmarshal(
+                          janet_core_image,
+                          janet_core_image_size,
+                          0,
+                          dict,
+                          NULL);
+
+    /* Memoize */
+    janet_gcroot(marsh_out);
+    JanetTable *env = janet_unwrap_table(marsh_out);
+    janet_vm_core_env = env;
+
+    /* Invert image dict manually here. We can't do this in boot.janet as it
+     * breaks deterministic builds */
+    Janet lidv, midv;
+    lidv = midv = janet_wrap_nil();
+    janet_resolve(env, janet_csymbol("load-image-dict"), &lidv);
+    janet_resolve(env, janet_csymbol("make-image-dict"), &midv);
+    JanetTable *lid = janet_unwrap_table(lidv);
+    JanetTable *mid = janet_unwrap_table(midv);
+    for (int32_t i = 0; i < lid->capacity; i++) {
+        const JanetKV *kv = lid->data + i;
+        if (!janet_checktype(kv->key, JANET_NIL)) {
+            janet_table_put(mid, kv->value, kv->key);
+        }
+    }
+
+    return env;
+}
+
+#endif
+
+JanetTable *janet_core_lookup_table(JanetTable *replacements) {
     JanetTable *dict = janet_table(512);
     janet_load_libs(dict);
 
@@ -1237,20 +1274,5 @@ JanetTable *janet_core_env(JanetTable *replacements) {
         }
     }
 
-    /* Unmarshal bytecode */
-    Janet marsh_out = janet_unmarshal(
-                          janet_core_image,
-                          janet_core_image_size,
-                          0,
-                          dict,
-                          NULL);
-
-    /* Memoize */
-    janet_gcroot(marsh_out);
-    JanetTable *env = janet_unwrap_table(marsh_out);
-    janet_vm_core_env = env;
-
-    return env;
+    return dict;
 }
-
-#endif

src/core/debug.c

@@ -102,7 +102,9 @@ void janet_stacktrace(JanetFiber *fiber, Janet err) {
     int32_t fi;
     const char *errstr = (const char *)janet_to_string(err);
     JanetFiber **fibers = NULL;
-    int wrote_error = 0;
+
+    /* Don't print error line if it is nil. */
+    int wrote_error = janet_checktype(err, JANET_NIL);
 
     int print_color = janet_truthy(janet_dyn("err-color"));
     if (print_color) janet_eprintf("\x1b[31m");
@@ -299,9 +301,10 @@ static Janet cfun_debug_stack(int32_t argc, Janet *argv) {
 }
 
 static Janet cfun_debug_stacktrace(int32_t argc, Janet *argv) {
-    janet_fixarity(argc, 2);
+    janet_arity(argc, 1, 2);
     JanetFiber *fiber = janet_getfiber(argv, 0);
-    janet_stacktrace(fiber, argv[1]);
+    Janet x = argc == 1 ? janet_wrap_nil() : argv[1];
+    janet_stacktrace(fiber, x);
     return argv[0];
 }
 
@@ -325,12 +328,12 @@ static Janet cfun_debug_step(int32_t argc, Janet *argv) {
 static const JanetReg debug_cfuns[] = {
     {
         "debug/break", cfun_debug_break,
-        JDOC("(debug/break source byte-offset)\n\n"
-             "Sets a breakpoint with source a key at a given line and column. "
+        JDOC("(debug/break source line col)\n\n"
+             "Sets a breakpoint in `source` at a given line and column. "
              "Will throw an error if the breakpoint location "
              "cannot be found. For example\n\n"
-             "\t(debug/break \"core.janet\" 1000)\n\n"
-             "wil set a breakpoint at the 1000th byte of the file core.janet.")
+             "\t(debug/break \"core.janet\" 10 4)\n\n"
+             "wil set a breakpoint at line 10, 4th column of the file core.janet.")
     },
     {
         "debug/unbreak", cfun_debug_unbreak,
@@ -362,25 +365,25 @@ static const JanetReg debug_cfuns[] = {
         "debug/stack", cfun_debug_stack,
         JDOC("(debug/stack fib)\n\n"
              "Gets information about the stack as an array of tables. Each table "
-             "in the array contains information about a stack frame. The top most, current "
-             "stack frame is the first table in the array, and the bottom most stack frame "
+             "in the array contains information about a stack frame. The top-most, current "
+             "stack frame is the first table in the array, and the bottom-most stack frame "
              "is the last value. Each stack frame contains some of the following attributes:\n\n"
-             "\t:c - true if the stack frame is a c function invocation\n"
-             "\t:column - the current source column of the stack frame\n"
-             "\t:function - the function that the stack frame represents\n"
-             "\t:line - the current source line of the stack frame\n"
-             "\t:name - the human friendly name of the function\n"
-             "\t:pc - integer indicating the location of the program counter\n"
-             "\t:source - string with the file path or other identifier for the source code\n"
-             "\t:slots - array of all values in each slot\n"
-             "\t:tail - boolean indicating a tail call")
+             "* :c - true if the stack frame is a c function invocation\n\n"
+             "* :column - the current source column of the stack frame\n\n"
+             "* :function - the function that the stack frame represents\n\n"
+             "* :line - the current source line of the stack frame\n\n"
+             "* :name - the human-friendly name of the function\n\n"
+             "* :pc - integer indicating the location of the program counter\n\n"
+             "* :source - string with the file path or other identifier for the source code\n\n"
+             "* :slots - array of all values in each slot\n\n"
+             "* :tail - boolean indicating a tail call")
     },
     {
         "debug/stacktrace", cfun_debug_stacktrace,
-        JDOC("(debug/stacktrace fiber err)\n\n"
-             "Prints a nice looking stacktrace for a fiber. The error message "
-             "err must be passed to the function as fiber's do not keep track of "
-             "the last error they have thrown. Returns the fiber.")
+        JDOC("(debug/stacktrace fiber &opt err)\n\n"
+             "Prints a nice looking stacktrace for a fiber. Can optionally provide "
+             "an error value to print the stack trace with. If `err` is nil or not "
+             "provided, will skipp the error line. Returns the fiber.")
     },
     {
         "debug/lineage", cfun_debug_lineage,

src/core/features.h

@@ -27,7 +27,7 @@
 
 #if defined(__NetBSD__) || defined(__APPLE__) || defined(__OpenBSD__) \
     || defined(__bsdi__) || defined(__DragonFly__)
-/* Use BSD soucre on any BSD systems, include OSX */
+/* Use BSD source on any BSD systems, include OSX */
 # define _BSD_SOURCE
 #else
 /* Use POSIX feature flags */

src/core/fiber.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -37,9 +37,11 @@ static void fiber_reset(JanetFiber *fiber) {
     fiber->child = NULL;
     fiber->flags = JANET_FIBER_MASK_YIELD | JANET_FIBER_RESUME_NO_USEVAL | JANET_FIBER_RESUME_NO_SKIP;
     fiber->env = NULL;
+    fiber->last_value = janet_wrap_nil();
 #ifdef JANET_EV
     fiber->waiting = NULL;
     fiber->sched_id = 0;
+    fiber->supervisor_channel = NULL;
 #endif
     janet_fiber_set_status(fiber, JANET_STATUS_NEW);
 }
@@ -81,7 +83,10 @@ JanetFiber *janet_fiber_reset(JanetFiber *fiber, JanetFunction *callee, int32_t
     }
     if (janet_fiber_funcframe(fiber, callee)) return NULL;
     janet_fiber_frame(fiber)->flags |= JANET_STACKFRAME_ENTRANCE;
+#ifdef JANET_EV
     fiber->waiting = NULL;
+    fiber->supervisor_channel = NULL;
+#endif
     return fiber;
 }
 
@@ -418,8 +423,7 @@ void janet_fiber_cframe(JanetFiber *fiber, JanetCFunction cfun) {
     newframe->flags = 0;
 }
 
-/* Pop a stack frame from the fiber. Returns the new stack frame, or
- * NULL if there are no more frames */
+/* Pop a stack frame from the fiber. */
 void janet_fiber_popframe(JanetFiber *fiber) {
     JanetStackFrame *frame = janet_fiber_frame(fiber);
     if (fiber->frame == 0) return;
@@ -585,6 +589,12 @@ static Janet cfun_fiber_can_resume(int32_t argc, Janet *argv) {
     return janet_wrap_boolean(!isFinished);
 }
 
+static Janet cfun_fiber_last_value(int32_t argc, Janet *argv) {
+    janet_fixarity(argc, 1);
+    JanetFiber *fiber = janet_getfiber(argv, 0);
+    return fiber->last_value;
+}
+
 static const JanetReg fiber_cfuns[] = {
     {
         "fiber/new", cfun_fiber_new,
@@ -592,34 +602,37 @@ static const JanetReg fiber_cfuns[] = {
              "Create a new fiber with function body func. Can optionally "
              "take a set of signals to block from the current parent fiber "
              "when called. The mask is specified as a keyword where each character "
-             "is used to indicate a signal to block. The default sigmask is :y. "
-             "For example, \n\n"
-             "\t(fiber/new myfun :e123)\n\n"
+             "is used to indicate a signal to block. If the ev module is enabled, and "
+             "this fiber is used as an argument to `ev/go`, these \"blocked\" signals "
+             "will result in messages being sent to the supervisor channel. "
+             "The default sigmask is :y. "
+             "For example,\n\n"
+             "    (fiber/new myfun :e123)\n\n"
              "blocks error signals and user signals 1, 2 and 3. The signals are "
-             "as follows: \n\n"
-             "\ta - block all signals\n"
-             "\td - block debug signals\n"
-             "\te - block error signals\n"
-             "\tt - block termination signals: error + user[0-4]\n"
-             "\tu - block user signals\n"
-             "\ty - block yield signals\n"
-             "\t0-9 - block a specific user signal\n\n"
+             "as follows:\n\n"
+             "* :a - block all signals\n"
+             "* :d - block debug signals\n"
+             "* :e - block error signals\n"
+             "* :t - block termination signals: error + user[0-4]\n"
+             "* :u - block user signals\n"
+             "* :y - block yield signals\n"
+             "* :0-9 - block a specific user signal\n\n"
              "The sigmask argument also can take environment flags. If any mutually "
              "exclusive flags are present, the last flag takes precedence.\n\n"
-             "\ti - inherit the environment from the current fiber\n"
-             "\tp - the environment table's prototype is the current environment table")
+             "* :i - inherit the environment from the current fiber\n"
+             "* :p - the environment table's prototype is the current environment table")
     },
     {
         "fiber/status", cfun_fiber_status,
         JDOC("(fiber/status fib)\n\n"
              "Get the status of a fiber. The status will be one of:\n\n"
-             "\t:dead - the fiber has finished\n"
-             "\t:error - the fiber has errored out\n"
-             "\t:debug - the fiber is suspended in debug mode\n"
-             "\t:pending - the fiber has been yielded\n"
-             "\t:user(0-9) - the fiber is suspended by a user signal\n"
-             "\t:alive - the fiber is currently running and cannot be resumed\n"
-             "\t:new - the fiber has just been created and not yet run")
+             "* :dead - the fiber has finished\n"
+             "* :error - the fiber has errored out\n"
+             "* :debug - the fiber is suspended in debug mode\n"
+             "* :pending - the fiber has been yielded\n"
+             "* :user(0-9) - the fiber is suspended by a user signal\n"
+             "* :alive - the fiber is currently running and cannot be resumed\n"
+             "* :new - the fiber has just been created and not yet run")
     },
     {
         "fiber/root", cfun_fiber_root,
@@ -662,6 +675,11 @@ static const JanetReg fiber_cfuns[] = {
         JDOC("(fiber/can-resume? fiber)\n\n"
              "Check if a fiber is finished and cannot be resumed.")
     },
+    {
+        "fiber/last-value", cfun_fiber_last_value,
+        JDOC("(fiber/last-value\n\n"
+             "Get the last value returned or signaled from the fiber.")
+    },
     {NULL, NULL, NULL}
 };
 

src/core/gc.c

@@ -226,11 +226,14 @@ static void janet_mark_function(JanetFunction *func) {
     if (janet_gc_reachable(func))
         return;
     janet_gc_mark(func);
-    numenvs = func->def->environments_length;
-    for (i = 0; i < numenvs; ++i) {
-        janet_mark_funcenv(func->envs[i]);
+    if (NULL != func->def) {
+        /* this should always be true, except if function is only partially constructed */
+        numenvs = func->def->environments_length;
+        for (i = 0; i < numenvs; ++i) {
+            janet_mark_funcenv(func->envs[i]);
+        }
+        janet_mark_funcdef(func->def);
     }
-    janet_mark_funcdef(func->def);
 }
 
 static void janet_mark_fiber(JanetFiber *fiber) {
@@ -241,6 +244,8 @@ recur:
         return;
     janet_gc_mark(fiber);
 
+    janet_mark(fiber->last_value);
+
     /* Mark values on the argument stack */
     janet_mark_many(fiber->data + fiber->stackstart,
                     fiber->stacktop - fiber->stackstart);
@@ -262,6 +267,12 @@ recur:
     if (fiber->env)
         janet_mark_table(fiber->env);
 
+#ifdef JANET_EV
+    if (fiber->supervisor_channel) {
+        janet_mark_abstract(fiber->supervisor_channel);
+    }
+#endif
+
     /* Explicit tail recursion */
     if (fiber->child) {
         fiber = fiber->child;
@@ -404,6 +415,7 @@ void janet_collect(void) {
 #ifdef JANET_EV
     janet_ev_mark();
 #endif
+    janet_mark_fiber(janet_vm_root_fiber);
     for (i = 0; i < orig_rootcount; i++)
         janet_mark(janet_vm_roots[i]);
     while (orig_rootcount < janet_vm_root_count) {

src/core/inttypes.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose & contributors
+* Copyright (c) 2021 Calvin Rose & contributors
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -39,6 +39,8 @@
 
 static int it_s64_get(void *p, Janet key, Janet *out);
 static int it_u64_get(void *p, Janet key, Janet *out);
+static Janet janet_int64_next(void *p, Janet key);
+static Janet janet_uint64_next(void *p, Janet key);
 
 static int32_t janet_int64_hash(void *p1, size_t size) {
     (void) size;
@@ -92,7 +94,8 @@ const JanetAbstractType janet_s64_type = {
     it_s64_tostring,
     janet_int64_compare,
     janet_int64_hash,
-    JANET_ATEND_HASH
+    janet_int64_next,
+    JANET_ATEND_NEXT
 };
 
 const JanetAbstractType janet_u64_type = {
@@ -106,7 +109,8 @@ const JanetAbstractType janet_u64_type = {
     it_u64_tostring,
     janet_uint64_compare,
     janet_int64_hash,
-    JANET_ATEND_HASH
+    janet_uint64_next,
+    JANET_ATEND_NEXT
 };
 
 int64_t janet_unwrap_s64(Janet x) {
@@ -134,7 +138,7 @@ int64_t janet_unwrap_s64(Janet x) {
             break;
         }
     }
-    janet_panic("bad s64 initializer");
+    janet_panicf("bad s64 initializer: %t", x);
     return 0;
 }
 
@@ -144,7 +148,9 @@ uint64_t janet_unwrap_u64(Janet x) {
             break;
         case JANET_NUMBER : {
             double dbl = janet_unwrap_number(x);
-            if ((dbl >= 0) && (dbl <= MAX_INT_IN_DBL))
+            /* Allow negative values to be cast to "wrap around".
+             * This let's addition and subtraction work as expected. */
+            if (fabs(dbl) <=  MAX_INT_IN_DBL)
                 return (uint64_t)dbl;
             break;
         }
@@ -163,7 +169,7 @@ uint64_t janet_unwrap_u64(Janet x) {
             break;
         }
     }
-    janet_panic("bad u64 initializer");
+    janet_panicf("bad u64 initializer: %t", x);
     return 0;
 }
 
@@ -197,15 +203,14 @@ static Janet cfun_it_u64_new(int32_t argc, Janet *argv) {
     return janet_wrap_u64(janet_unwrap_u64(argv[0]));
 }
 
-// Code to support polymorphic comparison.
-//
-// int/u64 and int/s64 support a "compare" method that allows
-// comparison to each other, and to Janet numbers, using the
-// "compare" "compare<" ... functions.
-//
-// In the following code explicit casts are sometimes used to help
-// make it clear when int/float conversions are happening.
-//
+/*
+ * Code to support polymorphic comparison.
+ * int/u64 and int/s64 support a "compare" method that allows
+ * comparison to each other, and to Janet numbers, using the
+ * "compare" "compare<" ... functions.
+ * In the following code explicit casts are sometimes used to help
+ * make it clear when int/float conversions are happening.
+ */
 static int compare_double_double(double x, double y) {
     return (x < y) ? -1 : ((x > y) ? 1 : 0);
 }
@@ -242,7 +247,6 @@ static int compare_uint64_double(uint64_t x, double y) {
     }
 }
 
-
 static Janet cfun_it_s64_compare(int32_t argc, Janet *argv) {
     janet_fixarity(argc, 2);
     if (janet_is_int(argv[0]) != JANET_INT_S64)
@@ -383,31 +387,14 @@ static Janet cfun_it_##type##_##name(int32_t argc, Janet *argv) { \
 } \
 
 static Janet cfun_it_s64_mod(int32_t argc, Janet *argv) {
-    janet_arity(argc, 2, -1);
-    int64_t *box = janet_abstract(&janet_s64_type, sizeof(int64_t));
-    *box = janet_unwrap_s64(argv[0]);
-    for (int32_t i = 1; i < argc; i++) {
-        int64_t value = janet_unwrap_s64(argv[i]);
-        if (value == 0) janet_panic("division by zero");
-        int64_t x = *box % value;
-        if (x < 0) {
-            x = (*box < 0) ? x - *box : x + *box;
-        }
-        *box = x;
-    }
-    return janet_wrap_abstract(box);
-}
-
-static Janet cfun_it_s64_modi(int32_t argc, Janet *argv) {
     janet_fixarity(argc, 2);
     int64_t *box = janet_abstract(&janet_s64_type, sizeof(int64_t));
     int64_t op1 = janet_unwrap_s64(argv[0]);
     int64_t op2 = janet_unwrap_s64(argv[1]);
     int64_t x = op1 % op2;
-    if (x < 0) {
-        x = (op1 < 0) ? x - op1 : x + op1;
-    }
-    *box = x;
+    *box = (op1 > 0)
+           ? ((op2 > 0) ? x : (0 == x ? x : x + op2))
+           : ((op2 > 0) ? (0 == x ? x : x + op2) : x);
     return janet_wrap_abstract(box);
 }
 
@@ -418,7 +405,6 @@ OPMETHOD(int64_t, s64, mul, *)
 DIVMETHOD_SIGNED(int64_t, s64, div, /)
 DIVMETHOD_SIGNED(int64_t, s64, rem, %)
 DIVMETHODINVERT_SIGNED(int64_t, s64, divi, /)
-DIVMETHODINVERT_SIGNED(int64_t, s64, remi, %)
 OPMETHOD(int64_t, s64, and, &)
 OPMETHOD(int64_t, s64, or, |)
 OPMETHOD(int64_t, s64, xor, ^)
@@ -431,7 +417,6 @@ OPMETHOD(uint64_t, u64, mul, *)
 DIVMETHOD(uint64_t, u64, div, /)
 DIVMETHOD(uint64_t, u64, mod, %)
 DIVMETHODINVERT(uint64_t, u64, divi, /)
-DIVMETHODINVERT(uint64_t, u64, modi, %)
 OPMETHOD(uint64_t, u64, and, &)
 OPMETHOD(uint64_t, u64, or, |)
 OPMETHOD(uint64_t, u64, xor, ^)
@@ -454,9 +439,9 @@ static JanetMethod it_s64_methods[] = {
     {"/", cfun_it_s64_div},
     {"r/", cfun_it_s64_divi},
     {"mod", cfun_it_s64_mod},
-    {"rmod", cfun_it_s64_modi},
+    {"rmod", cfun_it_s64_mod},
     {"%", cfun_it_s64_rem},
-    {"r%", cfun_it_s64_remi},
+    {"r%", cfun_it_s64_rem},
     {"&", cfun_it_s64_and},
     {"r&", cfun_it_s64_and},
     {"|", cfun_it_s64_or},
@@ -480,9 +465,9 @@ static JanetMethod it_u64_methods[] = {
     {"/", cfun_it_u64_div},
     {"r/", cfun_it_u64_divi},
     {"mod", cfun_it_u64_mod},
-    {"rmod", cfun_it_u64_modi},
+    {"rmod", cfun_it_u64_mod},
     {"%", cfun_it_u64_mod},
-    {"r%", cfun_it_u64_modi},
+    {"r%", cfun_it_u64_mod},
     {"&", cfun_it_u64_and},
     {"r&", cfun_it_u64_and},
     {"|", cfun_it_u64_or},
@@ -496,6 +481,16 @@ static JanetMethod it_u64_methods[] = {
     {NULL, NULL}
 };
 
+static Janet janet_int64_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(it_s64_methods, key);
+}
+
+static Janet janet_uint64_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(it_u64_methods, key);
+}
+
 static int it_s64_get(void *p, Janet key, Janet *out) {
     (void) p;
     if (!janet_checktype(key, JANET_KEYWORD))

src/core/io.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -39,6 +39,7 @@ static int cfun_io_gc(void *p, size_t len);
 static int io_file_get(void *p, Janet key, Janet *out);
 static void io_file_marshal(void *p, JanetMarshalContext *ctx);
 static void *io_file_unmarshal(JanetMarshalContext *ctx);
+static Janet io_file_next(void *p, Janet key);
 
 const JanetAbstractType janet_file_type = {
     "core/file",
@@ -48,7 +49,11 @@ const JanetAbstractType janet_file_type = {
     NULL,
     io_file_marshal,
     io_file_unmarshal,
-    JANET_ATEND_UNMARSHAL
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    io_file_next,
+    JANET_ATEND_NEXT
 };
 
 /* Check arguments to fopen */
@@ -259,20 +264,29 @@ static Janet cfun_io_fflush(int32_t argc, Janet *argv) {
 #define WEXITSTATUS(x) x
 #endif
 
+/* For closing files from C API */
+int janet_file_close(JanetFile *file) {
+    int ret = 0;
+    if (!(file->flags & (JANET_FILE_NOT_CLOSEABLE | JANET_FILE_CLOSED))) {
+#ifndef JANET_NO_PROCESSES
+        if (file->flags & JANET_FILE_PIPED) {
+            ret = pclose(file->file);
+        } else
+#endif
+        {
+            ret = fclose(file->file);
+        }
+        file->flags |= JANET_FILE_CLOSED;
+        return ret;
+    }
+    return 0;
+}
+
 /* Cleanup a file */
 static int cfun_io_gc(void *p, size_t len) {
     (void) len;
     JanetFile *iof = (JanetFile *)p;
-    if (!(iof->flags & (JANET_FILE_NOT_CLOSEABLE | JANET_FILE_CLOSED))) {
-        /* We can't panic inside a gc, so just ignore bad statuses here */
-        if (iof->flags & JANET_FILE_PIPED) {
-#ifndef JANET_NO_PROCESSES
-            pclose(iof->file);
-#endif
-        } else {
-            fclose(iof->file);
-        }
-    }
+    janet_file_close(iof);
     return 0;
 }
 
@@ -346,6 +360,11 @@ static int io_file_get(void *p, Janet key, Janet *out) {
     return janet_getmethod(janet_unwrap_keyword(key), io_file_methods, out);
 }
 
+static Janet io_file_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(io_file_methods, key);
+}
+
 static void io_file_marshal(void *p, JanetMarshalContext *ctx) {
     JanetFile *iof = (JanetFile *)p;
     if (ctx->flags & JANET_MARSHAL_UNSAFE) {
@@ -713,23 +732,24 @@ static const JanetReg io_cfuns[] = {
     {
         "file/temp", cfun_io_temp,
         JDOC("(file/temp)\n\n"
-             "Open an anonymous temporary file that is removed on close."
+             "Open an anonymous temporary file that is removed on close. "
              "Raises an error on failure.")
     },
     {
         "file/open", cfun_io_fopen,
         JDOC("(file/open path &opt mode)\n\n"
-             "Open a file. path is an absolute or relative path, and "
-             "mode is a set of flags indicating the mode to open the file in. "
-             "mode is a keyword where each character represents a flag. If the file "
+             "Open a file. `path` is an absolute or relative path, and "
+             "`mode` is a set of flags indicating the mode to open the file in. "
+             "`mode` is a keyword where each character represents a flag. If the file "
              "cannot be opened, returns nil, otherwise returns the new file handle. "
              "Mode flags:\n\n"
-             "\tr - allow reading from the file\n"
-             "\tw - allow writing to the file\n"
-             "\ta - append to the file\n"
-             "\tb - open the file in binary mode (rather than text mode)\n"
-             "\t+ - append to the file instead of overwriting it\n"
-             "\tn - error if the file cannot be opened instead of returning nil")
+             "* r - allow reading from the file\n\n"
+             "* w - allow writing to the file\n\n"
+             "* a - append to the file\n\n"
+             "Following one fo the initial flags, 0 or more of the following flags can be appended:\n\n"
+             "* b - open the file in binary mode (rather than text mode)\n\n"
+             "* + - append to the file instead of overwriting it\n\n"
+             "* n - error if the file cannot be opened instead of returning nil")
     },
     {
         "file/close", cfun_io_fclose,
@@ -742,14 +762,14 @@ static const JanetReg io_cfuns[] = {
     {
         "file/read", cfun_io_fread,
         JDOC("(file/read f what &opt buf)\n\n"
-             "Read a number of bytes from a file into a buffer. A buffer can "
-             "be provided as an optional fourth argument, otherwise a new buffer "
-             "is created. 'what' can either be an integer or a keyword. Returns the "
+             "Read a number of bytes from a file `f` into a buffer. A buffer `buf` can "
+             "be provided as an optional third argument, otherwise a new buffer "
+             "is created. `what` can either be an integer or a keyword. Returns the "
              "buffer with file contents. "
-             "Values for 'what':\n\n"
-             "\t:all - read the whole file\n"
-             "\t:line - read up to and including the next newline character\n"
-             "\tn (integer) - read up to n bytes from the file")
+             "Values for `what`:\n\n"
+             "* :all - read the whole file\n\n"
+             "* :line - read up to and including the next newline character\n\n"
+             "* n (integer) - read up to n bytes from the file")
     },
     {
         "file/write", cfun_io_fwrite,
@@ -766,18 +786,18 @@ static const JanetReg io_cfuns[] = {
     {
         "file/seek", cfun_io_fseek,
         JDOC("(file/seek f &opt whence n)\n\n"
-             "Jump to a relative location in the file. 'whence' must be one of\n\n"
-             "\t:cur - jump relative to the current file location\n"
-             "\t:set - jump relative to the beginning of the file\n"
-             "\t:end - jump relative to the end of the file\n\n"
-             "By default, 'whence' is :cur. Optionally a value n may be passed "
-             "for the relative number of bytes to seek in the file. n may be a real "
-             "number to handle large files of more the 4GB. Returns the file handle.")
+             "Jump to a relative location in the file `f`. `whence` must be one of:\n\n"
+             "* :cur - jump relative to the current file location\n\n"
+             "* :set - jump relative to the beginning of the file\n\n"
+             "* :end - jump relative to the end of the file\n\n"
+             "By default, `whence` is :cur. Optionally a value `n` may be passed "
+             "for the relative number of bytes to seek in the file. `n` may be a real "
+             "number to handle large files of more than 4GB. Returns the file handle.")
     },
 #ifndef JANET_NO_PROCESSES
     {
         "file/popen", cfun_io_popen,
-        JDOC("(file/popen command &opt mode)\n\n"
+        JDOC("(file/popen command &opt mode) (DEPRECATED for os/spawn)\n\n"
              "Open a file that is backed by a process. The file must be opened in either "
              "the :r (read) or the :w (write) mode. In :r mode, the stdout of the "
              "process can be read from the file. In :w mode, the stdin of the process "

src/core/marsh.c

@@ -286,7 +286,7 @@ static void marshal_one_def(MarshalState *st, JanetFuncDef *def, int flags) {
 
 #define JANET_FIBER_FLAG_HASCHILD (1 << 29)
 #define JANET_FIBER_FLAG_HASENV   (1 << 30)
-#define JANET_STACKFRAME_HASENV   (1 << 31)
+#define JANET_STACKFRAME_HASENV   (INT32_MIN)
 
 /* Marshal a fiber */
 static void marshal_one_fiber(MarshalState *st, JanetFiber *fiber, int flags) {
@@ -542,9 +542,10 @@ static void marshal_one(MarshalState *st, Janet x, int flags) {
         case JANET_FUNCTION: {
             pushbyte(st, LB_FUNCTION);
             JanetFunction *func = janet_unwrap_function(x);
-            marshal_one_def(st, func->def, flags);
-            /* Mark seen after reading def, but before envs */
+            /* Mark seen before reading def */
             MARK_SEEN();
+            pushint(st, func->def->environments_length);
+            marshal_one_def(st, func->def, flags);
             for (int32_t i = 0; i < func->def->environments_length; i++)
                 marshal_one_env(st, func->envs[i], flags + 1);
             return;
@@ -937,6 +938,7 @@ static const uint8_t *unmarshal_one_fiber(
 #ifdef JANET_EV
     fiber->waiting = NULL;
     fiber->sched_id = 0;
+    fiber->supervisor_channel = NULL;
 #endif
 
     /* Push fiber to seen stack */
@@ -1228,12 +1230,20 @@ static const uint8_t *unmarshal_one(
         case LB_FUNCTION: {
             JanetFunction *func;
             JanetFuncDef *def;
-            data = unmarshal_one_def(st, data + 1, &def, flags + 1);
+            data++;
+            int32_t len = readnat(st, &data);
+            if (len > 255) {
+                janet_panicf("invalid function");
+            }
             func = janet_gcalloc(JANET_MEMORY_FUNCTION, sizeof(JanetFunction) +
-                                 def->environments_length * sizeof(JanetFuncEnv));
-            func->def = def;
+                                 len * sizeof(JanetFuncEnv));
             *out = janet_wrap_function(func);
             janet_v_push(st->lookup, *out);
+            data = unmarshal_one_def(st, data, &def, flags + 1);
+            if (def->environments_length != len) {
+                janet_panicf("invalid function");
+            }
+            func->def = def;
             for (int32_t i = 0; i < def->environments_length; i++) {
                 data = unmarshal_one_env(st, data, &(func->envs[i]), flags + 1);
             }
@@ -1418,9 +1428,9 @@ static const JanetReg marsh_cfuns[] = {
         "marshal", cfun_marshal,
         JDOC("(marshal x &opt reverse-lookup buffer)\n\n"
              "Marshal a value into a buffer and return the buffer. The buffer "
-             "can the later be unmarshalled to reconstruct the initial value. "
+             "can then later be unmarshalled to reconstruct the initial value. "
              "Optionally, one can pass in a reverse lookup table to not marshal "
-             "aliased values that are found in the table. Then a forward"
+             "aliased values that are found in the table. Then a forward "
              "lookup table can be used to recover the original value when "
              "unmarshalling.")
     },

src/core/math.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -31,6 +31,7 @@
 static JANET_THREAD_LOCAL JanetRNG janet_vm_rng = {0, 0, 0, 0, 0};
 
 static int janet_rng_get(void *p, Janet key, Janet *out);
+static Janet janet_rng_next(void *p, Janet key);
 
 static void janet_rng_marshal(void *p, JanetMarshalContext *ctx) {
     JanetRNG *rng = (JanetRNG *)p;
@@ -60,7 +61,11 @@ const JanetAbstractType janet_rng_type = {
     NULL,
     janet_rng_marshal,
     janet_rng_unmarshal,
-    JANET_ATEND_UNMARSHAL
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    janet_rng_next,
+    JANET_ATEND_NEXT
 };
 
 JanetRNG *janet_default_rng(void) {
@@ -203,6 +208,11 @@ static int janet_rng_get(void *p, Janet key, Janet *out) {
     return janet_getmethod(janet_unwrap_keyword(key), rng_methods, out);
 }
 
+static Janet janet_rng_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(rng_methods, key);
+}
+
 /* Get a random number */
 static Janet janet_rand(int32_t argc, Janet *argv) {
     (void) argv;
@@ -500,13 +510,13 @@ void janet_lib_math(JanetTable *env) {
     janet_def(env, "math/-inf", janet_wrap_number(-INFINITY),
               JDOC("The number representing negative infinity"));
     janet_def(env, "math/int32-min", janet_wrap_number(INT32_MIN),
-              JDOC("The maximum contiguous integer representable by a 32 bit signed integer"));
+              JDOC("The minimum contiguous integer representable by a 32 bit signed integer"));
     janet_def(env, "math/int32-max", janet_wrap_number(INT32_MAX),
-              JDOC("The minimum contiguous integer represtenable by a 32 bit signed integer"));
+              JDOC("The maximum contiguous integer represtenable by a 32 bit signed integer"));
     janet_def(env, "math/int-min", janet_wrap_number(JANET_INTMIN_DOUBLE),
-              JDOC("The maximum contiguous integer representable by a double (2^53)"));
+              JDOC("The minimum contiguous integer representable by a double (2^53)"));
     janet_def(env, "math/int-max", janet_wrap_number(JANET_INTMAX_DOUBLE),
-              JDOC("The minimum contiguous integer represtenable by a double (-(2^53))"));
+              JDOC("The maximum contiguous integer represtenable by a double (-(2^53))"));
 #ifdef NAN
     janet_def(env, "math/nan", janet_wrap_number(NAN),
 #else

src/core/net.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose and contributors.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -28,6 +28,7 @@
 
 #ifdef JANET_NET
 
+#include <math.h>
 #ifdef JANET_WINDOWS
 #include <winsock2.h>
 #include <windows.h>
@@ -47,490 +48,57 @@
 #include <netinet/tcp.h>
 #include <netdb.h>
 #include <fcntl.h>
-#include <math.h>
 #endif
 
-/*
- * Streams - simple abstract type that wraps a pollable + extra flags
- */
-
-#define JANET_STREAM_READABLE 0x200
-#define JANET_STREAM_WRITABLE 0x400
-#define JANET_STREAM_ACCEPTABLE 0x800
-#define JANET_STREAM_UDPSERVER 0x1000
-
-static int janet_stream_close(void *p, size_t s);
-static int janet_stream_mark(void *p, size_t s);
-static int janet_stream_getter(void *p, Janet key, Janet *out);
-static const JanetAbstractType StreamAT = {
-    "core/stream",
-    janet_stream_close,
-    janet_stream_mark,
-    janet_stream_getter,
-    JANET_ATEND_GET
-};
-
-typedef JanetPollable JanetStream;
-
-static const JanetAbstractType AddressAT = {
+const JanetAbstractType janet_address_type = {
     "core/socket-address",
     JANET_ATEND_NAME
 };
 
 #ifdef JANET_WINDOWS
-#define JANET_NET_CHUNKSIZE 4096
 #define JSOCKCLOSE(x) closesocket((SOCKET) x)
 #define JSOCKDEFAULT INVALID_SOCKET
-#define JLASTERR WSAGetLastError()
 #define JSOCKVALID(x) ((x) != INVALID_SOCKET)
-#define JEINTR WSAEINTR
-#define JEWOULDBLOCK WSAEWOULDBLOCK
-#define JEAGAIN WSAEWOULDBLOCK
-#define JPOLL WSAPoll
 #define JSock SOCKET
-#define JReadInt long
 #define JSOCKFLAGS 0
-static JanetStream *make_stream(SOCKET fd, uint32_t flags) {
-    u_long iMode = 0;
-    JanetStream *stream = janet_abstract(&StreamAT, sizeof(JanetStream));
-    janet_pollable_init(stream, (JanetHandle) fd);
-    ioctlsocket(fd, FIONBIO, &iMode);
-    stream->flags = flags;
-    return stream;
-}
-static Janet net_lasterr(void) {
-    int code = WSAGetLastError();
-    char msgbuf[256];
-    msgbuf[0] = '\0';
-    FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
-                  NULL,
-                  code,
-                  MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-                  msgbuf,
-                  sizeof(msgbuf),
-                  NULL);
-    if (!*msgbuf) sprintf(msgbuf, "%d", code);
-    char *c = msgbuf;
-    while (*c) {
-        if (*c == '\n' || *c == '\r') {
-            *c = '\0';
-            break;
-        }
-        c++;
-    }
-    return janet_cstringv(msgbuf);
-}
 #else
 #define JSOCKCLOSE(x) close(x)
 #define JSOCKDEFAULT 0
-#define JLASTERR errno
 #define JSOCKVALID(x) ((x) >= 0)
-#define JEINTR EINTR
-#define JEWOULDBLOCK EWOULDBLOCK
-#define JEAGAIN EAGAIN
-#define JPOLL poll
 #define JSock int
-#define JReadInt ssize_t
 #ifdef SOCK_CLOEXEC
 #define JSOCKFLAGS SOCK_CLOEXEC
 #else
 #define JSOCKFLAGS 0
 #endif
-static JanetStream *make_stream(int fd, uint32_t flags) {
-    JanetStream *stream = janet_abstract(&StreamAT, sizeof(JanetStream));
-    janet_pollable_init(stream, fd);
-#if !defined(SOCK_CLOEXEC) && defined(O_CLOEXEC)
-    int extra = O_CLOEXEC;
-#else
-    int extra = 0;
-#endif
-    fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK | extra);
-    stream->flags = flags;
-    return stream;
-}
-static Janet net_lasterr(void) {
-    return janet_cstringv(strerror(errno));
-}
 #endif
 
+static JanetStream *make_stream(JSock handle, uint32_t flags);
+
 /* We pass this flag to all send calls to prevent sigpipe */
 #ifndef MSG_NOSIGNAL
 #define MSG_NOSIGNAL 0
 #endif
 
-static int janet_stream_close(void *p, size_t s) {
-    (void) s;
-    JanetStream *stream = p;
-    if (!(stream->flags & JANET_POLL_FLAG_CLOSED)) {
-        JSOCKCLOSE(stream->handle);
-        janet_pollable_deinit(stream);
-    }
-    return 0;
-}
-
-static void nosigpipe(JSock s) {
+/* Make sure a socket doesn't block */
+static void janet_net_socknoblock(JSock s) {
+#ifdef JANET_WINDOWS
+    unsigned long arg = 1;
+    ioctlsocket(s, FIONBIO, &arg);
+#else
+#if !defined(SOCK_CLOEXEC) && defined(O_CLOEXEC)
+    int extra = O_CLOEXEC;
+#else
+    int extra = 0;
+#endif
+    fcntl(s, F_SETFL, fcntl(s, F_GETFL, 0) | O_NONBLOCK | extra);
 #ifdef SO_NOSIGPIPE
     int enable = 1;
-    if (setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, &enable, sizeof(int)) < 0) {
-        JSOCKCLOSE(s);
-        janet_panic("setsockopt(SO_NOSIGPIPE) failed");
-    }
-#else
-    (void) s;
+    setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, &enable, sizeof(int));
+#endif
 #endif
 }
 
-static int janet_stream_mark(void *p, size_t s) {
-    (void) s;
-    janet_pollable_mark((JanetPollable *) p);
-    return 0;
-}
-
-/*
- * State machine for read
- */
-
-typedef struct {
-    JanetListenerState head;
-    int32_t bytes_left;
-    JanetBuffer *buf;
-    int is_chunk;
-    int is_recv_from;
-#ifdef JANET_WINDOWS
-    WSAOVERLAPPED overlapped;
-    WSABUF wbuf;
-    DWORD flags;
-    int32_t chunk_size;
-    struct sockaddr from;
-    int fromlen;
-    uint8_t chunk_buf[JANET_NET_CHUNKSIZE];
-#endif
-} NetStateRead;
-
-JanetAsyncStatus net_machine_read(JanetListenerState *s, JanetAsyncEvent event) {
-    NetStateRead *state = (NetStateRead *) s;
-    switch (event) {
-        default:
-            break;
-        case JANET_ASYNC_EVENT_MARK:
-            janet_mark(janet_wrap_buffer(state->buf));
-            break;
-        case JANET_ASYNC_EVENT_CLOSE:
-            janet_cancel(s->fiber, janet_cstringv("stream closed"));
-            return JANET_ASYNC_STATUS_DONE;
-#ifdef JANET_WINDOWS
-        case JANET_ASYNC_EVENT_COMPLETE: {
-            /* Called when read finished */
-            if (s->bytes == 0 && !state->is_recv_from) {
-                janet_schedule(s->fiber, janet_wrap_nil());
-                return JANET_ASYNC_STATUS_DONE;
-            }
-
-            janet_buffer_push_bytes(state->buf, state->chunk_buf, s->bytes);
-            state->bytes_left -= s->bytes;
-
-            if (state->bytes_left <= 0 || !state->is_chunk) {
-                Janet resume_val;
-                if (state->is_recv_from) {
-                    void *abst = janet_abstract(&AddressAT, state->fromlen);
-                    memcpy(abst, &state->from, state->fromlen);
-                    resume_val = janet_wrap_abstract(abst);
-                } else {
-                    resume_val = janet_wrap_buffer(state->buf);
-                }
-                janet_schedule(s->fiber, resume_val);
-                return JANET_ASYNC_STATUS_DONE;
-            }
-        }
-
-        /* fallthrough */
-        case JANET_ASYNC_EVENT_USER: {
-            state->flags = 0;
-            int32_t chunk_size = state->bytes_left > JANET_NET_CHUNKSIZE ? JANET_NET_CHUNKSIZE : state->bytes_left;
-            state->wbuf.len = (ULONG) chunk_size;
-            state->wbuf.buf = state->chunk_buf;
-            state->chunk_size = chunk_size;
-            s->tag = &state->overlapped;
-            memset(&(state->overlapped), 0, sizeof(WSAOVERLAPPED));
-            int status;
-            if (state->is_recv_from) {
-                status = WSARecvFrom((SOCKET) s->pollable->handle, &state->wbuf, 1, NULL, &state->flags, &state->from, &state->fromlen, &state->overlapped, NULL);
-            } else {
-                status = WSARecv((SOCKET) s->pollable->handle, &state->wbuf, 1, NULL, &state->flags, &state->overlapped, NULL);
-            }
-            if (status && WSA_IO_PENDING != WSAGetLastError()) {
-                janet_cancel(s->fiber, net_lasterr());
-                return JANET_ASYNC_STATUS_DONE;
-            }
-        }
-        break;
-#else
-        case JANET_ASYNC_EVENT_READ:
-            /* Read in bytes */
-        {
-            JanetBuffer *buffer = state->buf;
-            int32_t bytes_left = state->bytes_left;
-            janet_buffer_extra(buffer, bytes_left);
-            JReadInt nread;
-            char saddr[256];
-            socklen_t socklen = sizeof(saddr);
-            do {
-                if (state->is_recv_from) {
-                    nread = recvfrom(s->pollable->handle, buffer->data + buffer->count, bytes_left, 0,
-                                     (struct sockaddr *)&saddr, &socklen);
-                } else {
-                    nread = recv(s->pollable->handle, buffer->data + buffer->count, bytes_left, 0);
-                }
-            } while (nread == -1 && JLASTERR == JEINTR);
-
-            /* Check for errors - special case errors that can just be waited on to fix */
-            if (nread == -1) {
-                if (JLASTERR == JEAGAIN || JLASTERR == JEWOULDBLOCK) break;
-                janet_cancel(s->fiber, net_lasterr());
-                return JANET_ASYNC_STATUS_DONE;
-            }
-
-            /* Only allow 0-length packets in recv-from. In stream protocols, a zero length packet is EOS. */
-            if (nread == 0 && !state->is_recv_from) {
-                janet_schedule(s->fiber, janet_wrap_nil());
-                return JANET_ASYNC_STATUS_DONE;
-            }
-
-            /* Increment buffer counts */
-            if (nread > 0) {
-                buffer->count += nread;
-                bytes_left -= nread;
-            } else {
-                bytes_left = 0;
-            }
-            state->bytes_left = bytes_left;
-
-            /* Resume if done */
-            if (!state->is_chunk || bytes_left == 0) {
-                Janet resume_val;
-                if (state->is_recv_from) {
-                    void *abst = janet_abstract(&AddressAT, socklen);
-                    memcpy(abst, &saddr, socklen);
-                    resume_val = janet_wrap_abstract(abst);
-                } else {
-                    resume_val = janet_wrap_buffer(buffer);
-                }
-                janet_schedule(s->fiber, resume_val);
-                return JANET_ASYNC_STATUS_DONE;
-            }
-        }
-        break;
-#endif
-    }
-    return JANET_ASYNC_STATUS_NOT_DONE;
-}
-
-JANET_NO_RETURN static void janet_sched_read(JanetStream *stream, JanetBuffer *buf, int32_t nbytes) {
-    NetStateRead *state = (NetStateRead *) janet_listen(stream, net_machine_read,
-                          JANET_ASYNC_LISTEN_READ, sizeof(NetStateRead), NULL);
-    state->is_chunk = 0;
-    state->buf = buf;
-    state->bytes_left = nbytes;
-    state->is_recv_from = 0;
-    net_machine_read((JanetListenerState *) state, JANET_ASYNC_EVENT_USER);
-    janet_await();
-}
-
-JANET_NO_RETURN static void janet_sched_chunk(JanetStream *stream, JanetBuffer *buf, int32_t nbytes) {
-    NetStateRead *state = (NetStateRead *) janet_listen(stream, net_machine_read,
-                          JANET_ASYNC_LISTEN_READ, sizeof(NetStateRead), NULL);
-    state->is_chunk = 1;
-    state->buf = buf;
-    state->bytes_left = nbytes;
-    state->is_recv_from = 0;
-    net_machine_read((JanetListenerState *) state, JANET_ASYNC_EVENT_USER);
-    janet_await();
-}
-
-JANET_NO_RETURN static void janet_sched_recv_from(JanetStream *stream, JanetBuffer *buf, int32_t nbytes) {
-    NetStateRead *state = (NetStateRead *) janet_listen(stream, net_machine_read,
-                          JANET_ASYNC_LISTEN_READ, sizeof(NetStateRead), NULL);
-    state->is_chunk = 0;
-    state->buf = buf;
-    state->bytes_left = nbytes;
-    state->is_recv_from = 1;
-    net_machine_read((JanetListenerState *) state, JANET_ASYNC_EVENT_USER);
-    janet_await();
-}
-
-/*
- * State machine for write/send-to
- */
-
-typedef struct {
-    JanetListenerState head;
-    union {
-        JanetBuffer *buf;
-        const uint8_t *str;
-    } src;
-    int32_t start;
-    int is_buffer;
-    void *dest_abst;
-#ifdef JANET_WINDOWS
-    WSAOVERLAPPED overlapped;
-    WSABUF wbuf;
-    DWORD flags;
-#endif
-} NetStateWrite;
-
-JanetAsyncStatus net_machine_write(JanetListenerState *s, JanetAsyncEvent event) {
-    NetStateWrite *state = (NetStateWrite *) s;
-    switch (event) {
-        default:
-            break;
-        case JANET_ASYNC_EVENT_MARK:
-            janet_mark(state->is_buffer
-                       ? janet_wrap_buffer(state->src.buf)
-                       : janet_wrap_string(state->src.str));
-            if (state->dest_abst != NULL) {
-                janet_mark(janet_wrap_abstract(state->dest_abst));
-            }
-            break;
-        case JANET_ASYNC_EVENT_CLOSE:
-            janet_cancel(s->fiber, janet_cstringv("stream closed"));
-            return JANET_ASYNC_STATUS_DONE;
-#ifdef JANET_WINDOWS
-        case JANET_ASYNC_EVENT_COMPLETE: {
-            /* Called when write finished */
-            if (s->bytes == 0 && !state->dest_abst) {
-                janet_cancel(s->fiber, janet_cstringv("disconnect"));
-                return JANET_ASYNC_STATUS_DONE;
-            }
-
-            janet_schedule(s->fiber, janet_wrap_nil());
-            return JANET_ASYNC_STATUS_DONE;
-        }
-        break;
-        case JANET_ASYNC_EVENT_USER: {
-            /* Begin write */
-            int32_t start, len;
-            const uint8_t *bytes;
-            start = state->start;
-            if (state->is_buffer) {
-                /* If buffer, convert to string. */
-                /* TODO - be more efficient about this */
-                JanetBuffer *buffer = state->src.buf;
-                JanetString str = janet_string(buffer->data, buffer->count);
-                bytes = str;
-                len = buffer->count;
-                state->is_buffer = 0;
-                state->src.str = str;
-            } else {
-                bytes = state->src.str;
-                len = janet_string_length(bytes);
-            }
-            state->wbuf.buf = (char *) bytes;
-            state->wbuf.len = len;
-            state->flags = 0;
-            s->tag = &state->overlapped;
-            memset(&(state->overlapped), 0, sizeof(WSAOVERLAPPED));
-
-            int status;
-            SOCKET sock = (SOCKET) s->pollable->handle;
-            if (state->dest_abst) {
-                const struct sockaddr *to = state->dest_abst;
-                int tolen = (int) janet_abstract_size((void *) to);
-                status = WSASendTo(sock, &state->wbuf, 1, NULL, state->flags, to, tolen, &state->overlapped, NULL);
-            } else {
-                status = WSASend(sock, &state->wbuf, 1, NULL, state->flags, &state->overlapped, NULL);
-            }
-
-            if (status && WSA_IO_PENDING != WSAGetLastError()) {
-                janet_cancel(s->fiber, janet_cstringv("failed to write to stream"));
-                return JANET_ASYNC_STATUS_DONE;
-            }
-        }
-        break;
-#else
-        case JANET_ASYNC_EVENT_WRITE: {
-            int32_t start, len;
-            const uint8_t *bytes;
-            start = state->start;
-            if (state->is_buffer) {
-                JanetBuffer *buffer = state->src.buf;
-                bytes = buffer->data;
-                len = buffer->count;
-            } else {
-                bytes = state->src.str;
-                len = janet_string_length(bytes);
-            }
-            JReadInt nwrote = 0;
-            if (start < len) {
-                int32_t nbytes = len - start;
-                void *dest_abst = state->dest_abst;
-                do {
-                    if (dest_abst) {
-                        nwrote = sendto(s->pollable->handle, bytes + start, nbytes, 0,
-                                        (struct sockaddr *) dest_abst, janet_abstract_size(dest_abst));
-                    } else {
-                        nwrote = send(s->pollable->handle, bytes + start, nbytes, MSG_NOSIGNAL);
-                    }
-                } while (nwrote == -1 && JLASTERR == JEINTR);
-
-                /* Handle write errors */
-                if (nwrote == -1) {
-                    if (JLASTERR == JEAGAIN || JLASTERR == JEWOULDBLOCK) break;
-                    janet_cancel(s->fiber, net_lasterr());
-                    return JANET_ASYNC_STATUS_DONE;
-                }
-
-                /* Unless using datagrams, empty message is a disconnect */
-                if (nwrote == 0 && !dest_abst) {
-                    janet_cancel(s->fiber, janet_cstringv("disconnect"));
-                    return JANET_ASYNC_STATUS_DONE;
-                }
-
-                if (nwrote > 0) {
-                    start += nwrote;
-                } else {
-                    start = len;
-                }
-            }
-            state->start = start;
-            if (start >= len) {
-                janet_schedule(s->fiber, janet_wrap_nil());
-                return JANET_ASYNC_STATUS_DONE;
-            }
-            break;
-        }
-        break;
-#endif
-    }
-    return JANET_ASYNC_STATUS_NOT_DONE;
-}
-
-JANET_NO_RETURN static void janet_sched_write_buffer(JanetStream *stream, JanetBuffer *buf, void *dest_abst) {
-    NetStateWrite *state = (NetStateWrite *) janet_listen(stream, net_machine_write,
-                           JANET_ASYNC_LISTEN_WRITE, sizeof(NetStateWrite), NULL);
-    state->is_buffer = 1;
-    state->start = 0;
-    state->src.buf = buf;
-    state->dest_abst = dest_abst;
-    net_machine_write((JanetListenerState *) state, JANET_ASYNC_EVENT_USER);
-    janet_await();
-}
-
-
-JANET_NO_RETURN static void janet_sched_write_stringlike(JanetStream *stream, const uint8_t *str, void *dest_abst) {
-    NetStateWrite *state = (NetStateWrite *) janet_listen(stream, net_machine_write,
-                           JANET_ASYNC_LISTEN_WRITE, sizeof(NetStateWrite), NULL);
-    state->is_buffer = 0;
-    state->start = 0;
-    state->src.str = str;
-    state->dest_abst = dest_abst;
-    net_machine_write((JanetListenerState *) state, JANET_ASYNC_EVENT_USER);
-    janet_await();
-}
-
-/*
- * State machine for simple server
- */
-
 /* State machine for accepting connections. */
 
 #ifdef JANET_WINDOWS
@@ -578,6 +146,7 @@ JanetAsyncStatus net_machine_accept(JanetListenerState *s, JanetAsyncEvent event
             if (state->function) {
                 /* Schedule worker */
                 JanetFiber *fiber = janet_fiber(state->function, 64, 1, &streamv);
+                fiber->supervisor_channel = s->fiber->supervisor_channel;
                 janet_schedule(fiber, janet_wrap_nil());
                 /* Now listen again for next connection */
                 Janet err;
@@ -612,7 +181,7 @@ static int net_sched_accept_impl(NetStateAccept *state, Janet *err) {
     SOCKET lsock = (SOCKET) state->lstream->handle;
     SOCKET asock = WSASocketW(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
     if (asock == INVALID_SOCKET) {
-        *err = net_lasterr();
+        *err = janet_ev_lasterr();
         return 1;
     }
     JanetStream *astream = make_stream(asock, JANET_STREAM_READABLE | JANET_STREAM_WRITABLE);
@@ -621,7 +190,7 @@ static int net_sched_accept_impl(NetStateAccept *state, Janet *err) {
     if (FALSE == AcceptEx(lsock, asock, state->buf, 0, socksize, socksize, NULL, &state->overlapped)) {
         int code = WSAGetLastError();
         if (code == WSA_IO_PENDING) return 0; /* indicates io is happening async */
-        *err = net_lasterr();
+        *err = janet_ev_lasterr();
         return 1;
     }
     return 0;
@@ -647,13 +216,14 @@ JanetAsyncStatus net_machine_accept(JanetListenerState *s, JanetAsyncEvent event
             janet_schedule(s->fiber, janet_wrap_nil());
             return JANET_ASYNC_STATUS_DONE;
         case JANET_ASYNC_EVENT_READ: {
-            JSock connfd = accept(s->pollable->handle, NULL, NULL);
+            JSock connfd = accept(s->stream->handle, NULL, NULL);
             if (JSOCKVALID(connfd)) {
-                nosigpipe(connfd);
+                janet_net_socknoblock(connfd);
                 JanetStream *stream = make_stream(connfd, JANET_STREAM_READABLE | JANET_STREAM_WRITABLE);
                 Janet streamv = janet_wrap_abstract(stream);
                 if (state->function) {
                     JanetFiber *fiber = janet_fiber(state->function, 64, 1, &streamv);
+                    fiber->supervisor_channel = s->fiber->supervisor_channel;
                     janet_schedule(fiber, janet_wrap_nil());
                 } else {
                     janet_schedule(s->fiber, streamv);
@@ -695,12 +265,11 @@ static struct addrinfo *janet_get_addrinfo(Janet *argv, int32_t offset, int sock
 #ifndef JANET_WINDOWS
     if (janet_keyeq(argv[offset], "unix")) {
         const char *path = janet_getcstring(argv, offset + 1);
-        struct sockaddr_un *saddr = malloc(sizeof(struct sockaddr_un));
+        struct sockaddr_un *saddr = calloc(1, sizeof(struct sockaddr_un));
         if (saddr == NULL) {
             JANET_OUT_OF_MEMORY;
         }
         saddr->sun_family = AF_UNIX;
-        memset(&saddr->sun_path, 0, 108);
 #ifdef JANET_LINUX
         if (path[0] == '@') {
             saddr->sun_path[0] = '\0';
@@ -750,7 +319,7 @@ static Janet cfun_net_sockaddr(int32_t argc, Janet *argv) {
 #ifndef JANET_WINDOWS
     /* no unix domain socket support on windows yet */
     if (is_unix) {
-        void *abst = janet_abstract(&AddressAT, sizeof(struct sockaddr_un));
+        void *abst = janet_abstract(&janet_address_type, sizeof(struct sockaddr_un));
         memcpy(abst, ai, sizeof(struct sockaddr_un));
         Janet ret = janet_wrap_abstract(abst);
         return make_arr ? janet_wrap_array(janet_array_n(&ret, 1)) : ret;
@@ -761,7 +330,7 @@ static Janet cfun_net_sockaddr(int32_t argc, Janet *argv) {
         JanetArray *arr = janet_array(10);
         struct addrinfo *iter = ai;
         while (NULL != iter) {
-            void *abst = janet_abstract(&AddressAT, iter->ai_addrlen);
+            void *abst = janet_abstract(&janet_address_type, iter->ai_addrlen);
             memcpy(abst, iter->ai_addr, iter->ai_addrlen);
             janet_array_push(arr, janet_wrap_abstract(abst));
             iter = iter->ai_next;
@@ -773,7 +342,7 @@ static Janet cfun_net_sockaddr(int32_t argc, Janet *argv) {
         if (NULL == ai) {
             janet_panic("no data for given address");
         }
-        void *abst = janet_abstract(&AddressAT, ai->ai_addrlen);
+        void *abst = janet_abstract(&janet_address_type, ai->ai_addrlen);
         memcpy(abst, ai->ai_addr, ai->ai_addrlen);
         freeaddrinfo(ai);
         return janet_wrap_abstract(abst);
@@ -790,12 +359,12 @@ static Janet cfun_net_connect(int32_t argc, Janet *argv) {
     /* Create socket */
     JSock sock = JSOCKDEFAULT;
     void *addr = NULL;
-    socklen_t addrlen;
+    socklen_t addrlen = 0;
 #ifndef JANET_WINDOWS
     if (is_unix) {
         sock = socket(AF_UNIX, socktype | JSOCKFLAGS, 0);
         if (!JSOCKVALID(sock)) {
-            janet_panic("could not create socket");
+            janet_panicf("could not create socket: %V", janet_ev_lasterr());
         }
         addr = (void *) ai;
         addrlen = sizeof(struct sockaddr_un);
@@ -817,24 +386,30 @@ static Janet cfun_net_connect(int32_t argc, Janet *argv) {
         }
         if (NULL == addr) {
             freeaddrinfo(ai);
-            janet_panic("could not create socket");
+            janet_panicf("could not create socket: %V", janet_ev_lasterr());
         }
     }
 
     /* Connect to socket */
+#ifdef JANET_WINDOWS
+    int status = WSAConnect(sock, addr, addrlen, NULL, NULL, NULL, NULL);
+    freeaddrinfo(ai);
+#else
     int status = connect(sock, addr, addrlen);
     if (is_unix) {
         free(ai);
     } else {
         freeaddrinfo(ai);
     }
+#endif
 
     if (status == -1) {
         JSOCKCLOSE(sock);
-        janet_panic("could not connect to socket");
+        janet_panicf("could not connect to socket: %V", janet_ev_lasterr());
     }
 
-    nosigpipe(sock);
+    /* Set up the socket for non-blocking IO after connect - TODO - non-blocking connect? */
+    janet_net_socknoblock(sock);
 
     /* Wrap socket in abstract type JanetStream */
     JanetStream *stream = make_stream(sock, JANET_STREAM_READABLE | JANET_STREAM_WRITABLE);
@@ -852,6 +427,7 @@ static const char *serverify_socket(JSock sfd) {
         return "setsockopt(SO_REUSEPORT) failed";
     }
 #endif
+    janet_net_socknoblock(sfd);
     return NULL;
 }
 
@@ -869,13 +445,17 @@ static Janet cfun_net_listen(int32_t argc, Janet *argv) {
         sfd = socket(AF_UNIX, socktype | JSOCKFLAGS, 0);
         if (!JSOCKVALID(sfd)) {
             free(ai);
-            janet_panic("could not create socket");
+            janet_panicf("could not create socket: %V", janet_ev_lasterr());
         }
         const char *err = serverify_socket(sfd);
         if (NULL != err || bind(sfd, (struct sockaddr *)ai, sizeof(struct sockaddr_un))) {
             JSOCKCLOSE(sfd);
             free(ai);
-            janet_panic(err ? err : "could not bind socket");
+            if (err) {
+                janet_panic(err);
+            } else {
+                janet_panicf("could not bind socket: %V", janet_ev_lasterr());
+            }
         }
         free(ai);
     } else
@@ -905,8 +485,6 @@ static Janet cfun_net_listen(int32_t argc, Janet *argv) {
         }
     }
 
-    nosigpipe(sfd);
-
     if (socktype == SOCK_DGRAM) {
         /* Datagram server (UDP) */
         JanetStream *stream = make_stream(sfd, JANET_STREAM_UDPSERVER | JANET_STREAM_READABLE);
@@ -918,7 +496,7 @@ static Janet cfun_net_listen(int32_t argc, Janet *argv) {
         int status = listen(sfd, 1024);
         if (status) {
             JSOCKCLOSE(sfd);
-            janet_panic("could not listen on file descriptor");
+            janet_panicf("could not listen on file descriptor: %V", janet_ev_lasterr());
         }
 
         /* Put sfd on our loop */
@@ -927,29 +505,18 @@ static Janet cfun_net_listen(int32_t argc, Janet *argv) {
     }
 }
 
-static void check_stream_flag(JanetStream *stream, int flag) {
-    if (!(stream->flags & flag) || (stream->flags & JANET_POLL_FLAG_CLOSED)) {
-        const char *msg = "";
-        if (flag == JANET_STREAM_READABLE) msg = "readable";
-        if (flag == JANET_STREAM_WRITABLE) msg = "writable";
-        if (flag == JANET_STREAM_ACCEPTABLE) msg = "server";
-        if (flag == JANET_STREAM_UDPSERVER) msg = "datagram server";
-        janet_panicf("bad stream, expected %s stream", msg);
-    }
-}
-
 static Janet cfun_stream_accept_loop(int32_t argc, Janet *argv) {
     janet_fixarity(argc, 2);
-    JanetStream *stream = janet_getabstract(argv, 0, &StreamAT);
-    check_stream_flag(stream, JANET_STREAM_ACCEPTABLE);
+    JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type);
+    janet_stream_flags(stream, JANET_STREAM_ACCEPTABLE | JANET_STREAM_SOCKET);
     JanetFunction *fun = janet_getfunction(argv, 1);
     janet_sched_accept(stream, fun);
 }
 
 static Janet cfun_stream_accept(int32_t argc, Janet *argv) {
     janet_arity(argc, 1, 2);
-    JanetStream *stream = janet_getabstract(argv, 0, &StreamAT);
-    check_stream_flag(stream, JANET_STREAM_ACCEPTABLE);
+    JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type);
+    janet_stream_flags(stream, JANET_STREAM_ACCEPTABLE | JANET_STREAM_SOCKET);
     double to = janet_optnumber(argv, argc, 1, INFINITY);
     if (to != INFINITY) janet_addtimeout(to);
     janet_sched_accept(stream, NULL);
@@ -957,85 +524,82 @@ static Janet cfun_stream_accept(int32_t argc, Janet *argv) {
 
 static Janet cfun_stream_read(int32_t argc, Janet *argv) {
     janet_arity(argc, 2, 4);
-    JanetStream *stream = janet_getabstract(argv, 0, &StreamAT);
-    check_stream_flag(stream, JANET_STREAM_READABLE);
-    int32_t n = janet_getnat(argv, 1);
+    JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type);
+    janet_stream_flags(stream, JANET_STREAM_READABLE | JANET_STREAM_SOCKET);
     JanetBuffer *buffer = janet_optbuffer(argv, argc, 2, 10);
     double to = janet_optnumber(argv, argc, 3, INFINITY);
-    if (to != INFINITY) janet_addtimeout(to);
-    janet_sched_read(stream, buffer, n);
+    if (janet_keyeq(argv[1], "all")) {
+        if (to != INFINITY) janet_addtimeout(to);
+        janet_ev_recvchunk(stream, buffer, INT32_MAX, MSG_NOSIGNAL);
+    } else {
+        int32_t n = janet_getnat(argv, 1);
+        if (to != INFINITY) janet_addtimeout(to);
+        janet_ev_recv(stream, buffer, n, MSG_NOSIGNAL);
+    }
+    janet_await();
 }
 
 static Janet cfun_stream_chunk(int32_t argc, Janet *argv) {
     janet_arity(argc, 2, 4);
-    JanetStream *stream = janet_getabstract(argv, 0, &StreamAT);
-    check_stream_flag(stream, JANET_STREAM_READABLE);
+    JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type);
+    janet_stream_flags(stream, JANET_STREAM_READABLE | JANET_STREAM_SOCKET);
     int32_t n = janet_getnat(argv, 1);
     JanetBuffer *buffer = janet_optbuffer(argv, argc, 2, 10);
     double to = janet_optnumber(argv, argc, 3, INFINITY);
     if (to != INFINITY) janet_addtimeout(to);
-    janet_sched_chunk(stream, buffer, n);
+    janet_ev_recvchunk(stream, buffer, n, MSG_NOSIGNAL);
+    janet_await();
 }
 
 static Janet cfun_stream_recv_from(int32_t argc, Janet *argv) {
     janet_arity(argc, 3, 4);
-    JanetStream *stream = janet_getabstract(argv, 0, &StreamAT);
-    check_stream_flag(stream, JANET_STREAM_UDPSERVER);
+    JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type);
+    janet_stream_flags(stream, JANET_STREAM_UDPSERVER | JANET_STREAM_SOCKET);
     int32_t n = janet_getnat(argv, 1);
     JanetBuffer *buffer = janet_getbuffer(argv, 2);
     double to = janet_optnumber(argv, argc, 3, INFINITY);
     if (to != INFINITY) janet_addtimeout(to);
-    janet_sched_recv_from(stream, buffer, n);
-}
-
-static Janet cfun_stream_close(int32_t argc, Janet *argv) {
-    janet_fixarity(argc, 1);
-    JanetStream *stream = janet_getabstract(argv, 0, &StreamAT);
-    janet_stream_close(stream, 0);
-    return janet_wrap_nil();
-}
-
-static Janet cfun_stream_closed(int32_t argc, Janet *argv) {
-    janet_fixarity(argc, 1);
-    JanetStream *stream = janet_getabstract(argv, 0, &StreamAT);
-    return janet_wrap_boolean(stream->flags & JANET_POLL_FLAG_CLOSED);
+    janet_ev_recvfrom(stream, buffer, n, MSG_NOSIGNAL);
+    janet_await();
 }
 
 static Janet cfun_stream_write(int32_t argc, Janet *argv) {
     janet_arity(argc, 2, 3);
-    JanetStream *stream = janet_getabstract(argv, 0, &StreamAT);
-    check_stream_flag(stream, JANET_STREAM_WRITABLE);
+    JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type);
+    janet_stream_flags(stream, JANET_STREAM_WRITABLE | JANET_STREAM_SOCKET);
     double to = janet_optnumber(argv, argc, 2, INFINITY);
     if (janet_checktype(argv[1], JANET_BUFFER)) {
         if (to != INFINITY) janet_addtimeout(to);
-        janet_sched_write_buffer(stream, janet_getbuffer(argv, 1), NULL);
+        janet_ev_send_buffer(stream, janet_getbuffer(argv, 1), MSG_NOSIGNAL);
     } else {
         JanetByteView bytes = janet_getbytes(argv, 1);
         if (to != INFINITY) janet_addtimeout(to);
-        janet_sched_write_stringlike(stream, bytes.bytes, NULL);
+        janet_ev_send_string(stream, bytes.bytes, MSG_NOSIGNAL);
     }
+    janet_await();
 }
 
 static Janet cfun_stream_send_to(int32_t argc, Janet *argv) {
     janet_arity(argc, 3, 4);
-    JanetStream *stream = janet_getabstract(argv, 0, &StreamAT);
-    check_stream_flag(stream, JANET_STREAM_UDPSERVER);
-    void *dest = janet_getabstract(argv, 1, &AddressAT);
+    JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type);
+    janet_stream_flags(stream, JANET_STREAM_UDPSERVER | JANET_STREAM_SOCKET);
+    void *dest = janet_getabstract(argv, 1, &janet_address_type);
     double to = janet_optnumber(argv, argc, 3, INFINITY);
     if (janet_checktype(argv[2], JANET_BUFFER)) {
         if (to != INFINITY) janet_addtimeout(to);
-        janet_sched_write_buffer(stream, janet_getbuffer(argv, 2), dest);
+        janet_ev_sendto_buffer(stream, janet_getbuffer(argv, 2), dest, MSG_NOSIGNAL);
     } else {
         JanetByteView bytes = janet_getbytes(argv, 2);
         if (to != INFINITY) janet_addtimeout(to);
-        janet_sched_write_stringlike(stream, bytes.bytes, dest);
+        janet_ev_sendto_string(stream, bytes.bytes, dest, MSG_NOSIGNAL);
     }
+    janet_await();
 }
 
 static Janet cfun_stream_flush(int32_t argc, Janet *argv) {
-    janet_fixarity(argc, 2);
-    JanetStream *stream = janet_getabstract(argv, 0, &StreamAT);
-    check_stream_flag(stream, JANET_STREAM_WRITABLE);
+    janet_fixarity(argc, 1);
+    JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type);
+    janet_stream_flags(stream, JANET_STREAM_WRITABLE | JANET_STREAM_SOCKET);
     /* Toggle no delay flag */
     int flag = 1;
     setsockopt((JSock) stream->handle, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int));
@@ -1044,10 +608,9 @@ static Janet cfun_stream_flush(int32_t argc, Janet *argv) {
     return argv[0];
 }
 
-static const JanetMethod stream_methods[] = {
+static const JanetMethod net_stream_methods[] = {
     {"chunk", cfun_stream_chunk},
-    {"close", cfun_stream_close},
-    {"closed?", cfun_stream_closed},
+    {"close", janet_cfun_stream_close},
     {"read", cfun_stream_read},
     {"write", cfun_stream_write},
     {"flush", cfun_stream_flush},
@@ -1055,13 +618,15 @@ static const JanetMethod stream_methods[] = {
     {"accept-loop", cfun_stream_accept_loop},
     {"send-to", cfun_stream_send_to},
     {"recv-from", cfun_stream_recv_from},
+    {"recv-from", cfun_stream_recv_from},
+    {"evread", janet_cfun_stream_read},
+    {"evchunk", janet_cfun_stream_chunk},
+    {"evwrite", janet_cfun_stream_write},
     {NULL, NULL}
 };
 
-static int janet_stream_getter(void *p, Janet key, Janet *out) {
-    (void) p;
-    if (!janet_checktype(key, JANET_KEYWORD)) return 0;
-    return janet_getmethod(janet_unwrap_keyword(key), stream_methods, out);
+static JanetStream *make_stream(JSock handle, uint32_t flags) {
+    return janet_stream((JanetHandle) handle, flags | JANET_STREAM_SOCKET, net_stream_methods);
 }
 
 static const JanetReg net_cfuns[] = {
@@ -1078,7 +643,7 @@ static const JanetReg net_cfuns[] = {
         "net/listen", cfun_net_listen,
         JDOC("(net/listen host port &opt type)\n\n"
              "Creates a server. Returns a new stream that is neither readable nor "
-             "writeable. Use net/accept or net/accept-loop be to handle connections and start the server."
+             "writeable. Use net/accept or net/accept-loop be to handle connections and start the server. "
              "The type parameter specifies the type of network connection, either "
              "a :stream (usually tcp), or :datagram (usually udp). If not specified, the default is "
              ":stream. The host and port arguments are the same as in net/address.")
@@ -1093,13 +658,14 @@ static const JanetReg net_cfuns[] = {
     {
         "net/accept-loop", cfun_stream_accept_loop,
         JDOC("(net/accept-loop stream handler)\n\n"
-             "Shorthand for running a server stream that will continuously accept new connections."
+             "Shorthand for running a server stream that will continuously accept new connections. "
              "Blocks the current fiber until the stream is closed, and will return the stream.")
     },
     {
         "net/read", cfun_stream_read,
         JDOC("(net/read stream nbytes &opt buf timeout)\n\n"
              "Read up to n bytes from a stream, suspending the current fiber until the bytes are available. "
+             "`n` can also be the keyword `:all` to read into the buffer until end of stream. "
              "If less than n bytes are available (and more than 0), will push those bytes and return early. "
              "Takes an optional timeout in seconds, after which will return nil. "
              "Returns a buffer with up to n more bytes in it, or raises an error if the read failed.")
@@ -1136,19 +702,9 @@ static const JanetReg net_cfuns[] = {
              "Make sure that a stream is not buffering any data. This temporarily disables Nagle's algorithm. "
              "Use this to make sure data is sent without delay. Returns stream.")
     },
-    {
-        "net/close", cfun_stream_close,
-        JDOC("(net/close stream)\n\n"
-             "Close a stream so that no further communication can occur.")
-    },
-    {
-        "net/closed?", cfun_stream_closed,
-        JDOC("(net/closed? stream)\n\n"
-             "Check if a stream is closed.")
-    },
     {
         "net/connect", cfun_net_connect,
-        JDOC("(net/connect host porti &opt type)\n\n"
+        JDOC("(net/connect host port &opt type)\n\n"
              "Open a connection to communicate with a server. Returns a duplex stream "
              "that can be used to communicate with the server. Type is an optional keyword "
              "to specify a connection type, either :stream or :datagram. The default is :stream. ")

src/core/parse.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -175,7 +175,14 @@ static void popstate(JanetParser *p, Janet val) {
         if (newtop->flags & PFLAG_CONTAINER) {
             newtop->argn++;
             /* Keep track of number of values in the root state */
-            if (p->statecount == 1) p->pending++;
+            if (p->statecount == 1) {
+                p->pending++;
+                /* Root items are always wrapped in a tuple for source map info. */
+                const Janet *tup = janet_tuple_n(&val, 1);
+                janet_tuple_sm_line(tup) = (int32_t) top.line;
+                janet_tuple_sm_column(tup) = (int32_t) top.column;
+                val = janet_wrap_tuple(tup);
+            }
             push_arg(p, val);
             return;
         } else if (newtop->flags & PFLAG_READERMAC) {
@@ -313,11 +320,48 @@ static int stringend(JanetParser *p, JanetParseState *state) {
     uint8_t *bufstart = p->buf;
     int32_t buflen = (int32_t) p->bufcount;
     if (state->flags & PFLAG_LONGSTRING) {
-        /* Check for leading newline character so we can remove it */
-        if (bufstart[0] == '\n') {
-            bufstart++;
-            buflen--;
+        /* Post process to remove leading whitespace */
+        JanetParseState top = p->states[p->statecount - 1];
+        int32_t indent_col = (int32_t) top.column - 1;
+        uint8_t *r = bufstart, *end = r + buflen;
+        /* Check if there are any characters before the start column -
+         * if so, do not reindent. */
+        int reindent = 1;
+        while (reindent && (r < end)) {
+            if (*r++ == '\n') {
+                for (int32_t j = 0; (r < end) && (*r != '\n') && (j < indent_col); j++, r++) {
+                    if (*r != ' ') {
+                        reindent = 0;
+                        break;
+                    }
+                }
+            }
         }
+        /* Now reindent if able to, otherwise just drop leading newline. */
+        if (!reindent) {
+            if (buflen > 0 && bufstart[0] == '\n') {
+                buflen--;
+                bufstart++;
+            }
+        } else {
+            uint8_t *w = bufstart;
+            r = bufstart;
+            while (r < end) {
+                if (*r == '\n') {
+                    if (r == bufstart) {
+                        /* Skip leading newline */
+                        r++;
+                    } else {
+                        *w++ = *r++;
+                    }
+                    for (int32_t j = 0; (r < end) && (*r != '\n') && (j < indent_col); j++, r++);
+                } else {
+                    *w++ = *r++;
+                }
+            }
+            buflen = (int32_t)(w - bufstart);
+        }
+        /* Check for trailing newline character so we can remove it */
         if (buflen > 0 && bufstart[buflen - 1] == '\n') {
             buflen--;
         }
@@ -693,6 +737,19 @@ const char *janet_parser_error(JanetParser *parser) {
 }
 
 Janet janet_parser_produce(JanetParser *parser) {
+    Janet ret;
+    size_t i;
+    if (parser->pending == 0) return janet_wrap_nil();
+    ret = janet_unwrap_tuple(parser->args[0])[0];
+    for (i = 1; i < parser->argcount; i++) {
+        parser->args[i - 1] = parser->args[i];
+    }
+    parser->pending--;
+    parser->argcount--;
+    return ret;
+}
+
+Janet janet_parser_produce_wrapped(JanetParser *parser) {
     Janet ret;
     size_t i;
     if (parser->pending == 0) return janet_wrap_nil();
@@ -803,13 +860,21 @@ static int parsergc(void *p, size_t size) {
 }
 
 static int parserget(void *p, Janet key, Janet *out);
+static Janet parsernext(void *p, Janet key);
 
 const JanetAbstractType janet_parser_type = {
     "core/parser",
     parsergc,
     parsermark,
     parserget,
-    JANET_ATEND_GET
+    NULL, /* put */
+    NULL, /* marshal */
+    NULL, /* unmarshal */
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    parsernext,
+    JANET_ATEND_NEXT
 };
 
 /* C Function parser */
@@ -865,8 +930,13 @@ static Janet cfun_parse_insert(int32_t argc, Janet *argv) {
     if (s->flags & PFLAG_COMMENT) s--;
     if (s->flags & PFLAG_CONTAINER) {
         s->argn++;
-        if (p->statecount == 1) p->pending++;
-        push_arg(p, argv[1]);
+        if (p->statecount == 1) {
+            p->pending++;
+            Janet tup = janet_wrap_tuple(janet_tuple_n(argv + 1, 1));
+            push_arg(p, tup);
+        } else {
+            push_arg(p, argv[1]);
+        }
     } else if (s->flags & (PFLAG_STRING | PFLAG_LONGSTRING)) {
         const uint8_t *str = janet_to_string(argv[1]);
         int32_t slen = janet_string_length(str);
@@ -935,9 +1005,13 @@ static Janet cfun_parse_error(int32_t argc, Janet *argv) {
 }
 
 static Janet cfun_parse_produce(int32_t argc, Janet *argv) {
-    janet_fixarity(argc, 1);
+    janet_arity(argc, 1, 2);
     JanetParser *p = janet_getabstract(argv, 0, &janet_parser_type);
-    return janet_parser_produce(p);
+    if (argc == 2 && janet_truthy(argv[1])) {
+        return janet_parser_produce_wrapped(p);
+    } else {
+        return janet_parser_produce(p);
+    }
 }
 
 static Janet cfun_parse_flush(int32_t argc, Janet *argv) {
@@ -948,8 +1022,20 @@ static Janet cfun_parse_flush(int32_t argc, Janet *argv) {
 }
 
 static Janet cfun_parse_where(int32_t argc, Janet *argv) {
-    janet_fixarity(argc, 1);
+    janet_arity(argc, 1, 3);
     JanetParser *p = janet_getabstract(argv, 0, &janet_parser_type);
+    if (argc > 1) {
+        int32_t line = janet_getinteger(argv, 1);
+        if (line < 1)
+            janet_panicf("invalid line number %d", line);
+        p->line = (size_t) line;
+    }
+    if (argc > 2) {
+        int32_t column = janet_getinteger(argv, 2);
+        if (column < 0)
+            janet_panicf("invalid column number %d", column);
+        p->column = (size_t) column;
+    }
     Janet *tup = janet_tuple_begin(2);
     tup[0] = janet_wrap_integer(p->line);
     tup[1] = janet_wrap_integer(p->column);
@@ -1134,6 +1220,11 @@ static int parserget(void *p, Janet key, Janet *out) {
     return janet_getmethod(janet_unwrap_keyword(key), parser_methods, out);
 }
 
+static Janet parsernext(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(parser_methods, key);
+}
+
 static const JanetReg parse_cfuns[] = {
     {
         "parser/new", cfun_parse_parser,
@@ -1155,10 +1246,12 @@ static const JanetReg parse_cfuns[] = {
     },
     {
         "parser/produce", cfun_parse_produce,
-        JDOC("(parser/produce parser)\n\n"
+        JDOC("(parser/produce parser &opt wrap)\n\n"
              "Dequeue the next value in the parse queue. Will return nil if "
              "no parsed values are in the queue, otherwise will dequeue the "
-             "next value.")
+             "next value. If `wrap` is truthy, will return a 1-element tuple that "
+             "wraps the result. This tuple can be used for source-mapping "
+             "purposes.")
     },
     {
         "parser/consume", cfun_parse_consume,
@@ -1185,9 +1278,9 @@ static const JanetReg parse_cfuns[] = {
         JDOC("(parser/status parser)\n\n"
              "Gets the current status of the parser state machine. The status will "
              "be one of:\n\n"
-             "\t:pending - a value is being parsed.\n"
-             "\t:error - a parsing error was encountered.\n"
-             "\t:root - the parser can either read more values or safely terminate.")
+             "* :pending - a value is being parsed.\n\n"
+             "* :error - a parsing error was encountered.\n\n"
+             "* :root - the parser can either read more values or safely terminate.")
     },
     {
         "parser/flush", cfun_parse_flush,
@@ -1201,17 +1294,19 @@ static const JanetReg parse_cfuns[] = {
         JDOC("(parser/state parser &opt key)\n\n"
              "Returns a representation of the internal state of the parser. If a key is passed, "
              "only that information about the state is returned. Allowed keys are:\n\n"
-             "\t:delimiters - Each byte in the string represents a nested data structure. For example, "
+             "* :delimiters - Each byte in the string represents a nested data structure. For example, "
              "if the parser state is '([\"', then the parser is in the middle of parsing a "
-             "string inside of square brackets inside parentheses. Can be used to augment a REPL prompt."
-             "\t:frames - Each table in the array represents a 'frame' in the parser state. Frames "
+             "string inside of square brackets inside parentheses. Can be used to augment a REPL prompt.\n\n"
+             "* :frames - Each table in the array represents a 'frame' in the parser state. Frames "
              "contain information about the start of the expression being parsed as well as the "
              "type of that expression and some type-specific information.")
     },
     {
         "parser/where", cfun_parse_where,
-        JDOC("(parser/where parser)\n\n"
-             "Returns the current line number and column of the parser's internal state.")
+        JDOC("(parser/where parser &opt line col)\n\n"
+             "Returns the current line number and column of the parser's internal state. If line is "
+             "provided, the current line number of the parser is first set to that value. If column is "
+             "also provided, the current column number of the parser is also first set to that value.")
     },
     {
         "parser/eof", cfun_parse_eof,

src/core/peg.c

@@ -44,9 +44,13 @@ typedef struct {
     JanetArray *captures;
     JanetBuffer *scratch;
     JanetBuffer *tags;
+    JanetArray *tagged_captures;
     const Janet *extrav;
+    int32_t *linemap;
     int32_t extrac;
     int32_t depth;
+    int32_t linemaplen;
+    int32_t has_backref;
     enum {
         PEG_MODE_NORMAL,
         PEG_MODE_ACCUMULATE
@@ -58,6 +62,7 @@ typedef struct {
  * if one branch fails and try a new branch. */
 typedef struct {
     int32_t cap;
+    int32_t tcap;
     int32_t scratch;
 } CapState;
 
@@ -66,6 +71,7 @@ static CapState cap_save(PegState *s) {
     CapState cs;
     cs.scratch = s->scratch->count;
     cs.cap = s->captures->count;
+    cs.tcap = s->tagged_captures->count;
     return cs;
 }
 
@@ -73,7 +79,15 @@ static CapState cap_save(PegState *s) {
 static void cap_load(PegState *s, CapState cs) {
     s->scratch->count = cs.scratch;
     s->captures->count = cs.cap;
-    s->tags->count = cs.cap;
+    s->tags->count = cs.tcap;
+    s->tagged_captures->count = cs.tcap;
+}
+
+/* Load a saved capture state in the case of success. Keeps
+ * tagged captures around for backref. */
+static void cap_load_keept(PegState *s, CapState cs) {
+    s->scratch->count = cs.scratch;
+    s->captures->count = cs.cap;
 }
 
 /* Add a capture */
@@ -81,12 +95,66 @@ static void pushcap(PegState *s, Janet capture, uint32_t tag) {
     if (s->mode == PEG_MODE_ACCUMULATE) {
         janet_to_string_b(s->scratch, capture);
     }
-    if (tag || s->mode == PEG_MODE_NORMAL) {
+    if (s->mode == PEG_MODE_NORMAL) {
         janet_array_push(s->captures, capture);
+    }
+    if (s->has_backref) {
+        janet_array_push(s->tagged_captures, capture);
         janet_buffer_push_u8(s->tags, tag);
     }
 }
 
+/* Lazily generate line map to get line and column information for PegState.
+ * line and column are 1-indexed. */
+typedef struct {
+    int32_t line;
+    int32_t col;
+} LineCol;
+static LineCol get_linecol_from_position(PegState *s, int32_t position) {
+    /* Generate if not made yet */
+    if (s->linemaplen < 0) {
+        int32_t newline_count = 0;
+        for (const uint8_t *c = s->text_start; c < s->text_end; c++) {
+            if (*c == '\n') newline_count++;
+        }
+        int32_t *mem = janet_smalloc(sizeof(int32_t) * newline_count);
+        size_t index = 0;
+        for (const uint8_t *c = s->text_start; c < s->text_end; c++) {
+            if (*c == '\n') mem[index++] = (int32_t)(c - s->text_start);
+        }
+        s->linemaplen = newline_count;
+        s->linemap = mem;
+    }
+    /* Do binary search for line. Slightly modified from classic binary search:
+     * - if we find that our current character is a line break, just return immediately.
+     *   a newline character is consider to be on the same line as the character before
+     *   (\n is line terminator, not line separator).
+     * - in the not-found case, we still want to find the greatest-indexed newline that
+     *   is before position. we use that to calcuate the line and column.
+     * - in the case that lo = 0 and s->linemap[0] is still greater than position, we
+     *   are on the first line and our column is position + 1. */
+    int32_t hi = s->linemaplen; /* hi is greater than the actual line */
+    int32_t lo = 0; /* lo is less than or equal to the actual line */
+    LineCol ret;
+    while (lo + 1 < hi) {
+        int32_t mid = lo + (hi - lo) / 2;
+        if (s->linemap[mid] >= position) {
+            hi = mid;
+        } else {
+            lo = mid;
+        }
+    }
+    /* first line case */
+    if (s->linemaplen == 0 || (lo == 0 && s->linemap[0] >= position)) {
+        ret.line = 1;
+        ret.col = position + 1;
+    } else {
+        ret.line = lo + 2;
+        ret.col = position - s->linemap[lo];
+    }
+    return ret;
+}
+
 /* Convert a uint64_t to a int64_t by wrapping to a maximum number of bytes */
 static int64_t peg_convert_u64_s64(uint64_t from, int width) {
     int shift = 8 * (8 - width);
@@ -268,7 +336,7 @@ tail:
             uint32_t tag = rule[2];
             for (int32_t i = s->tags->count - 1; i >= 0; i--) {
                 if (s->tags->data[i] == search) {
-                    pushcap(s, s->captures->data[i], tag);
+                    pushcap(s, s->tagged_captures->data[i], tag);
                     return text;
                 }
             }
@@ -280,6 +348,18 @@ tail:
             return text;
         }
 
+        case RULE_LINE: {
+            LineCol lc = get_linecol_from_position(s, (int32_t)(text - s->text_start));
+            pushcap(s, janet_wrap_number((double)(lc.line)), rule[1]);
+            return text;
+        }
+
+        case RULE_COLUMN: {
+            LineCol lc = get_linecol_from_position(s, (int32_t)(text - s->text_start));
+            pushcap(s, janet_wrap_number((double)(lc.col)), rule[1]);
+            return text;
+        }
+
         case RULE_ARGUMENT: {
             int32_t index = ((int32_t *)rule)[1];
             Janet capture = (index >= s->extrac) ? janet_wrap_nil() : s->extrav[index];
@@ -293,15 +373,15 @@ tail:
         }
 
         case RULE_CAPTURE: {
-            uint32_t tag = rule[2];
             down1(s);
             const uint8_t *result = peg_rule(s, s->bytecode + rule[1], text);
             up1(s);
             if (!result) return NULL;
             /* Specialized pushcap - avoid intermediate string creation */
-            if (!tag && s->mode == PEG_MODE_ACCUMULATE) {
+            if (!s->has_backref && s->mode == PEG_MODE_ACCUMULATE) {
                 janet_buffer_push_bytes(s->scratch, text, (int32_t)(result - text));
             } else {
+                uint32_t tag = rule[2];
                 pushcap(s, janet_stringv(text, (int32_t)(result - text)), tag);
             }
             return result;
@@ -323,7 +403,7 @@ tail:
             if (!result) return NULL;
             Janet cap = janet_stringv(s->scratch->data + cs.scratch,
                                       s->scratch->count - cs.scratch);
-            cap_load(s, cs);
+            cap_load_keept(s, cs);
             pushcap(s, cap, tag);
             return result;
         }
@@ -354,7 +434,7 @@ tail:
                         s->captures->data + cs.cap,
                         sizeof(Janet) * num_sub_captures);
             sub_captures->count = num_sub_captures;
-            cap_load(s, cs);
+            cap_load_keept(s, cs);
             pushcap(s, janet_wrap_array(sub_captures), tag);
             return result;
         }
@@ -399,7 +479,7 @@ tail:
                                      s->captures->data + cs.cap);
                     break;
             }
-            cap_load(s, cs);
+            cap_load_keept(s, cs);
             if (rule[0] == RULE_MATCHTIME && !janet_truthy(cap)) return NULL;
             pushcap(s, cap, tag);
             return result;
@@ -420,8 +500,8 @@ tail:
             } else {
                 /* Throw generic error */
                 int32_t start = (int32_t)(text - s->text_start);
-                int32_t end = (int32_t)(result - s->text_start);
-                janet_panicf("match error in range (%d:%d)", start, end);
+                LineCol lc = get_linecol_from_position(s, start);
+                janet_panicf("match error at line %d, column %d", lc.line, lc.col);
             }
             return NULL;
         }
@@ -430,7 +510,7 @@ tail:
             uint32_t search = rule[1];
             for (int32_t i = s->tags->count - 1; i >= 0; i--) {
                 if (s->tags->data[i] == search) {
-                    Janet capture = s->captures->data[i];
+                    Janet capture = s->tagged_captures->data[i];
                     if (!janet_checktype(capture, JANET_STRING))
                         return NULL;
                     const uint8_t *bytes = janet_unwrap_string(capture);
@@ -532,6 +612,7 @@ typedef struct {
     Janet form;
     int depth;
     uint32_t nexttag;
+    int has_backref;
 } Builder;
 
 /* Forward declaration to allow recursion */
@@ -830,7 +911,13 @@ static void spec_not(Builder *b, int32_t argc, const Janet *argv) {
     spec_onerule(b, argc, argv, RULE_NOT);
 }
 static void spec_error(Builder *b, int32_t argc, const Janet *argv) {
-    spec_onerule(b, argc, argv, RULE_ERROR);
+    if (argc == 0) {
+        Reserve r = reserve(b, 2);
+        uint32_t rule = peg_compile1(b, janet_wrap_number(0));
+        emit_1(r, RULE_ERROR, rule);
+    } else {
+        spec_onerule(b, argc, argv, RULE_ERROR);
+    }
 }
 static void spec_drop(Builder *b, int32_t argc, const Janet *argv) {
     spec_onerule(b, argc, argv, RULE_DROP);
@@ -866,6 +953,7 @@ static void spec_reference(Builder *b, int32_t argc, const Janet *argv) {
     Reserve r = reserve(b, 3);
     uint32_t search = emit_tag(b, argv[0]);
     uint32_t tag = (argc == 2) ? emit_tag(b, argv[1]) : 0;
+    b->has_backref = 1;
     emit_2(r, RULE_GETTAG, search, tag);
 }
 
@@ -880,8 +968,15 @@ static void spec_tag1(Builder *b, int32_t argc, const Janet *argv, uint32_t op)
 static void spec_position(Builder *b, int32_t argc, const Janet *argv) {
     spec_tag1(b, argc, argv, RULE_POSITION);
 }
+static void spec_line(Builder *b, int32_t argc, const Janet *argv) {
+    spec_tag1(b, argc, argv, RULE_LINE);
+}
+static void spec_column(Builder *b, int32_t argc, const Janet *argv) {
+    spec_tag1(b, argc, argv, RULE_COLUMN);
+}
 
 static void spec_backmatch(Builder *b, int32_t argc, const Janet *argv) {
+    b->has_backref = 1;
     spec_tag1(b, argc, argv, RULE_BACKMATCH);
 }
 
@@ -983,6 +1078,7 @@ static const SpecialPair peg_specials[] = {
     {"capture", spec_capture},
     {"choice", spec_choice},
     {"cmt", spec_matchtime},
+    {"column", spec_column},
     {"constant", spec_constant},
     {"drop", spec_drop},
     {"error", spec_error},
@@ -992,6 +1088,7 @@ static const SpecialPair peg_specials[] = {
     {"int", spec_int_le},
     {"int-be", spec_int_be},
     {"lenprefix", spec_lenprefix},
+    {"line", spec_line},
     {"look", spec_look},
     {"not", spec_not},
     {"opt", spec_opt},
@@ -1216,6 +1313,7 @@ static void *peg_unmarshal(JanetMarshalContext *ctx) {
     }
 
     /* verify peg bytecode */
+    int32_t has_backref = 0;
     uint32_t i = 0;
     while (i < blen) {
         uint32_t instr = bytecode[i];
@@ -1229,9 +1327,15 @@ static void *peg_unmarshal(JanetMarshalContext *ctx) {
             case RULE_NOTNCHAR:
             case RULE_RANGE:
             case RULE_POSITION:
+            case RULE_LINE:
+            case RULE_COLUMN:
+                /* [1 word] */
+                i += 2;
+                break;
             case RULE_BACKMATCH:
                 /* [1 word] */
                 i += 2;
+                has_backref = 1;
                 break;
             case RULE_SET:
                 /* [8 words] */
@@ -1272,9 +1376,13 @@ static void *peg_unmarshal(JanetMarshalContext *ctx) {
                 i += 4;
                 break;
             case RULE_ARGUMENT:
+                /* [searchtag, tag] */
+                i += 3;
+                break;
             case RULE_GETTAG:
                 /* [searchtag, tag] */
                 i += 3;
+                has_backref = 1;
                 break;
             case RULE_CONSTANT:
                 /* [constant, tag] */
@@ -1328,6 +1436,7 @@ static void *peg_unmarshal(JanetMarshalContext *ctx) {
     /* Good return */
     peg->bytecode = bytecode;
     peg->constants = constants;
+    peg->has_backref = has_backref;
     free(op_flags);
     return peg;
 
@@ -1337,16 +1446,21 @@ bad:
 }
 
 static int cfun_peg_getter(JanetAbstract a, Janet key, Janet *out);
+static Janet peg_next(void *p, Janet key);
 
 const JanetAbstractType janet_peg_type = {
     "core/peg",
     NULL,
     peg_mark,
     cfun_peg_getter,
-    NULL,
+    NULL, /* put */
     peg_marshal,
     peg_unmarshal,
-    JANET_ATEND_UNMARSHAL
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    peg_next,
+    JANET_ATEND_NEXT
 };
 
 /* Convert Builder to JanetPeg (Janet Abstract Value) */
@@ -1364,6 +1478,7 @@ static JanetPeg *make_peg(Builder *b) {
     safe_memcpy(peg->bytecode, b->bytecode, bytecode_size);
     safe_memcpy(peg->constants, b->constants, constants_size);
     peg->bytecode_len = janet_v_count(b->bytecode);
+    peg->has_backref = b->has_backref;
     return peg;
 }
 
@@ -1371,13 +1486,20 @@ static JanetPeg *make_peg(Builder *b) {
 static JanetPeg *compile_peg(Janet x) {
     Builder builder;
     builder.grammar = janet_table(0);
-    builder.default_grammar = janet_get_core_table("default-peg-grammar");
+    builder.default_grammar = NULL;
+    {
+        Janet default_grammarv = janet_dyn("peg-grammar");
+        if (janet_checktype(default_grammarv, JANET_TABLE)) {
+            builder.default_grammar = janet_unwrap_table(default_grammarv);
+        }
+    }
     builder.tags = janet_table(0);
     builder.constants = NULL;
     builder.bytecode = NULL;
     builder.nexttag = 1;
     builder.form = x;
     builder.depth = JANET_RECURSION_GUARD;
+    builder.has_backref = 0;
     peg_compile1(&builder, x);
     JanetPeg *peg = make_peg(&builder);
     builder_cleanup(&builder);
@@ -1434,10 +1556,14 @@ static PegCall peg_cfun_init(int32_t argc, Janet *argv, int get_replace) {
     ret.s.text_end = ret.bytes.bytes + ret.bytes.len;
     ret.s.depth = JANET_RECURSION_GUARD;
     ret.s.captures = janet_array(0);
+    ret.s.tagged_captures = janet_array(0);
     ret.s.scratch = janet_buffer(10);
     ret.s.tags = janet_buffer(10);
     ret.s.constants = ret.peg->constants;
     ret.s.bytecode = ret.peg->bytecode;
+    ret.s.linemap = NULL;
+    ret.s.linemaplen = -1;
+    ret.s.has_backref = ret.peg->has_backref;
     return ret;
 }
 
@@ -1526,12 +1652,18 @@ static int cfun_peg_getter(JanetAbstract a, Janet key, Janet *out) {
     return janet_getmethod(janet_unwrap_keyword(key), peg_methods, out);
 }
 
+static Janet peg_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(peg_methods, key);
+}
+
 static const JanetReg peg_cfuns[] = {
     {
         "peg/compile", cfun_peg_compile,
         JDOC("(peg/compile peg)\n\n"
              "Compiles a peg source data structure into a <core/peg>. This will speed up matching "
-             "if the same peg will be used multiple times.")
+             "if the same peg will be used multiple times. Will also use `(dyn :peg-grammar)` to suppliment "
+             "the grammar of the peg for otherwise undefined peg keywords.")
     },
     {
         "peg/match", cfun_peg_match,

src/core/pp.c

@@ -42,7 +42,14 @@ static void number_to_string_b(JanetBuffer *buffer, double x) {
     const char *fmt = (x == floor(x) &&
                        x <= JANET_INTMAX_DOUBLE &&
                        x >= JANET_INTMIN_DOUBLE) ? "%.0f" : "%g";
-    int count = snprintf((char *) buffer->data + buffer->count, BUFSIZE, fmt, x);
+    int count;
+    if (x == 0.0) {
+        /* Prevent printing of '-0' */
+        count = 1;
+        buffer->data[buffer->count] = '0';
+    } else {
+        count = snprintf((char *) buffer->data + buffer->count, BUFSIZE, fmt, x);
+    }
     buffer->count += count;
 }
 
@@ -565,7 +572,7 @@ static void janet_pretty_one(struct pretty *S, Janet x, int is_dict_value) {
                 JanetTable *t = janet_unwrap_table(x);
                 JanetTable *proto = t->proto;
                 if (NULL != proto) {
-                    Janet name = janet_table_get(proto, janet_ckeywordv("name"));
+                    Janet name = janet_table_get(proto, janet_ckeywordv("_name"));
                     const uint8_t *n;
                     int32_t len;
                     if (janet_bytes_view(name, &n, &len)) {

src/core/specials.c

@@ -336,10 +336,8 @@ static int defleaf(
 
         /* Put value in table when evaulated */
         janetc_emit_sss(c, JOP_PUT, tabslot, valsym, s, 0);
-        return 1;
-    } else {
-        return namelocal(c, sym, 0, s);
     }
+    return namelocal(c, sym, 0, s);
 }
 
 static JanetSlot janetc_def(JanetFopts opts, int32_t argn, const Janet *argv) {
@@ -470,6 +468,28 @@ static JanetSlot janetc_do(JanetFopts opts, int32_t argn, const Janet *argv) {
     return ret;
 }
 
+
+/* Compile an upscope form. Upscope forms execute their body sequentially and
+ * evaluate to the last expression in the body, but without lexical scope. */
+static JanetSlot janetc_upscope(JanetFopts opts, int32_t argn, const Janet *argv) {
+    int32_t i;
+    JanetSlot ret = janetc_cslot(janet_wrap_nil());
+    JanetCompiler *c = opts.compiler;
+    JanetFopts subopts = janetc_fopts_default(c);
+    for (i = 0; i < argn; i++) {
+        if (i != argn - 1) {
+            subopts.flags = JANET_FOPTS_DROP;
+        } else {
+            subopts = opts;
+        }
+        ret = janetc_value(subopts, argv[i]);
+        if (i != argn - 1) {
+            janetc_freeslot(c, ret);
+        }
+    }
+    return ret;
+}
+
 /* Add a funcdef to the top most function scope */
 static int32_t janetc_addfuncdef(JanetCompiler *c, JanetFuncDef *def) {
     JanetScope *scope = c->scope;
@@ -854,6 +874,7 @@ static const JanetSpecial janetc_specials[] = {
     {"set", janetc_varset},
     {"splice", janetc_splice},
     {"unquote", janetc_unquote},
+    {"upscope", janetc_upscope},
     {"var", janetc_var},
     {"while", janetc_while}
 };

src/core/string.c

@@ -398,6 +398,7 @@ static Janet cfun_string_split(int32_t argc, Janet *argv) {
         const uint8_t *slice = janet_string(state.text + lastindex, result - lastindex);
         janet_array_push(array, janet_wrap_string(slice));
         lastindex = result + state.patlen;
+        kmp_seti(&state, lastindex);
     }
     const uint8_t *slice = janet_string(state.text + lastindex, state.textlen - lastindex);
     janet_array_push(array, janet_wrap_string(slice));
@@ -598,9 +599,8 @@ static const JanetReg string_cfuns[] = {
         JDOC("(string/find-all patt str)\n\n"
              "Searches for all instances of pattern patt in string "
              "str. Returns an array of all indices of found patterns. Overlapping "
-             "instances of the pattern are not counted, meaning a byte in string "
-             "will only contribute to finding at most on occurrence of pattern. If no "
-             "occurrences are found, will return an empty array.")
+             "instances of the pattern are counted individually, meaning a byte in str "
+             "may contribute to multiple found patterns.")
     },
     {
         "string/has-prefix?", cfun_string_hasprefix,
@@ -621,7 +621,8 @@ static const JanetReg string_cfuns[] = {
     {
         "string/replace-all", cfun_string_replaceall,
         JDOC("(string/replace-all patt subst str)\n\n"
-             "Replace all instances of patt with subst in the string str. "
+             "Replace all instances of patt with subst in the string str. Overlapping "
+             "matches will not be counted, only the first match in such a span will be replaced. "
              "Will return the new string if patt is found, otherwise returns str.")
     },
     {

src/core/strtod.c

@@ -447,7 +447,7 @@ int janet_scan_int64(const uint8_t *str, int32_t len, int64_t *out) {
     int neg;
     uint64_t bi;
     if (scan_uint64(str, len, &bi, &neg)) {
-        if (neg && bi <= (UINT64_MAX / 2)) {
+        if (neg && bi <= ((UINT64_MAX / 2) + 1)) {
             if (bi > INT64_MAX) {
                 *out = INT64_MIN;
             } else {

src/core/thread.c

@@ -84,6 +84,9 @@ static JANET_THREAD_LOCAL JanetTable *janet_vm_thread_decode = NULL;
 static JanetTable *janet_thread_get_decode(void) {
     if (janet_vm_thread_decode == NULL) {
         janet_vm_thread_decode = janet_get_core_table("load-image-dict");
+        if (NULL == janet_vm_thread_decode) {
+            janet_vm_thread_decode = janet_table(0);
+        }
         janet_gcroot(janet_wrap_table(janet_vm_thread_decode));
     }
     return janet_vm_thread_decode;
@@ -418,13 +421,21 @@ int janet_thread_receive(Janet *msg_out, double timeout) {
 }
 
 static int janet_thread_getter(void *p, Janet key, Janet *out);
+static Janet janet_thread_next(void *p, Janet key);
 
 const JanetAbstractType janet_thread_type = {
     "core/thread",
     thread_gc,
     thread_mark,
     janet_thread_getter,
-    JANET_ATEND_GET
+    NULL, /* put */
+    NULL, /* marshal */
+    NULL, /* unmarshal */
+    NULL, /* tostring */
+    NULL, /* compare */
+    NULL, /* hash */
+    janet_thread_next,
+    JANET_ATEND_NEXT
 };
 
 static JanetThread *janet_make_thread(JanetMailbox *mailbox, JanetTable *encode) {
@@ -512,7 +523,7 @@ static int thread_worker(JanetMailboxPair *pair) {
         janet_stacktrace(fiber, out);
     }
 
-#ifdef JANET_NET
+#ifdef JANET_EV
     janet_loop();
 #endif
 
@@ -585,6 +596,14 @@ void janet_threads_deinit(void) {
     janet_vm_thread_decode = NULL;
 }
 
+JanetThread *janet_thread_current(void) {
+    if (NULL == janet_vm_thread_current) {
+        janet_vm_thread_current = janet_make_thread(janet_vm_mailbox, janet_get_core_table("make-image-dict"));
+        janet_gcroot(janet_wrap_abstract(janet_vm_thread_current));
+    }
+    return janet_vm_thread_current;
+}
+
 /*
  * Cfuns
  */
@@ -592,11 +611,7 @@ void janet_threads_deinit(void) {
 static Janet cfun_thread_current(int32_t argc, Janet *argv) {
     (void) argv;
     janet_fixarity(argc, 0);
-    if (NULL == janet_vm_thread_current) {
-        janet_vm_thread_current = janet_make_thread(janet_vm_mailbox, janet_get_core_table("make-image-dict"));
-        janet_gcroot(janet_wrap_abstract(janet_vm_thread_current));
-    }
-    return janet_wrap_abstract(janet_vm_thread_current);
+    return janet_wrap_abstract(janet_thread_current());
 }
 
 static Janet cfun_thread_new(int32_t argc, Janet *argv) {
@@ -704,6 +719,11 @@ static int janet_thread_getter(void *p, Janet key, Janet *out) {
     return janet_getmethod(janet_unwrap_keyword(key), janet_thread_methods, out);
 }
 
+static Janet janet_thread_next(void *p, Janet key) {
+    (void) p;
+    return janet_nextmethod(janet_thread_methods, key);
+}
+
 static const JanetReg threadlib_cfuns[] = {
     {
         "thread/current", cfun_thread_current,
@@ -716,10 +736,10 @@ static const JanetReg threadlib_cfuns[] = {
              "Start a new thread that will start immediately. "
              "If capacity is provided, that is how many messages can be stored in the thread's mailbox before blocking senders. "
              "The capacity must be between 1 and 65535 inclusive, and defaults to 10. "
-             "Can optionally provide flags to the new thread - supported flags are:\n"
-             "\t:h - Start a heavyweight thread. This loads the core environment by default, so may use more memory initially. Messages may compress better, though.\n"
-             "\t:a - Allow sending over registered abstract types to the new thread\n"
-             "\t:c - Send over cfunction information to the new thread.\n"
+             "Can optionally provide flags to the new thread - supported flags are:\n\n"
+             "* :h - Start a heavyweight thread. This loads the core environment by default, so may use more memory initially. Messages may compress better, though.\n\n"
+             "* :a - Allow sending over registered abstract types to the new thread\n\n"
+             "* :c - Send over cfunction information to the new thread.\n\n"
              "Returns a handle to the new thread.")
     },
     {

src/core/util.c

@@ -227,19 +227,21 @@ int32_t janet_string_calchash(const uint8_t *str, int32_t len) {
 /* Computes hash of an array of values */
 int32_t janet_array_calchash(const Janet *array, int32_t len) {
     const Janet *end = array + len;
-    uint32_t hash = 5381;
-    while (array < end)
-        hash = (hash << 5) + hash + janet_hash(*array++);
+    uint32_t hash = 0;
+    while (array < end) {
+        uint32_t elem = janet_hash(*array++);
+        hash ^= elem + 0x9e3779b9 + (hash << 6) + (hash >> 2);
+    }
     return (int32_t) hash;
 }
 
 /* Computes hash of an array of values */
 int32_t janet_kv_calchash(const JanetKV *kvs, int32_t len) {
     const JanetKV *end = kvs + len;
-    uint32_t hash = 5381;
+    uint32_t hash = 0;
     while (kvs < end) {
-        hash = (hash << 5) + hash + janet_hash(kvs->key);
-        hash = (hash << 5) + hash + janet_hash(kvs->value);
+        hash ^= janet_hash(kvs->key) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
+        hash ^= janet_hash(kvs->value) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
         kvs++;
     }
     return (int32_t) hash;

src/core/util.h

@@ -91,6 +91,7 @@ void janet_buffer_format(
     int32_t argstart,
     int32_t argc,
     Janet *argv);
+Janet janet_next_impl(Janet ds, Janet key, int is_interpreter);
 
 /* Inside the janet core, defining globals is different
  * at bootstrap time and normal runtime */
@@ -107,6 +108,11 @@ void janet_core_cfuns(JanetTable *env, const char *regprefix, const JanetReg *cf
 int janet_gettime(struct timespec *spec);
 #endif
 
+/* strdup */
+#ifdef JANET_WINDOWS
+#define strdup(x) _strdup(x)
+#endif
+
 #define RETRY_EINTR(RC, CALL) do { (RC) = CALL; } while((RC) < 0 && errno == EINTR)
 
 /* Initialize builtin libraries */
@@ -140,10 +146,12 @@ void janet_lib_thread(JanetTable *env);
 #endif
 #ifdef JANET_NET
 void janet_lib_net(JanetTable *env);
+extern const JanetAbstractType janet_address_type;
 #endif
 #ifdef JANET_EV
 void janet_lib_ev(JanetTable *env);
 void janet_ev_mark(void);
+int janet_make_pipe(JanetHandle handles[2], int mode);
 #endif
 
 #endif

src/core/value.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -25,9 +25,12 @@
 #include "util.h"
 #include "state.h"
 #include "gc.h"
+#include "fiber.h"
 #include <janet.h>
 #endif
 
+#include <math.h>
+
 JANET_THREAD_LOCAL JanetTraversalNode *janet_vm_traversal = NULL;
 JANET_THREAD_LOCAL JanetTraversalNode *janet_vm_traversal_top = NULL;
 JANET_THREAD_LOCAL JanetTraversalNode *janet_vm_traversal_base = NULL;
@@ -113,6 +116,10 @@ static int traversal_next(Janet *x, Janet *y) {
  */
 
 Janet janet_next(Janet ds, Janet key) {
+    return janet_next_impl(ds, key, 0);
+}
+
+Janet janet_next_impl(Janet ds, Janet key, int is_interpreter) {
     JanetType t = janet_type(ds);
     switch (t) {
         default:
@@ -175,6 +182,44 @@ Janet janet_next(Janet ds, Janet key) {
             if (NULL == at->next) break;
             return at->next(abst, key);
         }
+        case JANET_FIBER: {
+            JanetFiber *child = janet_unwrap_fiber(ds);
+            Janet retreg;
+            JanetFiberStatus status = janet_fiber_status(child);
+            if (status == JANET_STATUS_ALIVE ||
+                    status == JANET_STATUS_DEAD ||
+                    status == JANET_STATUS_ERROR ||
+                    status == JANET_STATUS_USER0 ||
+                    status == JANET_STATUS_USER1 ||
+                    status == JANET_STATUS_USER2 ||
+                    status == JANET_STATUS_USER3 ||
+                    status == JANET_STATUS_USER4) {
+                return janet_wrap_nil();
+            }
+            janet_vm_fiber->child = child;
+            JanetSignal sig = janet_continue(child, janet_wrap_nil(), &retreg);
+            if (sig != JANET_SIGNAL_OK && !(child->flags & (1 << sig))) {
+                if (is_interpreter) {
+                    janet_signalv(sig, retreg);
+                } else {
+                    janet_vm_fiber->child = NULL;
+                    janet_panicv(retreg);
+                }
+            }
+            janet_vm_fiber->child = NULL;
+            if (sig == JANET_SIGNAL_OK ||
+                    sig == JANET_SIGNAL_ERROR ||
+                    sig == JANET_SIGNAL_USER0 ||
+                    sig == JANET_SIGNAL_USER1 ||
+                    sig == JANET_SIGNAL_USER2 ||
+                    sig == JANET_SIGNAL_USER3 ||
+                    sig == JANET_SIGNAL_USER4) {
+                /* Fiber cannot be resumed, so discard last value. */
+                return janet_wrap_nil();
+            } else {
+                return janet_wrap_integer(0);
+            }
+        }
     }
     return janet_wrap_nil();
 }
@@ -261,6 +306,17 @@ int32_t janet_hash(Janet x) {
         case JANET_STRUCT:
             hash = janet_struct_hash(janet_unwrap_struct(x));
             break;
+        case JANET_NUMBER: {
+            union {
+                double d;
+                uint64_t u;
+            } as;
+            as.d = janet_unwrap_number(x);
+            uint32_t lo = (uint32_t)(as.u & 0xFFFFFFFF);
+            uint32_t hi = (uint32_t)(as.u >> 32);
+            hash = (int32_t)(hi ^ (lo >> 3));
+            break;
+        }
         case JANET_ABSTRACT: {
             JanetAbstract xx = janet_unwrap_abstract(x);
             const JanetAbstractType *at = janet_abstract_type(xx);
@@ -417,6 +473,14 @@ Janet janet_in(Janet ds, Janet key) {
             }
             break;
         }
+        case JANET_FIBER: {
+            /* Bit of a hack to allow iterating over fibers. */
+            if (janet_equals(key, janet_wrap_integer(0))) {
+                return janet_unwrap_fiber(ds)->last_value;
+            } else {
+                janet_panicf("expected key 0, got %v", key);
+            }
+        }
     }
     return value;
 }
@@ -472,6 +536,14 @@ Janet janet_get(Janet ds, Janet key) {
             const JanetKV *st = janet_unwrap_struct(ds);
             return janet_struct_get(st, key);
         }
+        case JANET_FIBER: {
+            /* Bit of a hack to allow iterating over fibers. */
+            if (janet_equals(key, janet_wrap_integer(0))) {
+                return janet_unwrap_fiber(ds)->last_value;
+            } else {
+                return janet_wrap_nil();
+            }
+        }
     }
 }
 
@@ -528,6 +600,14 @@ Janet janet_getindex(Janet ds, int32_t index) {
             }
             break;
         }
+        case JANET_FIBER: {
+            if (index == 0) {
+                value = janet_unwrap_fiber(ds)->last_value;
+            } else {
+                value = janet_wrap_nil();
+            }
+            break;
+        }
     }
     return value;
 }

src/core/vm.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2020 Calvin Rose
+* Copyright (c) 2021 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -202,6 +202,20 @@ JANET_THREAD_LOCAL jmp_buf *janet_vm_jmp_buf = NULL;
             vm_checkgc_pcnext();\
         }\
     }
+#define vm_compop_imm(op) \
+    {\
+        Janet op1 = stack[B];\
+        if (janet_checktype(op1, JANET_NUMBER)) {\
+            double x1 = janet_unwrap_number(op1);\
+            double x2 = (double) CS; \
+            stack[A] = janet_wrap_boolean(x1 op x2);\
+            vm_pcnext();\
+        } else {\
+            vm_commit();\
+            stack[A] = janet_wrap_boolean(janet_compare(op1, janet_wrap_integer(CS)) op 0);\
+            vm_checkgc_pcnext();\
+        }\
+    }
 
 /* Trace a function call */
 static void vm_do_trace(JanetFunction *func, int32_t argc, const Janet *argv) {
@@ -780,8 +794,7 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in) {
     vm_compop( <=);
 
     VM_OP(JOP_LESS_THAN_IMMEDIATE)
-    stack[A] = janet_wrap_boolean(janet_unwrap_integer(stack[B]) < CS);
-    vm_pcnext();
+    vm_compop_imm( <);
 
     VM_OP(JOP_GREATER_THAN)
     vm_compop( >);
@@ -790,15 +803,14 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in) {
     vm_compop( >=);
 
     VM_OP(JOP_GREATER_THAN_IMMEDIATE)
-    stack[A] = janet_wrap_boolean(janet_unwrap_integer(stack[B]) > CS);
-    vm_pcnext();
+    vm_compop_imm( >);
 
     VM_OP(JOP_EQUALS)
     stack[A] = janet_wrap_boolean(janet_equals(stack[B], stack[C]));
     vm_pcnext();
 
     VM_OP(JOP_EQUALS_IMMEDIATE)
-    stack[A] = janet_wrap_boolean(janet_unwrap_integer(stack[B]) == CS);
+    stack[A] = janet_wrap_boolean(janet_unwrap_number(stack[B]) == (double) CS);
     vm_pcnext();
 
     VM_OP(JOP_NOT_EQUALS)
@@ -806,7 +818,7 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in) {
     vm_pcnext();
 
     VM_OP(JOP_NOT_EQUALS_IMMEDIATE)
-    stack[A] = janet_wrap_boolean(janet_unwrap_integer(stack[B]) != CS);
+    stack[A] = janet_wrap_boolean(janet_unwrap_number(stack[B]) != (double) CS);
     vm_pcnext();
 
     VM_OP(JOP_COMPARE)
@@ -815,7 +827,11 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in) {
 
     VM_OP(JOP_NEXT)
     vm_commit();
-    stack[A] = janet_next(stack[B], stack[C]);
+    {
+        Janet temp = janet_next_impl(stack[B], stack[C], 1);
+        vm_restore();
+        stack[A] = temp;
+    }
     vm_pcnext();
 
     VM_OP(JOP_LOAD_NIL)
@@ -1272,7 +1288,14 @@ Janet janet_call(JanetFunction *fun, int32_t argc, const Janet *argv) {
     /* Push frame */
     janet_fiber_pushn(janet_vm_fiber, argv, argc);
     if (janet_fiber_funcframe(janet_vm_fiber, fun)) {
-        janet_panicf("arity mismatch in %v", janet_wrap_function(fun));
+        int32_t min = fun->def->min_arity;
+        int32_t max = fun->def->max_arity;
+        Janet funv = janet_wrap_function(fun);
+        if (min == max && min != argc)
+            janet_panicf("arity mismatch in %v, expected %d, got %d", funv, min, argc);
+        if (min >= 0 && argc < min)
+            janet_panicf("arity mismatch in %v, expected at least %d, got %d", funv, min, argc);
+        janet_panicf("arity mismatch in %v, expected at most %d, got %d", funv, max, argc);
     }
     janet_fiber_frame(janet_vm_fiber)->flags |= JANET_STACKFRAME_ENTRANCE;
 
@@ -1341,10 +1364,14 @@ static JanetSignal janet_continue_no_check(JanetFiber *fiber, Janet in, Janet *o
     janet_fiber_did_resume(fiber);
 #endif
 
+    /* Clear last value */
+    fiber->last_value = janet_wrap_nil();
+
     /* Continue child fiber if it exists */
     if (fiber->child) {
         if (janet_vm_root_fiber == NULL) janet_vm_root_fiber = fiber;
         JanetFiber *child = fiber->child;
+        uint32_t instr = (janet_stack_frame(fiber->data + fiber->frame)->pc)[0];
         janet_vm_stackn++;
         JanetSignal sig = janet_continue(child, in, &in);
         janet_vm_stackn--;
@@ -1353,6 +1380,25 @@ static JanetSignal janet_continue_no_check(JanetFiber *fiber, Janet in, Janet *o
             *out = in;
             return sig;
         }
+        /* Check if we need any special handling for certain opcodes */
+        switch (instr & 0x7F) {
+            default:
+                break;
+            case JOP_NEXT: {
+                if (sig == JANET_SIGNAL_OK ||
+                        sig == JANET_SIGNAL_ERROR ||
+                        sig == JANET_SIGNAL_USER0 ||
+                        sig == JANET_SIGNAL_USER1 ||
+                        sig == JANET_SIGNAL_USER2 ||
+                        sig == JANET_SIGNAL_USER3 ||
+                        sig == JANET_SIGNAL_USER4) {
+                    in = janet_wrap_nil();
+                } else {
+                    in = janet_wrap_integer(0);
+                }
+                break;
+            }
+        }
         fiber->child = NULL;
     }
 
@@ -1376,7 +1422,6 @@ static JanetSignal janet_continue_no_check(JanetFiber *fiber, Janet in, Janet *o
         /* Normal setup */
         if (janet_vm_root_fiber == NULL) janet_vm_root_fiber = fiber;
         janet_vm_fiber = fiber;
-        janet_gcroot(janet_wrap_fiber(fiber));
         janet_fiber_set_status(fiber, JANET_STATUS_ALIVE);
         signal = run_vm(fiber, in);
     }
@@ -1384,8 +1429,8 @@ static JanetSignal janet_continue_no_check(JanetFiber *fiber, Janet in, Janet *o
     /* Restore */
     if (janet_vm_root_fiber == fiber) janet_vm_root_fiber = NULL;
     janet_fiber_set_status(fiber, signal);
-    janet_gcunroot(janet_wrap_fiber(fiber));
     janet_restore(&tstate);
+    fiber->last_value = tstate.payload;
     *out = tstate.payload;
 
     return signal;
@@ -1480,7 +1525,6 @@ int janet_init(void) {
     janet_vm_fiber = NULL;
     janet_vm_root_fiber = NULL;
     janet_vm_stackn = 0;
-    /* Threads */
 #ifdef JANET_THREADS
     janet_threads_init();
 #endif

src/include/janet.h

@@ -20,6 +20,8 @@
 * IN THE SOFTWARE.
 */
 
+#include "janetconf.h"
+
 #ifndef JANET_H_defined
 #define JANET_H_defined
 
@@ -35,8 +37,6 @@ extern "C" {
 
 /***** START SECTION CONFIG *****/
 
-#include "janetconf.h"
-
 #ifndef JANET_VERSION
 #define JANET_VERSION "latest"
 #endif
@@ -499,25 +499,31 @@ typedef void *JanetAbstract;
 
 /* Event Loop Types */
 #ifdef JANET_EV
-#define JANET_POLL_FLAG_CLOSED 0x1
-#define JANET_POLL_FLAG_SOCKET 0x2
-#define JANET_POLL_FLAG_IOCP 0x4
+
+#define JANET_STREAM_CLOSED 0x1
+#define JANET_STREAM_SOCKET 0x2
+#define JANET_STREAM_IOCP 0x4
+#define JANET_STREAM_READABLE 0x200
+#define JANET_STREAM_WRITABLE 0x400
+#define JANET_STREAM_ACCEPTABLE 0x800
+#define JANET_STREAM_UDPSERVER 0x1000
 
 typedef enum {
     JANET_ASYNC_EVENT_INIT,
     JANET_ASYNC_EVENT_MARK,
     JANET_ASYNC_EVENT_DEINIT,
     JANET_ASYNC_EVENT_CLOSE,
+    JANET_ASYNC_EVENT_ERR,
+    JANET_ASYNC_EVENT_HUP,
     JANET_ASYNC_EVENT_READ,
     JANET_ASYNC_EVENT_WRITE,
-    JANET_ASYNC_EVENT_TIMEOUT,
+    JANET_ASYNC_EVENT_CANCEL,
     JANET_ASYNC_EVENT_COMPLETE, /* Used on windows for IOCP */
     JANET_ASYNC_EVENT_USER
 } JanetAsyncEvent;
 
 #define JANET_ASYNC_LISTEN_READ (1 << JANET_ASYNC_EVENT_READ)
 #define JANET_ASYNC_LISTEN_WRITE (1 << JANET_ASYNC_EVENT_WRITE)
-#define JANET_ASYNC_LISTEN_SPAWNER 0x1000
 
 typedef enum {
     JANET_ASYNC_STATUS_NOT_DONE,
@@ -526,18 +532,19 @@ typedef enum {
 
 /* Typedefs */
 typedef struct JanetListenerState JanetListenerState;
-typedef struct JanetPollable JanetPollable;
+typedef struct JanetStream JanetStream;
 typedef JanetAsyncStatus(*JanetListener)(JanetListenerState *state, JanetAsyncEvent event);
 
 /* Wrapper around file descriptors and HANDLEs that can be polled. */
-struct JanetPollable {
+struct JanetStream {
     JanetHandle handle;
     uint32_t flags;
-    /* Linked list of all in-flight IO routines for this pollable */
+    /* Linked list of all in-flight IO routines for this stream */
     JanetListenerState *state;
-    /* internal - used to disallow multiple concurrent reads / writes on the same pollable.
+    const void *methods; /* Methods for this stream */
+    /* internal - used to disallow multiple concurrent reads / writes on the same stream.
      * this constraint may be lifted later but allowing such would require more internal book keeping
-     * for some implementations. You can read and write at the same time on the same pollable, though. */
+     * for some implementations. You can read and write at the same time on the same stream, though. */
     int _mask;
 };
 
@@ -545,7 +552,7 @@ struct JanetPollable {
 struct JanetListenerState {
     JanetListener machine;
     JanetFiber *fiber;
-    JanetPollable *pollable;
+    JanetStream *stream;
     void *event; /* Used to pass data from asynchronous IO event. Contents depend on both
                     implementation of the event loop and the particular event. */
 #ifdef JANET_WINDOWS
@@ -553,7 +560,7 @@ struct JanetListenerState {
     int bytes; /* Used to track how many bytes were transfered. */
 #endif
     /* internal */
-    int _index; /* not used in all implementations */
+    size_t _index;
     int _mask;
     JanetListenerState *_next;
 };
@@ -831,9 +838,15 @@ struct JanetFiber {
     JanetTable *env; /* Dynamic bindings table (usually current environment). */
     Janet *data; /* Dynamically resized stack memory */
     JanetFiber *child; /* Keep linked list of fibers for restarting pending fibers */
+    Janet last_value; /* Last returned value from a fiber */
 #ifdef JANET_EV
+    /* These fields are only relevant for fibers that are used as "root fibers" -
+     * that is, fibers that are scheduled on the event loop and behave much like threads
+     * in a multi-tasking system. It would be possible to move these fields to a new
+     * type, say "JanetTask", that as separate from fibers to save a bit of space. */
     JanetListenerState *waiting;
     uint32_t sched_id; /* Increment everytime fiber is scheduled by event loop */
+    void *supervisor_channel; /* Channel to push self to when complete */
 #endif
 };
 
@@ -1248,28 +1261,102 @@ extern enum JanetInstructionType janet_instructions[JOP_INSTRUCTION_COUNT];
 
 #ifdef JANET_EV
 
+extern JANET_API const JanetAbstractType janet_stream_type;
+
 /* Run the event loop */
 JANET_API void janet_loop(void);
 
-/* Wrapper around pollables */
-JANET_API void janet_pollable_init(JanetPollable *pollable, JanetHandle handle);
-JANET_API void janet_pollable_mark(JanetPollable *pollable);
-JANET_API void janet_pollable_deinit(JanetPollable *pollable);
+/* Wrapper around streams */
+JANET_API JanetStream *janet_stream(JanetHandle handle, uint32_t flags, const JanetMethod *methods);
+JANET_API void janet_stream_close(JanetStream *stream);
+JANET_API Janet janet_cfun_stream_close(int32_t argc, Janet *argv);
+JANET_API Janet janet_cfun_stream_read(int32_t argc, Janet *argv);
+JANET_API Janet janet_cfun_stream_chunk(int32_t argc, Janet *argv);
+JANET_API Janet janet_cfun_stream_write(int32_t argc, Janet *argv);
+JANET_API void janet_stream_flags(JanetStream *stream, uint32_t flags);
 
 /* Queue a fiber to run on the event loop */
 JANET_API void janet_schedule(JanetFiber *fiber, Janet value);
 JANET_API void janet_cancel(JanetFiber *fiber, Janet value);
 JANET_API void janet_schedule_signal(JanetFiber *fiber, Janet value, JanetSignal sig);
 
-/* Start a state machine listening for events from a pollable */
-JANET_API JanetListenerState *janet_listen(JanetPollable *pollable, JanetListener behavior, int mask, size_t size, void *user);
+/* Start a state machine listening for events from a stream */
+JANET_API JanetListenerState *janet_listen(JanetStream *stream, JanetListener behavior, int mask, size_t size, void *user);
 
 /* Shorthand for yielding to event loop in C */
 JANET_NO_RETURN JANET_API void janet_await(void);
+JANET_NO_RETURN JANET_API void janet_sleep_await(double sec);
 
 /* For use inside listeners - adds a timeout to the current fiber, such that
  * it will be resumed after sec seconds if no other event schedules the current fiber. */
-void janet_addtimeout(double sec);
+JANET_API void janet_addtimeout(double sec);
+JANET_API void janet_ev_inc_refcount(void);
+JANET_API void janet_ev_dec_refcount(void);
+
+/* Get last error from a an IO operation */
+JANET_API Janet janet_ev_lasterr(void);
+
+/* Async service for calling a function or syscall in a background thread. This is not
+ * as efficient in the slightest as using Streams but can be used for arbitrary blocking
+ * functions and syscalls. */
+
+/* Used to pass data between the main thread and worker threads for simple tasks.
+ * We could just use a pointer but this prevents malloc/free in the common case
+ * of only a handful of arguments. */
+typedef struct {
+    int tag;
+    int argi;
+    void *argp;
+    JanetFiber *fiber;
+} JanetEVGenericMessage;
+
+/* How to resume or cancel after a threaded call. Not exhaustive of the possible
+ * ways one might want to resume after returning from a threaded call, but should
+ * cover most of the common cases. For something more complicated, such as resuming
+ * with an abstract type or a struct, one should use janet_ev_threaded_call instead
+ * of janet_ev_threaded_await with a custom callback. */
+
+#define JANET_EV_TCTAG_NIL 0          /* resume with nil */
+#define JANET_EV_TCTAG_INTEGER 1      /* resume with janet_wrap_integer(argi) */
+#define JANET_EV_TCTAG_STRING 2       /* resume with janet_cstringv((const char *) argp) */
+#define JANET_EV_TCTAG_STRINGF 3      /* resume with janet_cstringv((const char *) argp), then call free on argp. */
+#define JANET_EV_TCTAG_KEYWORD 4      /* resume with janet_ckeywordv((const char *) argp) */
+#define JANET_EV_TCTAG_ERR_STRING 5   /* cancel with janet_cstringv((const char *) argp) */
+#define JANET_EV_TCTAG_ERR_STRINGF 6  /* cancel with janet_cstringv((const char *) argp), then call free on argp. */
+#define JANET_EV_TCTAG_ERR_KEYWORD 7  /* cancel with janet_ckeywordv((const char *) argp) */
+#define JANET_EV_TCTAG_BOOLEAN 8      /* resume with janet_wrap_boolean(argi) */
+
+/* Function pointer that is run in the thread pool */
+typedef JanetEVGenericMessage(*JanetThreadedSubroutine)(JanetEVGenericMessage arguments);
+
+/* Handler that is run in the main thread with the result of the JanetAsyncSubroutine */
+typedef void (*JanetThreadedCallback)(JanetEVGenericMessage return_value);
+
+/* API calls for quickly offloading some work in C to a new thread or thread pool. */
+JANET_API void janet_ev_threaded_call(JanetThreadedSubroutine fp, JanetEVGenericMessage arguments, JanetThreadedCallback cb);
+JANET_NO_RETURN JANET_API void janet_ev_threaded_await(JanetThreadedSubroutine fp, int tag, int argi, void *argp);
+
+/* Callback used by janet_ev_threaded_await */
+JANET_API void janet_ev_default_threaded_callback(JanetEVGenericMessage return_value);
+
+/* Read async from a stream */
+JANET_API void janet_ev_read(JanetStream *stream, JanetBuffer *buf, int32_t nbytes);
+JANET_API void janet_ev_readchunk(JanetStream *stream, JanetBuffer *buf, int32_t nbytes);
+#ifdef JANET_NET
+JANET_API void janet_ev_recv(JanetStream *stream, JanetBuffer *buf, int32_t nbytes, int flags);
+JANET_API void janet_ev_recvchunk(JanetStream *stream, JanetBuffer *buf, int32_t nbytes, int flags);
+JANET_API void janet_ev_recvfrom(JanetStream *stream, JanetBuffer *buf, int32_t nbytes, int flags);
+#endif
+
+/* Write async to a stream */
+JANET_API void janet_ev_write_buffer(JanetStream *stream, JanetBuffer *buf);
+JANET_API void janet_ev_write_string(JanetStream *stream, JanetString str);
+#ifdef JANET_NET
+JANET_API void janet_ev_send_buffer(JanetStream *stream, JanetBuffer *buf, int flags);
+JANET_API void janet_ev_send_string(JanetStream *stream, JanetString str, int flags);
+JANET_API void janet_ev_sendto_buffer(JanetStream *stream, JanetBuffer *buf, void *dest, int flags);
+JANET_API void janet_ev_sendto_string(JanetStream *stream, JanetString str, void *dest, int flags);
+#endif
 
 #endif
 
@@ -1280,6 +1367,7 @@ JANET_API void janet_parser_deinit(JanetParser *parser);
 JANET_API void janet_parser_consume(JanetParser *parser, uint8_t c);
 JANET_API enum JanetParserStatus janet_parser_status(JanetParser *parser);
 JANET_API Janet janet_parser_produce(JanetParser *parser);
+JANET_API Janet janet_parser_produce_wrapped(JanetParser *parser);
 JANET_API const char *janet_parser_error(JanetParser *parser);
 JANET_API void janet_parser_flush(JanetParser *parser);
 JANET_API void janet_parser_eof(JanetParser *parser);
@@ -1319,6 +1407,7 @@ JANET_API JanetCompileResult janet_compile(Janet source, JanetTable *env, JanetS
 
 /* Get the default environment for janet */
 JANET_API JanetTable *janet_core_env(JanetTable *replacements);
+JANET_API JanetTable *janet_core_lookup_table(JanetTable *replacements);
 
 JANET_API int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char *sourcePath, Janet *out);
 JANET_API int janet_dostring(JanetTable *env, const char *str, const char *sourcePath, Janet *out);
@@ -1581,6 +1670,9 @@ JANET_API Janet janet_resolve_core(const char *name);
 
 /* New C API */
 
+/* Shorthand for janet C function declarations */
+#define JANET_CFUN(name) Janet name (int32_t argc, Janet *argv)
+
 /* Allow setting entry name for static libraries */
 #ifdef __cplusplus
 #define JANET_MODULE_PREFIX extern "C"
@@ -1611,6 +1703,7 @@ JANET_API void janet_arity(int32_t arity, int32_t min, int32_t max);
 JANET_API void janet_fixarity(int32_t arity, int32_t fix);
 
 JANET_API int janet_getmethod(JanetKeyword method, const JanetMethod *methods, Janet *out);
+JANET_API Janet janet_nextmethod(const JanetMethod *methods, Janet key);
 
 JANET_API double janet_getnumber(const Janet *argv, int32_t n);
 JANET_API JanetArray *janet_getarray(const Janet *argv, int32_t n);
@@ -1688,6 +1781,7 @@ JANET_API FILE *janet_dynfile(const char *name, FILE *def);
 JANET_API JanetFile *janet_getjfile(const Janet *argv, int32_t n);
 JANET_API JanetAbstract janet_checkfile(Janet j);
 JANET_API FILE *janet_unwrapfile(Janet j, int32_t *flags);
+JANET_API int janet_file_close(JanetFile *file);
 
 JANET_API int janet_cryptorand(uint8_t *out, size_t n);
 
@@ -1746,13 +1840,16 @@ typedef enum {
     RULE_THRU,         /* [rule] */
     RULE_LENPREFIX,    /* [rule_a, rule_b (repeat rule_b rule_a times)] */
     RULE_READINT,      /* [(signedness << 4) | (endianess << 5) | bytewidth, tag] */
-} JanetPegOpcode;
+    RULE_LINE,         /* [tag] */
+    RULE_COLUMN        /* [tag] */
+} JanetPegOpcod;
 
 typedef struct {
     uint32_t *bytecode;
     Janet *constants;
     size_t bytecode_len;
     uint32_t num_constants;
+    int has_backref;
 } JanetPeg;
 
 #endif
@@ -1837,6 +1934,7 @@ extern JANET_API const JanetAbstractType janet_thread_type;
 
 JANET_API int janet_thread_receive(Janet *msg_out, double timeout);
 JANET_API int janet_thread_send(JanetThread *thread, Janet msg, double timeout);
+JANET_API JanetThread *janet_thread_current(void);
 
 #endif
 

src/mainclient/shell.c

@@ -126,28 +126,21 @@ https://github.com/antirez/linenoise/blob/master/linenoise.c
 #define JANET_LINE_MAX 1024
 #define JANET_MATCH_MAX 256
 #define JANET_HISTORY_MAX 100
-static int gbl_israwmode = 0;
-static const char *gbl_prompt = "> ";
-static int gbl_plen = 2;
-static char gbl_buf[JANET_LINE_MAX];
-static int gbl_len = 0;
-static int gbl_pos = 0;
-static int gbl_cols = 80;
-static char *gbl_history[JANET_HISTORY_MAX];
-static int gbl_history_count = 0;
-static int gbl_historyi = 0;
-static int gbl_sigint_flag = 0;
-static struct termios gbl_termios_start;
-static JanetByteView gbl_matches[JANET_MATCH_MAX];
-static int gbl_match_count = 0;
-static int gbl_lines_below = 0;
-
-/* Put a lock around this global state so we don't screw up
- * the terminal in a multithreaded situation */
-#ifndef JANET_SINGLE_THREADED
-#include <pthread.h>
-static pthread_mutex_t gbl_lock = PTHREAD_MUTEX_INITIALIZER;
-#endif
+static JANET_THREAD_LOCAL int gbl_israwmode = 0;
+static JANET_THREAD_LOCAL const char *gbl_prompt = "> ";
+static JANET_THREAD_LOCAL int gbl_plen = 2;
+static JANET_THREAD_LOCAL char gbl_buf[JANET_LINE_MAX];
+static JANET_THREAD_LOCAL int gbl_len = 0;
+static JANET_THREAD_LOCAL int gbl_pos = 0;
+static JANET_THREAD_LOCAL int gbl_cols = 80;
+static JANET_THREAD_LOCAL char *gbl_history[JANET_HISTORY_MAX];
+static JANET_THREAD_LOCAL int gbl_history_count = 0;
+static JANET_THREAD_LOCAL int gbl_historyi = 0;
+static JANET_THREAD_LOCAL int gbl_sigint_flag = 0;
+static JANET_THREAD_LOCAL struct termios gbl_termios_start;
+static JANET_THREAD_LOCAL JanetByteView gbl_matches[JANET_MATCH_MAX];
+static JANET_THREAD_LOCAL int gbl_match_count = 0;
+static JANET_THREAD_LOCAL int gbl_lines_below = 0;
 
 /* Unsupported terminal list from linenoise */
 static const char *badterms[] = {
@@ -169,9 +162,6 @@ static char *sdup(const char *s) {
 /* Ansi terminal raw mode */
 static int rawmode(void) {
     struct termios t;
-#ifndef JANET_SINGLE_THREADED
-    pthread_mutex_lock(&gbl_lock);
-#endif
     if (!isatty(STDIN_FILENO)) goto fatal;
     if (tcgetattr(STDIN_FILENO, &gbl_termios_start) == -1) goto fatal;
     t = gbl_termios_start;
@@ -185,9 +175,6 @@ static int rawmode(void) {
     return 0;
 fatal:
     errno = ENOTTY;
-#ifndef JANET_SINGLE_THREADED
-    pthread_mutex_unlock(&gbl_lock);
-#endif
     return -1;
 }
 
@@ -195,9 +182,6 @@ fatal:
 static void norawmode(void) {
     if (gbl_israwmode && tcsetattr(STDIN_FILENO, TCSADRAIN, &gbl_termios_start) != -1)
         gbl_israwmode = 0;
-#ifndef JANET_SINGLE_THREADED
-    pthread_mutex_unlock(&gbl_lock);
-#endif
 }
 
 static int curpos(void) {
@@ -774,6 +758,10 @@ static int line() {
                 kleft();
                 break;
             case 3:     /* ctrl-c */
+                clearlines();
+                gbl_sigint_flag = 1;
+                return -1;
+            case 17:    /* ctrl-q */
                 gbl_cancel_current_repl_form = 1;
                 clearlines();
                 return -1;
@@ -1054,19 +1042,23 @@ int main(int argc, char **argv) {
     janet_table_put(env, janet_ckeywordv("executable"), janet_cstringv(argv[0]));
 
     /* Run startup script */
-    Janet mainfun, out;
+    Janet mainfun;
     janet_resolve(env, janet_csymbol("cli-main"), &mainfun);
     Janet mainargs[1] = { janet_wrap_array(args) };
     JanetFiber *fiber = janet_fiber(janet_unwrap_function(mainfun), 64, 1, mainargs);
     fiber->env = env;
+
+#ifdef JANET_EV
+    janet_gcroot(janet_wrap_fiber(fiber));
+    janet_schedule(fiber, janet_wrap_nil());
+    janet_loop();
+    status = janet_fiber_status(fiber);
+#else
+    Janet out;
     status = janet_continue(fiber, janet_wrap_nil(), &out);
     if (status != JANET_SIGNAL_OK && status != JANET_SIGNAL_EVENT) {
         janet_stacktrace(fiber, out);
     }
-
-#ifdef JANET_EV
-    status = JANET_SIGNAL_OK;
-    janet_loop();
 #endif
 
     /* Deinitialize vm */

test/helper.janet

@@ -13,10 +13,10 @@
   (when x (++ num-tests-passed))
   (def str (string e))
   (def truncated
-    (if (> (length e) 40) (string (string/slice e 0 35) "...") (string e)))
+    (if (> (length e) 40) (string (string/slice e 0 35) "...") (describe e)))
   (if x
-    (xprintf stdout "\e[32m✔\e[0m %s: %v" truncated x)
-    (xprintf stdout "\n\e[31m✘\e[0m %s: %v" truncated x))
+    (eprintf "\e[32m✔\e[0m %s: %v" truncated x)
+    (eprintf "\n\e[31m✘\e[0m %s: %v" truncated x))
   x)
 
 (defmacro assert-error
@@ -32,10 +32,10 @@
 (defn start-suite [x]
   (set suite-num x)
   (set start-time (os/clock))
-  (print "\nRunning test suite " x " tests...\n  "))
+  (eprint "\nRunning test suite " x " tests...\n  "))
 
 (defn end-suite []
   (def delta (- (os/clock) start-time))
-  (printf "\n\nTest suite %d finished in %.3f seconds" suite-num delta)
-  (print num-tests-passed " of " num-tests-run " tests passed.\n")
+  (eprintf "\n\nTest suite %d finished in %.3f seconds" suite-num delta)
+  (eprint num-tests-passed " of " num-tests-run " tests passed.\n")
   (if (not= num-tests-passed num-tests-run) (os/exit 1)))

test/install/test/test1.janet

@@ -1,3 +1,3 @@
-(import build/testmod :as testmod)
+(import /build/testmod :as testmod)
 
 (if (not= 5 (testmod/get5)) (error "testmod/get5 failed"))

test/install/testexec.janet

@@ -1,7 +1,7 @@
-(use build/testmod)
-(use build/testmod2)
-(use build/testmod3)
-(use build/test-mod-4)
+(use /build/testmod)
+(use /build/testmod2)
+(use /build/testmod3)
+(use /build/test-mod-4)
 
 (defn main [&]
   (print "Hello from executable!")

test/suite0001.janet

@@ -281,4 +281,17 @@
 (assert (not (even? -10.1)) "even? 8")
 (assert (not (even? -10.6)) "even? 9")
 
+# Map arities
+(assert (deep= (map inc [1 2 3]) @[2 3 4]))
+(assert (deep= (map + [1 2 3] [10 20 30]) @[11 22 33]))
+(assert (deep= (map + [1 2 3] [10 20 30] [100 200 300]) @[111 222 333]))
+(assert (deep= (map + [1 2 3] [10 20 30] [100 200 300] [1000 2000 3000]) @[1111 2222 3333]))
+(assert (deep= (map + [1 2 3] [10 20 30] [100 200 300] [1000 2000 3000] [10000 20000 30000]) @[11111 22222 33333]))
+
+# Sort function
+(assert (deep=
+          (range 99)
+          (sort (mapcat (fn [[x y z]] [z y x]) (partition 3 (range 99))))) "sort 5")
+(assert (<= ;(sort (map (fn [x] (math/random)) (range 1000)))) "sort 6")
+
 (end-suite)

test/suite0002.janet

@@ -49,7 +49,7 @@
 # Generators
 (def gen (generate [x :range [0 100] :when (pos? (% x 4))] x))
 (var gencount 0)
-(loop [x :generate gen]
+(loop [x :in gen]
   (++ gencount)
   (assert (pos? (% x 4)) "generate in loop"))
 (assert (= gencount 75) "generate loop count")

test/suite0003.janet

@@ -360,6 +360,14 @@
 (check-match janet-longstring "``` `` ```" true)
 (check-match janet-longstring "``  ```" false)
 
+# Line and column capture
+
+(def line-col (peg/compile '(any (* (line) (column) 1))))
+(check-deep line-col "abcd" @[1 1 1 2 1 3 1 4])
+(check-deep line-col "" @[])
+(check-deep line-col "abcd\n" @[1 1 1 2 1 3 1 4 1 5])
+(check-deep line-col "abcd\nz" @[1 1 1 2 1 3 1 4 1 5 2 1])
+
 # Backmatch
 
 (def backmatcher-1 '(* (capture (any "x") :1) "y" (backmatch :1) -1))

test/suite0006.janet

@@ -128,6 +128,18 @@
 (assert (not= nil (parse-error @"\xc3\x28")) "reject invalid utf-8 symbol")
 (assert (not= nil (parse-error @":\xc3\x28")) "reject invalid utf-8 keyword")
 
+# Parser line and column numbers
+(defn parser-location [input &opt location]
+  (def p (parser/new))
+  (parser/consume p input)
+  (if location
+    (parser/where p ;location)
+    (parser/where p)))
+
+(assert (= [1 7] (parser-location @"(+ 1 2)")) "parser location 1")
+(assert (= [5 7] (parser-location @"(+ 1 2)" [5])) "parser location 2")
+(assert (= [10 10] (parser-location @"(+ 1 2)" [10 10])) "parser location 3")
+
 # String check-set
 (assert (string/check-set "abc" "a") "string/check-set 1")
 (assert (not (string/check-set "abc" "z")) "string/check-set 2")
@@ -178,4 +190,39 @@
 (assert (= (test-expand "../def.txt" ":all:") "../def.txt") "module/expand-path 7")
 (assert (= (test-expand "../././././abcd/../def.txt" ":all:") "../def.txt") "module/expand-path 8")
 
+# Integer type checks
+(assert (compare= 0 (- (int/u64 "1000") 1000)) "subtract from int/u64")
+
+(assert (odd? (int/u64 "1001")) "odd? 1")
+(assert (not (odd? (int/u64 "1000"))) "odd? 2")
+(assert (odd? (int/s64 "1001")) "odd? 3")
+(assert (not (odd? (int/s64 "1000"))) "odd? 4")
+(assert (odd? (int/s64 "-1001")) "odd? 5")
+(assert (not (odd? (int/s64 "-1000"))) "odd? 6")
+
+(assert (even? (int/u64 "1000")) "even? 1")
+(assert (not (even? (int/u64 "1001"))) "even? 2")
+(assert (even? (int/s64 "1000")) "even? 3")
+(assert (not (even? (int/s64 "1001"))) "even? 4")
+(assert (even? (int/s64 "-1000")) "even? 5")
+(assert (not (even? (int/s64 "-1001"))) "even? 6")
+
+# integer type operations
+(defn modcheck [x y]
+  (assert (= (string (mod x y)) (string (mod (int/s64 x) y)))
+          (string "int/s64 (mod " x " " y ") expected " (mod x y) ", got "
+                  (mod (int/s64 x) y)))
+  (assert (= (string (% x y)) (string (% (int/s64 x) y)))
+          (string "int/s64 (% " x " " y ") expected " (% x y) ", got "
+                  (% (int/s64 x) y))))
+
+(modcheck 1 2)
+(modcheck 1 3)
+(modcheck 4 2)
+(modcheck 4 1)
+(modcheck 10 3)
+(modcheck 10 -3)
+(modcheck -10 3)
+(modcheck -10 -3)
+
 (end-suite)

test/suite0007.janet

@@ -320,4 +320,8 @@
   (array/push a x))
 (assert (deep= (range 4) a) "eachk 1")
 
+
+(tracev (def my-unique-var-name true))
+(assert my-unique-var-name "tracev upscopes")
+
 (end-suite)

test/suite0008.janet

@@ -125,6 +125,7 @@
 (assert (= :yes (match {:a 1} {:a _} :yes :no)) "match wildcard 5")
 (assert (= false (match {:a 1 :b 2 :c 3} {:a a :b _ :c _ :d _} :no {:a _ :b _ :c _} false :no)) "match wildcard 6")
 (assert (= nil (match {:a 1 :b 2 :c 3} {:a a :b _ :c _ :d _} :no {:a _ :b _ :c _} nil :no)) "match wildcard 7")
+(assert (= "t" (match [true nil] [true _] "t")) "match wildcard 8")
 
 # Regression #301
 (def b (buffer/new-filled 128 0x78))
@@ -221,20 +222,6 @@
 neldb\0\0\0\xD8\x05printG\x01\0\xDE\xDE\xDE'\x03\0marshal_tes/\x02
 \0\0\0\0\0*\xFE\x01\04\x02\0\0'\x03\0\r\0\r\0\r\0\r" load-image-dict))
 
-# No segfault, valgrind clean.
-
-(def x @"\xCC\xCD.nd\x80\0\r\x1C\xCDg!\0\x07\xCC\xCD\r\x1Ce\x10\0\r;\xCDb\x04\xFF9\xFF\x80\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04uu\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\0\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04}\x04\x04\x04\x04\x04\x04\x04\x04#\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\0\x01\0\0\x03\x04\x04\x04\xE2\x03\x04\x04\x04\x04\x04\x04\x04\x04\x04\x14\x1A\x04\x04\x04\x04\x04\x18\x04\x04!\x04\xE2\x03\x04\x04\x04\x04\x04\x04$\x04\x04\x04\x04\x04\x04\x04\x04\x04\x80\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04A\0\0\0\x03\0\0!\xBF\xFF")
-(assert-error "bad fiber status" (unmarshal x load-image-dict))
-(gccollect)
-(marshal x make-image-dict)
-
-(def b @"\xCC\xCD\0\x03\0\x08\x04\rm\xCD\x7F\xFF\xFF\xFF\x02\0\x02\xD7\xCD\0\x98\0\0\x05\x01\x01\x01\x01\x08\xCE\x01f\xCE../tools/afl/generate_unmarshal_testcases.janet\xCE\x012,\x01\0\0&\x03\0\06\x02\x03\x03)\x03\x01\0*\x04\0\00\x03\x04\0>\x03\0\0\x03\x03\0\0*\x05\0\x11\0\x11\0\x05\0\x05\0\x05\0\x05\0\x05\xC9\xDA\x04\xC9\xC9\xC9")
-(unmarshal b load-image-dict)
-(gccollect)
-
-(def v (unmarshal
-         @"\xD7\xCD0\xD4000000\0\x03\x01\xCE\00\0\x01\0\0000\x03\0\0\0000000000\xCC0\0000"
-         load-image-dict))
 (gccollect)
 
 # in vs get regression
@@ -322,7 +309,7 @@ neldb\0\0\0\xD8\x05printG\x01\0\xDE\xDE\xDE'\x03\0marshal_tes/\x02
 # Issue 428
 (var result nil)
 (defn f [] (yield {:a :ok}))
-(assert-no-error "issue 428 1" (loop [{:a x} :generate (fiber/new f)] (set result x)))
+(assert-no-error "issue 428 1" (loop [{:a x} :in (fiber/new f)] (set result x)))
 (assert (= result :ok) "issue 428 2")
 
 # Inline 3 argument get
@@ -349,4 +336,12 @@ neldb\0\0\0\xD8\x05printG\x01\0\xDE\xDE\xDE'\x03\0marshal_tes/\x02
 (check-replacer "aba" "ZZZZZZ" "ababababababa")
 (check-replacer "aba" "" "ababababababa")
 
+# Peg bug
+(assert (deep= @[] (peg/match '(any 1) @"")) "peg empty pattern 1")
+(assert (deep= @[] (peg/match '(any 1) (buffer))) "peg empty pattern 2")
+(assert (deep= @[] (peg/match '(any 1) "")) "peg empty pattern 3")
+(assert (deep= @[] (peg/match '(any 1) (string))) "peg empty pattern 4")
+(assert (deep= @[] (peg/match '(* "test" (any 1)) @"test")) "peg empty pattern 5")
+(assert (deep= @[] (peg/match '(* "test" (any 1)) (buffer "test"))) "peg empty pattern 6")
+
 (end-suite)

test/suite0009.janet

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Calvin Rose & contributors
+# Copyright (c) 2021 Calvin Rose & contributors
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to
@@ -21,38 +21,146 @@
 (import ./helper :prefix "" :exit true)
 (start-suite 9)
 
+# Subprocess
+
+(def janet (dyn :executable))
+
+(repeat 10
+
+  (let [p (os/spawn [janet "-e" `(print "hello")`] :p {:out :pipe})]
+    (os/proc-wait p)
+    (def x (:read (p :out) :all))
+    (assert (deep= "hello" (string/trim x)) "capture stdout from os/spawn pre close."))
+
+  (let [p (os/spawn [janet "-e" `(print "hello")`] :p {:out :pipe})]
+    (def x (:read (p :out) 1024))
+    (os/proc-wait p)
+    (assert (deep= "hello" (string/trim x)) "capture stdout from os/spawn post close."))
+
+  (let [p (os/spawn [janet "-e" `(file/read stdin :line)`] :px {:in :pipe})]
+    (:write (p :in) "hello!\n")
+    (assert-no-error "pipe stdin to process" (os/proc-wait p))))
+
+(let [p (os/spawn [janet "-e" `(print (file/read stdin :line))`] :px {:in :pipe :out :pipe})]
+  (:write (p :in) "hello!\n")
+  (def x (:read (p :out) 1024))
+  (assert-no-error "pipe stdin to process 2" (os/proc-wait p))
+  (assert (= "hello!" (string/trim x)) "round trip pipeline in process"))
+
+# Parallel subprocesses
+
+(defn calc-1
+  "Run subprocess, read from stdout, then wait on subprocess."
+  [code]
+  (let [p (os/spawn [janet "-e" (string `(printf "%j" ` code `)`)] :px {:out :pipe})]
+    (os/proc-wait p)
+    (def output (:read (p :out) :all))
+    (parse output)))
+
+(assert
+  (deep=
+    (ev/gather
+      (calc-1 "(+ 1 2 3 4)")
+      (calc-1 "(+ 5 6 7 8)")
+      (calc-1 "(+ 9 10 11 12)"))
+    @[10 26 42]) "parallel subprocesses 1")
+
+(defn calc-2
+  "Run subprocess, wait on subprocess, then read from stdout. Read only up to 10 bytes instead of :all"
+  [code]
+  (let [p (os/spawn [janet "-e" (string `(printf "%j" ` code `)`)] :px {:out :pipe})]
+    (def output (:read (p :out) 10))
+    (os/proc-wait p)
+    (parse output)))
+
+(assert
+  (deep=
+    (ev/gather
+      (calc-2 "(+ 1 2 3 4)")
+      (calc-2 "(+ 5 6 7 8)")
+      (calc-2 "(+ 9 10 11 12)"))
+    @[10 26 42]) "parallel subprocesses 2")
+
+# File piping
+
+(assert-no-error "file writing 1"
+  (with [f (file/temp)]
+    (os/execute [janet "-e" `(repeat 20 (print :hello))`] :p {:out f})))
+
+(assert-no-error "file writing 2"
+  (with [f (file/open "unique.txt" :w)]
+    (os/execute [janet "-e" `(repeat 20 (print :hello))`] :p {:out f})
+    (file/flush f)))
+
+# Issue #593
+(assert-no-error "file writing 3"
+  (def outfile (file/open "unique.txt" :w))
+  (os/execute [janet "-e" "(pp (seq [i :range (1 10)] i))"] :p {:out outfile})
+  (file/flush outfile)
+  (file/close outfile)
+  (os/rm "unique.txt"))
+
+# ev/gather
+
+(assert (deep= @[1 2 3] (ev/gather 1 2 3)) "ev/gather 1")
+(assert (deep= @[] (ev/gather)) "ev/gather 2")
+(assert-error "ev/gather 3" (ev/gather 1 2 (error 3)))
+
 # Net testing
 
-(defn handler
-  "Simple handler for connections."
-  [stream]
-  (defer (:close stream)
-    (def id (gensym))
-    (def b @"")
-    (:read stream 1024 b)
-    (:write stream b)
-    (buffer/clear b)))
+(repeat 10
 
-(def s (net/server "127.0.0.1" "8000"))
-(assert s "made server 1")
+  (defn handler
+    "Simple handler for connections."
+    [stream]
+    (defer (:close stream)
+      (def id (gensym))
+      (def b @"")
+      (net/read stream 1024 b)
+      (net/write stream b)
+      (buffer/clear b)))
 
-(ev/go
-  (coro
-    (while (not (net/closed? s))
-      (def conn (net/accept s))
-      (unless conn (break))
-      (ev/call handler conn))))
+  (def s (net/server "127.0.0.1" "8000" handler))
+  (assert s "made server 1")
 
-(defn test-echo [msg]
-  (with [conn (net/connect "127.0.0.1" "8000")]
-    (:write conn msg)
-    (def res (:read conn 1024))
-    (assert (= (string res) msg) (string "echo " msg))))
+  (defn test-echo [msg]
+    (with [conn (net/connect "127.0.0.1" "8000")]
+      (net/write conn msg)
+      (def res (net/read conn 1024))
+      (assert (= (string res) msg) (string "echo " msg))))
 
-(test-echo "hello")
-(test-echo "world")
-(test-echo (string/repeat "abcd" 200))
+  (test-echo "hello")
+  (test-echo "world")
+  (test-echo (string/repeat "abcd" 200))
 
-(:close s)
+  (:close s))
+
+# Create pipe
+
+(var pipe-counter 0)
+(def chan (ev/chan 10))
+(let [[reader writer] (os/pipe)]
+  (ev/spawn
+    (while (ev/read reader 3)
+      (++ pipe-counter))
+    (assert (= 20 pipe-counter) "ev/pipe 1")
+    (ev/give chan 1))
+
+  (for i 0 10
+    (ev/write writer "xxx---"))
+
+  (ev/close writer)
+  (ev/take chan))
+
+(var result nil)
+(var fiber nil)
+(set fiber
+  (ev/spawn
+    (set result (protect (ev/sleep 10)))
+    (assert (= result '(false "boop")) "ev/cancel 1")))
+(ev/sleep 0)
+(ev/cancel fiber "boop")
+
+(assert (os/execute [janet "-e" `(+ 1 2 3)`] :xp) "os/execute self")
 
 (end-suite)

test/suite0010.janet

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Calvin Rose & contributors
+#- Copyright (c) 2020 Calvin Rose & contributors
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to
@@ -65,4 +65,82 @@
 (assert (= :brackets (tuple/type (postwalk identity '[]))) "walk square brackets 1")
 (assert (= :brackets (tuple/type (walk identity '[]))) "walk square brackets 2")
 
+# # off by 1 error in inttypes
+(assert (= (int/s64 "-0x8000_0000_0000_0000") (+ (int/s64 "0x7FFF_FFFF_FFFF_FFFF") 1)) "int types wrap around")
+
+#
+# Longstring indentation
+#
+
+(defn reindent
+  "Reindent a the contents of a longstring as the Janet parser would.
+  This include removing leading and trailing newlines."
+  [text indent]
+
+  # Detect minimum indent
+  (var rewrite true)
+  (each index (string/find-all "\n" text)
+    (for i (+ index 1) (+ index indent 1)
+      (case (get text i)
+        nil (break)
+        (chr "\n") (break)
+        (chr " ") nil
+        (set rewrite false))))
+
+  # Only re-indent if no dedented characters.
+  (def str
+    (if rewrite
+      (peg/replace-all ~(* "\n" (between 0 ,indent " ")) "\n" text)
+      text))
+
+  (def first-nl (= (chr "\n") (first str)))
+  (def last-nl (= (chr "\n") (last str)))
+  (string/slice str (if first-nl 1 0) (if last-nl -2)))
+
+(defn reindent-reference
+  "Same as reindent but use parser functionality. Useful for validating conformance."
+  [text indent]
+  (if (empty? text) (break text))
+  (def source-code
+    (string (string/repeat " " indent) "``````"
+            text
+            "``````"))
+  (parse source-code))
+
+(var indent-counter 0)
+(defn check-indent
+  [text indent]
+  (++ indent-counter)
+  (let [a (reindent text indent)
+        b (reindent-reference text indent)]
+    (assert (= a b) (string "indent " indent-counter " (indent=" indent ")"))))
+
+(check-indent "" 0)
+(check-indent "\n" 0)
+(check-indent "\n" 1)
+(check-indent "\n\n" 0)
+(check-indent "\n\n" 1)
+(check-indent "\nHello, world!" 0)
+(check-indent "\nHello, world!" 1)
+(check-indent "Hello, world!" 0)
+(check-indent "Hello, world!" 1)
+(check-indent "\n    Hello, world!" 4)
+(check-indent "\n    Hello, world!\n" 4)
+(check-indent "\n    Hello, world!\n   " 4)
+(check-indent "\n    Hello, world!\n    " 4)
+(check-indent "\n    Hello, world!\n   dedented text\n    " 4)
+(check-indent "\n    Hello, world!\n    indented text\n    " 4)
+
+# String bugs
+(assert (deep= (string/find-all "qq" "qqq") @[0 1]) "string/find-all 1")
+(assert (deep= (string/find-all "q" "qqq") @[0 1 2]) "string/find-all 2")
+(assert (deep= (string/split "qq" "1qqqqz") @["1" "" "z"]) "string/split 1")
+(assert (deep= (string/split "aa" "aaa") @["" "a"]) "string/split 2")
+
+# Comparisons
+(assert (> 1e23 100) "less than immediate 1")
+(assert (> 1e23 1000) "less than immediate 2")
+(assert (< 100 1e23) "greater than immediate 1")
+(assert (< 1000 1e23) "greater than immediate 2")
+
 (end-suite)

tools/msi/janet.wxs

@@ -126,9 +126,6 @@
                 <File Source="dist\janet.h"/>
                 <RemoveFolder Id="RemoveCDir" On="uninstall" />
             </Component>
-            <Component Directory="CDir">
-                <File Source="dist\janetconf.h"/>
-            </Component>
             <Component Directory="CDir">
                 <File Source="dist\janet.lib"/>
             </Component>

tools/patch-header.janet

@@ -0,0 +1,3 @@
+# Patch janet.h 
+(def [_ janeth janetconf output] (dyn :args))
+(spit output (string/replace `#include "janetconf.h"` (slurp janetconf) (slurp janeth)))