Add #1737

Add some test cases and asserts for new pretty printing - we were not
properly handing the case when print formatter was not at the start of
the buffer.

A simple fix is to add a lookback_barrier, which is an index into the
buffer that backtracking print formatter can't go past. This ensures
that, whatever newlines are replace with spaces, we don't mess with any
explicit newlines that should be printed.

Also some tweaks to logic to try an make invariants more obvious and
simplify looping and indexing.

CHANGELOG.md

@@ -2,9 +2,13 @@
 All notable changes to this project will be documented in this file.
 
 ## Unreleased - ???
+- Improve pretty printing layout for %M and %m modifiers to be more code-like.
+- Add filewatch support to BSD and macos.
+- Add linting support for shadowed bindings.
+- Add nanboxing support for Linux on ARM64 and turn on nanboxing by default on macos on ARM64 (aarch64).
 - Documentation fixes
 - ev/thread-chan deadlock bug fixed
-- Re-add removed support for non-blocking net/connect on windows.
+- Re-add removed support for non-blocking net/connect on windows with bug fixes.
 
 ## 1.41.2 - 2026-02-18
 - Fix regressions in `put` for arrays and buffers.

examples/filewatch.janet

@@ -0,0 +1,14 @@
+###
+### example/filewatch.janet ...files
+###
+### Watch for all changes in a list of files and directories. Behavior
+### depends on the filewatch module, and different operating systems will
+### report different events.
+
+(def chan (ev/chan 1000))
+(def fw (filewatch/new chan))
+(each arg (drop 1 (dyn *args* []))
+    (filewatch/add fw arg :all))
+(filewatch/listen fw)
+
+(forever (let [event (ev/take chan)] (pp event)))

src/core/ev.c

@@ -1962,7 +1962,7 @@ void janet_stream_level_triggered(JanetStream *stream) {
     janet_register_stream_impl(stream, 0);
 }
 
-#define JANET_KQUEUE_MAX_EVENTS 64
+#define JANET_KQUEUE_MAX_EVENTS 512
 
 void janet_loop1_impl(int has_timeout, JanetTimestamp timeout) {
     /* Poll for events */
@@ -2026,6 +2026,7 @@ void janet_loop1_impl(int has_timeout, JanetTimestamp timeout) {
 
 void janet_ev_init(void) {
     janet_ev_init_common();
+    /* TODO - replace selfpipe with EVFILT_USER (or other events) */
     janet_ev_setup_selfpipe();
     janet_vm.kq = kqueue();
     janet_vm.timer_enabled = 0;

src/core/filewatch.c

@@ -38,6 +38,13 @@
 #include <windows.h>
 #endif
 
+#if defined(JANET_APPLE) || defined(JANET_BSD)
+#include <sys/event.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#endif
+
 typedef struct {
     const char *name;
     uint32_t flag;
@@ -89,7 +96,7 @@ static uint32_t decode_watch_flags(const Janet *options, int32_t n) {
                                            sizeof(JanetWatchFlagName),
                                            keyw);
         if (!result) {
-            janet_panicf("unknown inotify flag %v", options[i]);
+            janet_panicf("unknown linux flag %v", options[i]);
         }
         flags |= result->flag;
     }
@@ -128,8 +135,11 @@ static void janet_watcher_add(JanetWatcher *watcher, const char *path, uint32_t
 
 static void janet_watcher_remove(JanetWatcher *watcher, const char *path) {
     if (watcher->stream == NULL) janet_panic("watcher closed");
-    Janet check = janet_table_get(watcher->watch_descriptors, janet_cstringv(path));
+    Janet pathv = janet_cstringv(path);
-    janet_assert(janet_checktype(check, JANET_NUMBER), "bad watch descriptor");
+    Janet check = janet_table_get(watcher->watch_descriptors, pathv);
+    if (!janet_checktype(check, JANET_NUMBER)) {
+        janet_panic("bad watch descriptor");
+    }
     int watch_handle = janet_unwrap_integer(check);
     int result;
     do {
@@ -138,6 +148,10 @@ static void janet_watcher_remove(JanetWatcher *watcher, const char *path) {
     if (result == -1) {
         janet_panicv(janet_ev_lasterr());
     }
+    /*
+    janet_table_put(watcher->watch_descriptors, pathv, janet_wrap_nil());
+    janet_table_put(watcher->watch_descriptors, janet_wrap_integer(watch_handle), janet_wrap_nil());
+    */
 }
 
 static void watcher_callback_read(JanetFiber *fiber, JanetAsyncEvent event) {
@@ -500,6 +514,254 @@ static void janet_watcher_unlisten(JanetWatcher *watcher) {
     janet_gcunroot(janet_wrap_abstract(watcher));
 }
 
+#elif defined(JANET_APPLE) || defined(JANET_BSD)
+
+/* kqueue implementation */
+
+/* Cribbed from ev.c */
+#define EV_SETx(ev, a, b, c, d, e, f) EV_SET((ev), (a), (b), (c), (d), (e), ((__typeof__((ev)->udata))(f)))
+
+/* Different BSDs define different NOTE_* constants for different kinds of events. Use ifdef to
+   determine when they are available (assuming they are defines and not enums */
+static const JanetWatchFlagName watcher_flags_kqueue[] = {
+    {
+        "all", NOTE_ATTRIB | NOTE_DELETE | NOTE_EXTEND | NOTE_RENAME | NOTE_REVOKE | NOTE_WRITE | NOTE_LINK
+#ifdef NOTE_CLOSE
+        | NOTE_CLOSE
+#endif
+#ifdef NOTE_CLOSE_WRITE
+        | NOTE_CLOSE_WRITE
+#endif
+#ifdef NOTE_OPEN
+        | NOTE_OPEN
+#endif
+#ifdef NOTE_READ
+        | NOTE_READ
+#endif
+#ifdef NOTE_FUNLOCK
+        | NOTE_FUNLOCK
+#endif
+#ifdef NOTE_TRUNCATE
+        | NOTE_TRUNCATE
+#endif
+    },
+    {"attrib", NOTE_ATTRIB},
+#ifdef NOTE_CLOSE
+    {"close", NOTE_CLOSE},
+#endif
+#ifdef NOTE_CLOSE_WRITE
+    {"close-write", NOTE_CLOSE_WRITE},
+#endif
+    {"delete", NOTE_DELETE},
+    {"extend", NOTE_EXTEND},
+#ifdef NOTE_FUNLOCK
+    {"funlock", NOTE_FUNLOCK},
+#endif
+    {"link", NOTE_LINK},
+#ifdef NOTE_OPEN
+    {"open", NOTE_OPEN},
+#endif
+#ifdef NOTE_READ
+    {"read", NOTE_READ},
+#endif
+    {"rename", NOTE_RENAME},
+    {"revoke", NOTE_REVOKE},
+#ifdef NOTE_TRUNCATE
+    {"truncate", NOTE_TRUNCATE},
+#endif
+    {"write", NOTE_WRITE},
+};
+
+static uint32_t decode_watch_flags(const Janet *options, int32_t n) {
+    uint32_t flags = 0;
+    for (int32_t i = 0; i < n; i++) {
+        if (!(janet_checktype(options[i], JANET_KEYWORD))) {
+            janet_panicf("expected keyword, got %v", options[i]);
+        }
+        JanetKeyword keyw = janet_unwrap_keyword(options[i]);
+        const JanetWatchFlagName *result = janet_strbinsearch(watcher_flags_kqueue,
+                                           sizeof(watcher_flags_kqueue) / sizeof(JanetWatchFlagName),
+                                           sizeof(JanetWatchFlagName),
+                                           keyw);
+        if (!result) {
+            janet_panicf("unknown bsd flag %v", options[i]);
+        }
+        flags |= result->flag;
+    }
+    return flags;
+}
+
+static void janet_watcher_init(JanetWatcher *watcher, JanetChannel *channel, uint32_t default_flags) {
+    int kq = kqueue();
+    watcher->watch_descriptors = janet_table(0);
+    watcher->channel = channel;
+    watcher->default_flags = default_flags;
+    watcher->is_watching = 0;
+    watcher->stream = janet_stream(kq, JANET_STREAM_READABLE, NULL);
+    janet_stream_level_triggered(watcher->stream);
+}
+
+static void janet_watcher_add(JanetWatcher *watcher, const char *path, uint32_t flags) {
+    if (watcher->stream == NULL) janet_panic("watcher closed");
+    int kq = watcher->stream->handle;
+    struct kevent kev = {0};
+    /* Get file descriptor for path */
+    int file_fd;
+    do {
+        file_fd = open(path, O_RDONLY);
+    } while (file_fd == -1 && errno == EINTR);
+    if (file_fd == -1) {
+        janet_panicf("failed to open: %v", janet_ev_lasterr());
+    }
+    /* Watch for EVFILT_VNODE on the file descriptor */
+    EV_SETx(&kev, file_fd, EVFILT_VNODE, EV_ADD | EV_ENABLE | EV_CLEAR, flags, 0, NULL);
+    int status;
+    do {
+        status = kevent(kq, &kev, 1, NULL, 0, NULL);
+    } while (status == -1 && errno == EINTR);
+    if (status == -1) {
+        close(file_fd);
+        janet_panicf("failed to listen: %v", janet_ev_lasterr());
+    }
+    /* Bookkeeping */
+    Janet name = janet_cstringv(path);
+    Janet wd = janet_wrap_integer(file_fd);
+    janet_table_put(watcher->watch_descriptors, name, wd);
+    janet_table_put(watcher->watch_descriptors, wd, name);
+}
+
+static void janet_watcher_remove(JanetWatcher *watcher, const char *path) {
+    if (watcher->stream == NULL) janet_panic("watcher closed");
+    Janet pathv = janet_cstringv(path);
+    Janet check = janet_table_get(watcher->watch_descriptors, pathv);
+    if (!janet_checktype(check, JANET_NUMBER)) {
+        janet_panic("bad watch descriptor");
+    }
+    /* Closing the file descriptor will also remove it from the kqueue */
+    int wd = janet_unwrap_integer(check);
+    int result;
+    do {
+        result = close(wd);
+    } while (result != -1 && errno == EINTR);
+    if (result == -1) {
+        janet_panicv(janet_ev_lasterr());
+    }
+    janet_table_put(watcher->watch_descriptors, pathv, janet_wrap_nil());
+    janet_table_put(watcher->watch_descriptors, janet_wrap_integer(wd), janet_wrap_nil());
+}
+
+typedef struct {
+    JanetWatcher *watcher;
+    uint32_t cookie;
+} KqueueWatcherState;
+
+static void watcher_callback_read(JanetFiber *fiber, JanetAsyncEvent event) {
+    JanetStream *stream = fiber->ev_stream;
+    KqueueWatcherState *state = fiber->ev_state;
+    JanetWatcher *watcher = state->watcher;
+    switch (event) {
+        case JANET_ASYNC_EVENT_MARK:
+            janet_mark(janet_wrap_abstract(watcher));
+            break;
+        case JANET_ASYNC_EVENT_CLOSE:
+            janet_schedule(fiber, janet_wrap_nil());
+            janet_async_end(fiber);
+            break;
+        case JANET_ASYNC_EVENT_ERR: {
+            janet_schedule(fiber, janet_wrap_nil());
+            janet_async_end(fiber);
+            break;
+        }
+        case JANET_ASYNC_EVENT_HUP:
+        case JANET_ASYNC_EVENT_INIT:
+            break;
+        case JANET_ASYNC_EVENT_READ: {
+            /* Pump events from the sub kqueue */
+            const int num_events = 512; /* Extra will be pumped after another event loop rotation. */
+            struct kevent events[num_events];
+            int kq = stream->handle;
+            int status;
+            do {
+                status = kevent(kq, NULL, 0, events, num_events, NULL);
+            } while (status == -1 && errno == EINTR);
+            if (status == -1) {
+                janet_schedule(fiber, janet_wrap_nil());
+                janet_async_end(fiber);
+                break;
+            }
+            for (int i = 0; i < status; i++) {
+                state->cookie += 6700417;
+                struct kevent kev = events[i];
+                /* TODO - avoid stat call here, maybe just when adding listener? */
+                struct stat stat_buf = {0};
+                int status;
+                do {
+                    status = fstat(kev.ident, &stat_buf);
+                } while (status == -1 && errno == EINTR);
+                if (status == -1) continue;
+                int is_dir = S_ISDIR(stat_buf.st_mode);
+                Janet ident = janet_wrap_integer(kev.ident);
+                Janet path = janet_table_get(watcher->watch_descriptors, ident);
+                for (unsigned int j = 1; j < (sizeof(watcher_flags_kqueue) / sizeof(watcher_flags_kqueue[0])); j++) {
+                    uint32_t flagcheck = watcher_flags_kqueue[j].flag;
+                    if (kev.fflags & flagcheck) {
+                        JanetKV *event = janet_struct_begin(6);
+                        janet_struct_put(event, janet_ckeywordv("wd"), ident);
+                        janet_struct_put(event, janet_ckeywordv("wd-path"), path);
+                        janet_struct_put(event, janet_ckeywordv("cookie"), janet_wrap_number((double) state->cookie));
+                        janet_struct_put(event, janet_ckeywordv("type"), janet_ckeywordv(watcher_flags_kqueue[j].name));
+                        if (is_dir) {
+                            /* Pass in directly */
+                            janet_struct_put(event, janet_ckeywordv("file-name"), janet_cstringv(""));
+                            janet_struct_put(event, janet_ckeywordv("dir-name"), path);
+                        } else {
+                            /* Split path */
+                            JanetString spath = janet_unwrap_string(path);
+                            const uint8_t *cursor = spath + janet_string_length(spath);
+                            const uint8_t *cursor_end = cursor;
+                            while (cursor > spath && cursor[0] != '/') {
+                                cursor--;
+                            }
+                            if (cursor == spath) {
+                                /* No path separators */
+                                janet_struct_put(event, janet_ckeywordv("dir-name"), janet_cstringv("."));
+                                janet_struct_put(event, janet_ckeywordv("file-name"), janet_wrap_string(spath));
+                            } else {
+                                /* Found path separator */
+                                janet_struct_put(event, janet_ckeywordv("dir-name"), janet_wrap_string(janet_string(spath, (cursor - spath))));
+                                janet_struct_put(event, janet_ckeywordv("file-name"), janet_wrap_string(janet_string(cursor + 1, (cursor_end - cursor - 1))));
+                            }
+                        }
+                        Janet eventv = janet_wrap_struct(janet_struct_end(event));
+                        janet_channel_give(watcher->channel, eventv);
+                    }
+                }
+            }
+            break;
+        }
+        default:
+            break;
+    }
+}
+
+static void janet_watcher_listen(JanetWatcher *watcher) {
+    if (watcher->is_watching) janet_panic("already watching");
+    watcher->is_watching = 1;
+    JanetFunction *thunk = janet_thunk_delay(janet_wrap_nil());
+    JanetFiber *fiber = janet_fiber(thunk, 64, 0, NULL);
+    KqueueWatcherState *state = janet_malloc(sizeof(KqueueWatcherState));
+    state->watcher = watcher;
+    janet_async_start_fiber(fiber, watcher->stream, JANET_ASYNC_LISTEN_READ, watcher_callback_read, state);
+    janet_gcroot(janet_wrap_abstract(watcher));
+}
+
+static void janet_watcher_unlisten(JanetWatcher *watcher) {
+    if (!watcher->is_watching) return;
+    watcher->is_watching = 0;
+    janet_stream_close(watcher->stream);
+    janet_gcunroot(janet_wrap_abstract(watcher));
+}
+
 #else
 
 /* Default implementation */
@@ -582,10 +844,10 @@ JANET_CORE_FN(cfun_filewatch_make,
               "* `:dir-name` -- the directory name of the file that triggered the event.\n\n"
               "Events also will contain keys specific to the host OS.\n\n"
               "Windows has no extra properties on events.\n\n"
-              "Linux has the following extra properties on events:\n\n"
+              "Linux and the BSDs have the following extra properties on events:\n\n"
-              "* `:wd` -- the integer key returned by `filewatch/add` for the path that triggered this.\n\n"
+              "* `:wd` -- the integer key returned by `filewatch/add` for the path that triggered this. This is a file descriptor integer on BSD and macos.\n\n"
               "* `:wd-path` -- the string path for watched directory of file. For files, will be the same as `:file-name`, and for directories, will be the same as `:dir-name`.\n\n"
-              "* `:cookie` -- a randomized integer used to associate related events, such as :moved-from and :moved-to events.\n\n"
+              "* `:cookie` -- a semi-randomized integer used to associate related events, such as :moved-from and :moved-to events.\n\n"
               "") {
     janet_sandbox_assert(JANET_SANDBOX_FS_READ);
     janet_arity(argc, 1, -1);
@@ -600,6 +862,7 @@ JANET_CORE_FN(cfun_filewatch_add,
               "(filewatch/add watcher path flag & more-flags)",
               "Add a path to the watcher. Available flags depend on the current OS, and are as follows:\n\n"
               "Windows/MINGW (flags correspond to `FILE_NOTIFY_CHANGE_*` flags in win32 documentation):\n\n"
+              "FLAGS\n\n"
               "* `:all` - trigger an event for all of the below triggers.\n\n"
               "* `:attributes` - `FILE_NOTIFY_CHANGE_ATTRIBUTES`\n\n"
               "* `:creation` - `FILE_NOTIFY_CHANGE_CREATION`\n\n"
@@ -626,6 +889,22 @@ JANET_CORE_FN(cfun_filewatch_add,
               "* `:open` - `IN_OPEN`\n\n"
               "* `:q-overflow` - `IN_Q_OVERFLOW`\n\n"
               "* `:unmount` - `IN_UNMOUNT`\n\n\n"
+              "BSDs and macos (flags correspond to `NOTE_*` flags from <sys/event.h>). Not all flags are available on all systems:\n\n"
+              "* `:all` - `All available NOTE_* flags on the current platform`\n\n"
+              "* `:attrib` - `NOTE_ATTRIB`\n\n"
+              "* `:close-write` - `NOTE_CLOSE_WRITE`\n\n"
+              "* `:close` - `NOTE_CLOSE`\n\n"
+              "* `:delete` - `NOTE_DELETE`\n\n"
+              "* `:extend` - `NOTE_EXTEND`\n\n"
+              "* `:funlock` - `NOTE_FUNLOCK`\n\n"
+              "* `:link` - `NOTE_LINK`\n\n"
+              "* `:open` - `NOTE_OPEN`\n\n"
+              "* `:read` - `NOTE_READ`\n\n"
+              "* `:rename` - `NOTE_RENAME`\n\n"
+              "* `:revoke` - `NOTE_REVOKE`\n\n"
+              "* `:truncate` - `NOTE_TRUNCATE`\n\n"
+              "* `:write` - `NOTE_WRITE`\n\n\n"
+              "EVENT TYPES\n\n"
               "On Windows, events will have the following possible types:\n\n"
               "* `:unknown`\n\n"
               "* `:added`\n\n"
@@ -633,7 +912,7 @@ JANET_CORE_FN(cfun_filewatch_add,
               "* `:modified`\n\n"
               "* `:renamed-old`\n\n"
               "* `:renamed-new`\n\n"
-              "On Linux, events will have a `:type` corresponding to the possible flags, excluding `:all`.\n"
+              "On Linux and BSDs, events will have a `:type` corresponding to the possible flags, excluding `:all`.\n"
               "") {
     janet_arity(argc, 2, -1);
     JanetWatcher *watcher = janet_getabstract(argv, 0, &janet_filewatch_at);
@@ -648,6 +927,7 @@ JANET_CORE_FN(cfun_filewatch_remove,
               "Remove a path from the watcher.") {
     janet_fixarity(argc, 2);
     JanetWatcher *watcher = janet_getabstract(argv, 0, &janet_filewatch_at);
+    /* TODO - pass string in directly to avoid extra allocation */
     const char *path = janet_getcstring(argv, 1);
     janet_watcher_remove(watcher, path);
     return argv[0];

src/core/pp.c

@@ -374,9 +374,12 @@ const uint8_t *janet_to_string(Janet x) {
 struct pretty {
     JanetBuffer *buffer;
     int depth;
+    int width;
     int align;
+    int leaf_align;
     int flags;
     int32_t bufstartlen;
+    int32_t lookback_barrier;
     int32_t *keysort_buffer;
     int32_t keysort_capacity;
     int32_t keysort_start;
@@ -466,14 +469,75 @@ static int print_jdn_one(struct pretty *S, Janet x, int depth) {
     return 0;
 }
 
+static void backtrack_newlines(const struct pretty *S) {
+    if (S->flags & JANET_PRETTY_ONELINE || S->buffer->count <= 0)
+        return;
+    switch (S->buffer->data[S->buffer->count - 1]) {
+        case ')':
+        case '}':
+        case ']':
+            break;
+        default:
+            return;
+    }
+    int32_t removed = 0;
+    int32_t old_count = S->buffer->count;
+    int32_t offset = old_count;
+    int32_t b0 = S->lookback_barrier;
+    int32_t columns = S->width;
+    int32_t align = 0;
+    while (--offset >= b0) {
+        const char *s = (const char *)S->buffer->data + offset;
+        if (*s == '\n') {
+            if (align < S->leaf_align) {
+                break;
+            }
+            columns += align;
+            removed += align;
+            align = 0;
+        } else if (*s == ' ') {
+            align++;
+        } else {
+            align = 0;
+            /* Don't count color sequences: \x1B(0|3\d)m */
+            if (S->flags & JANET_PRETTY_COLOR && *s == 'm') {
+                if (offset >= (3 + b0) && strncmp("\x1B[0m", s - 3, 4) == 0) {
+                    offset -= 3;
+                    columns++;
+                } else if (offset >= (4 + b0) && strncmp("\x1B[3", s - 4, 3) == 0) {
+                    offset -= 4;
+                    columns++;
+                }
+            }
+        }
+        if (--columns <= 0) {
+            return;
+        }
+    }
+    offset++; /* Don't mess with the last newline we found */
+    janet_assert(offset >= b0, "bad buffer index");
+    S->buffer->count -= removed;
+    for (int32_t i = offset; i < S->buffer->count; i++) {
+        if (S->buffer->data[offset] == '\n') {
+            S->buffer->data[i] = ' ';
+            while (S->buffer->data[++offset] == ' ') {
+                janet_assert(offset < old_count, "bad replacement of newline");
+            }
+        } else {
+            S->buffer->data[i] = S->buffer->data[offset++];
+        }
+    }
+}
+
 static void print_newline(struct pretty *S, int align) {
     int i;
-    S->align = align;
     if (S->flags & JANET_PRETTY_ONELINE) {
         janet_buffer_push_u8(S->buffer, ' ');
         return;
     }
+    backtrack_newlines(S);
     janet_buffer_push_u8(S->buffer, '\n');
+    S->leaf_align = S->align = align;
     for (i = 0; i < S->align; i++) {
         janet_buffer_push_u8(S->buffer, ' ');
     }
@@ -543,6 +607,7 @@ static void janet_pretty_one(struct pretty *S, Janet x) {
             if (janet_checktype(x, JANET_BUFFER) && janet_unwrap_buffer(x) == S->buffer) {
                 janet_buffer_ensure(S->buffer, S->buffer->count + S->bufstartlen * 4 + 3, 1);
                 janet_buffer_push_u8(S->buffer, '@');
+                /* Use start len to print to self better */
                 S->align += 1 + janet_escape_string_impl(S->buffer, S->buffer->data, S->bufstartlen);
             } else {
                 S->align -= S->buffer->count;
@@ -564,7 +629,8 @@ static void janet_pretty_one(struct pretty *S, Janet x) {
             const char *startstr = isarray ? "@[" : hasbrackets ? "[" : "(";
             const char endchar = isarray ? ']' : hasbrackets ? ']' : ')';
             janet_buffer_push_cstring(S->buffer, startstr);
-            const int align = S->align += strlen(startstr);
+            S->align += strlen(startstr);
+            const int align = S->leaf_align = S->align;
             S->depth--;
             if (S->depth == 0) {
                 janet_buffer_push_cstring(S->buffer, "...");
@@ -639,7 +705,7 @@ static void janet_pretty_one(struct pretty *S, Janet x) {
                 }
             }
             janet_buffer_push_u8(S->buffer, '{');
-            const int align = ++S->align;
+            const int align = S->leaf_align = ++S->align;
 
             S->depth--;
             if (S->depth == 0) {
@@ -732,21 +798,27 @@ static void janet_pretty_one(struct pretty *S, Janet x) {
     return;
 }
 
-static JanetBuffer *janet_pretty_(JanetBuffer *buffer, int depth, int flags, Janet x, int32_t startlen) {
+#define JANET_COLUMNS 80
+
+static JanetBuffer *janet_pretty_(JanetBuffer *buffer, int depth, int width,
+                                  int flags, Janet x, int32_t startlen, int32_t lookback_barrier) {
     struct pretty S;
     if (NULL == buffer) {
         buffer = janet_buffer(0);
     }
     S.buffer = buffer;
     S.depth = depth;
+    S.width = width;
     S.align = 0;
     S.flags = flags;
     S.bufstartlen = startlen;
+    S.lookback_barrier = lookback_barrier;
     S.keysort_capacity = 0;
     S.keysort_buffer = NULL;
     S.keysort_start = 0;
     janet_table_init(&S.seen, 10);
     janet_pretty_one(&S, x);
+    backtrack_newlines(&S);
     janet_table_deinit(&S.seen);
     return S.buffer;
 }
@@ -754,10 +826,11 @@ static JanetBuffer *janet_pretty_(JanetBuffer *buffer, int depth, int flags, Jan
 /* Helper for printing a janet value in a pretty form. Not meant to be used
  * for serialization or anything like that. */
 JanetBuffer *janet_pretty(JanetBuffer *buffer, int depth, int flags, Janet x) {
-    return janet_pretty_(buffer, depth, flags, x, buffer ? buffer->count : 0);
+    return janet_pretty_(buffer, depth, JANET_COLUMNS, flags,
+                         x, buffer ? buffer->count : 0, buffer ? buffer->count : 0);
 }
 
-static JanetBuffer *janet_jdn_(JanetBuffer *buffer, int depth, Janet x, int32_t startlen) {
+static JanetBuffer *janet_jdn_(JanetBuffer *buffer, int depth, Janet x, int32_t startlen, int32_t lookback_barrier) {
     struct pretty S;
     if (NULL == buffer) {
         buffer = janet_buffer(0);
@@ -767,6 +840,7 @@ static JanetBuffer *janet_jdn_(JanetBuffer *buffer, int depth, Janet x, int32_t
     S.align = 0;
     S.flags = 0;
     S.bufstartlen = startlen;
+    S.lookback_barrier = lookback_barrier;
     S.keysort_capacity = 0;
     S.keysort_buffer = NULL;
     S.keysort_start = 0;
@@ -780,7 +854,7 @@ static JanetBuffer *janet_jdn_(JanetBuffer *buffer, int depth, Janet x, int32_t
 }
 
 JanetBuffer *janet_jdn(JanetBuffer *buffer, int depth, Janet x) {
-    return janet_jdn_(buffer, depth, x, buffer ? buffer->count : 0);
+    return janet_jdn_(buffer, depth, x, buffer ? buffer->count : 0, buffer ? buffer->count : 0);
 }
 
 static const char *typestr(Janet x) {
@@ -986,18 +1060,26 @@ void janet_formatbv(JanetBuffer *b, const char *format, va_list args) {
                     int has_color = (d == 'P') || (d == 'Q') || (d == 'M') || (d == 'N');
                     int has_oneline = (d == 'Q') || (d == 'q') || (d == 'N') || (d == 'n');
                     int has_notrunc = (d == 'M') || (d == 'm') || (d == 'N') || (d == 'n');
+                    int columns = atoi(width);
+                    if (columns == 0) {
+                        columns = JANET_COLUMNS;
+                    } else if (columns < 0) {
+                        has_oneline = 1;
+                    }
                     int flags = 0;
                     flags |= has_color ? JANET_PRETTY_COLOR : 0;
                     flags |= has_oneline ? JANET_PRETTY_ONELINE : 0;
                     flags |= has_notrunc ? JANET_PRETTY_NOTRUNC : 0;
-                    janet_pretty_(b, depth, flags, va_arg(args, Janet), startlen);
+                    janet_pretty_(b, depth, columns, flags,
+                                  va_arg(args, Janet), startlen, b->count);
                     break;
                 }
                 case 'j': {
                     int depth = atoi(precision);
-                    if (depth < 1)
+                    if (depth < 1) {
                         depth = JANET_RECURSION_GUARD;
-                    janet_jdn_(b, depth, va_arg(args, Janet), startlen);
+                    }
+                    janet_jdn_(b, depth, va_arg(args, Janet), startlen, b->count);
                     break;
                 }
                 default: {
@@ -1148,18 +1230,24 @@ void janet_buffer_format(
                     int has_color = (d == 'P') || (d == 'Q') || (d == 'M') || (d == 'N');
                     int has_oneline = (d == 'Q') || (d == 'q') || (d == 'N') || (d == 'n');
                     int has_notrunc = (d == 'M') || (d == 'm') || (d == 'N') || (d == 'n');
+                    int columns = atoi(width);
+                    if (columns == 0)
+                        columns = JANET_COLUMNS;
+                    else if (columns < 0)
+                        has_oneline = 1;
                     int flags = 0;
                     flags |= has_color ? JANET_PRETTY_COLOR : 0;
                     flags |= has_oneline ? JANET_PRETTY_ONELINE : 0;
                     flags |= has_notrunc ? JANET_PRETTY_NOTRUNC : 0;
-                    janet_pretty_(b, depth, flags, argv[arg], startlen);
+                    janet_pretty_(b, depth, columns, flags,
+                                  argv[arg], startlen, b->count);
                     break;
                 }
                 case 'j': {
                     int depth = atoi(precision);
                     if (depth < 1)
                         depth = JANET_RECURSION_GUARD;
-                    janet_jdn_(b, depth, argv[arg], startlen);
+                    janet_jdn_(b, depth, argv[arg], startlen, b->count);
                     break;
                 }
                 default: {

src/core/string.c

@@ -555,7 +555,9 @@ JANET_CORE_FN(cfun_string_format,
               "\n"
               "The following conversion specifiers are used for \"pretty-printing\", where the upper-case "
               "variants generate colored output. These specifiers can take a precision "
-              "argument to specify the maximum nesting depth to print.\n"
+              "argument to specify the maximum nesting depth to print. "
+              "The multiline specifiers can also take a width argument, "
+              "which defaults to 80 columns.\n"
               "- `p`, `P`: pretty format, truncating if necessary\n"
               "- `m`, `M`: pretty format without truncating.\n"
               "- `q`, `Q`: pretty format on one line, truncating if necessary.\n"

src/core/util.c

@@ -584,7 +584,7 @@ static void janet_check_pointer_align(void *p) {
     } un;
     un.p = p;
     janet_assert(!(un.u & (uintptr_t) ((1 << JANET_NANBOX_64_POINTER_SHIFT) - 1)),
-        "unaligned pointer wrap - cfunction pointers and abstract types must be aligned with this nanboxing configuration.");
+                 "unaligned pointer wrap - cfunction pointers and abstract types must be aligned with this nanboxing configuration.");
 #endif
 }
 

src/core/util.h

@@ -50,9 +50,9 @@
 #ifndef JANET_EXIT
 #include <stdio.h>
 #define JANET_EXIT(m) do { \
-    fprintf(stderr, "janet internal error at line %d in file %s: %s\n",\
+    fprintf(stderr, "janet abort at %s:%d: %s\n",\
-        __LINE__,\
         __FILE__,\
+        __LINE__,\
         (m));\
     abort();\
 } while (0)

src/include/janet.h

@@ -325,7 +325,7 @@ extern "C" {
 #if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(JANET_APPLE)
 /* All pointers, including function pointers, should be 4-byte aligned on aarch64 by default.
  * The exception is aarch64 macos, as it uses the same 47-bit userland address-space as on amd64. */
-#define JANET_NANBOX_64_POINTER_SHIFT 0 /* TODO - set me back to 2! (trying to trigger crash) */
+#define JANET_NANBOX_64_POINTER_SHIFT 2
 #endif
 #endif
 #endif

src/mainclient/shell.c

@@ -519,8 +519,13 @@ static void historymove(int delta) {
         } else if (gbl_historyi >= gbl_history_count) {
             gbl_historyi = gbl_history_count - 1;
         }
+        gbl_len = (int) strlen(gbl_history[gbl_historyi]);
+        /* If history element is longer the JANET_LINE_MAX - 1, truncate */
+        if (gbl_len > JANET_LINE_MAX - 1) {
+            gbl_len = JANET_LINE_MAX - 1;
+        }
+        gbl_pos = gbl_len;
         strncpy(gbl_buf, gbl_history[gbl_historyi], JANET_LINE_MAX - 1);
-        gbl_pos = gbl_len = (int) strlen(gbl_buf);
         gbl_buf[gbl_len] = '\0';
 
         refresh();
@@ -1232,7 +1237,7 @@ int main(int argc, char **argv) {
 #endif
 
 #if defined(JANET_PRF)
-    uint8_t hash_key[JANET_HASH_KEY_SIZE + 1];
+    uint8_t hash_key[JANET_HASH_KEY_SIZE + 1] = {0};
 #ifdef JANET_REDUCED_OS
     char *envvar = NULL;
 #else
@@ -1240,6 +1245,7 @@ int main(int argc, char **argv) {
 #endif
     if (NULL != envvar) {
         strncpy((char *) hash_key, envvar, sizeof(hash_key) - 1);
+        hash_key[JANET_HASH_KEY_SIZE] = '\0'; /* in case copy didn't get null byte */
     } else if (janet_cryptorand(hash_key, JANET_HASH_KEY_SIZE) != 0) {
         fputs("unable to initialize janet PRF hash function.\n", stderr);
         return 1;

test/suite-filewatch.janet

@@ -26,6 +26,8 @@
 (def chan (ev/chan 1000))
 (var is-win (or (= :mingw (os/which)) (= :windows (os/which))))
 (var is-linux (= :linux (os/which)))
+(def bsds [:freebsd :macos :openbsd :bsd :dragonfly :netbsd])
+(var is-kqueue (index-of (os/which) bsds))
 
 # If not supported, exit early
 (def [supported msg] (protect (filewatch/new chan)))
@@ -97,6 +99,10 @@
   (filewatch/add fw (string td3 "/file3.txt") :close-write :create :delete)
   (filewatch/add fw td1 :close-write :create :delete)
   (filewatch/add fw td2 :close-write :create :delete :ignored))
+(when is-kqueue
+  (filewatch/add fw (string td3 "/file3.txt") :all)
+  (filewatch/add fw td1 :all)
+  (filewatch/add fw td2 :all))
 (assert-no-error "filewatch/listen no error" (filewatch/listen fw))
 
 #
@@ -196,6 +202,30 @@
   (expect-empty)
   (gccollect))
 
+#
+# Macos and BSD file writing
+#
+
+# TODO - kqueue capabilities here are a bit more limited than inotify and windows by default.
+# This could be ammended with some heavier-weight functionality in userspace, though.
+(when is-kqueue
+  (spit-file td1 "file1.txt")
+  (expect :wd-path td1 :type :write)
+  (expect-empty)
+  (gccollect)
+  (spit-file td1 "file1.txt")
+  # Currently, only operations that modify the parent vnode do anything
+  (expect-empty)
+  (gccollect)
+  # Check that we don't get anymore events from test directory 2
+  (spit-file td2 "file2.txt")
+  (expect :wd-path td2 :type :write)
+  (expect-empty)
+  # Remove a file, then wait for remove event
+  (rmrf (string td1 "/file1.txt"))
+  (expect :type :write) # a "write" to the vnode
+  (expect-empty))
+
 (assert-no-error "filewatch/unlisten no error" (filewatch/unlisten fw))
 (assert-no-error "cleanup 1" (rmrf td1))
 (assert-no-error "cleanup 2" (rmrf td2))

test/suite-pp.janet

@@ -61,32 +61,37 @@
 (check-jdn "a string")
 (check-jdn @"a buffer")
 
+# Issue 1737
+(assert (deep= "@[]" (string/format "%M" @[])))
+(assert (deep= " @[]" (string/format " %M" @[])))
+(assert (deep= "  @[]" (string/format "  %M" @[])))
+(assert (deep= "   @[]" (string/format "   %M" @[])))
+(assert (deep= "    @[]" (string/format "    %M" @[])))
+(assert (deep= "     @[]" (string/format "     %M" @[])))
+(assert (deep= "@[1]" (string/format "%m" @[1])))
+(assert (deep= " @[2]" (string/format " %m" @[2])))
+(assert (deep= "  @[3]" (string/format "  %m" @[3])))
+(assert (deep= "   @[4]" (string/format "   %m" @[4])))
+(assert (deep= "    @[5]" (string/format "    %m" @[5])))
+(assert (deep= "     @[6]" (string/format "     %m" @[6])))
+
 # Test multiline pretty specifiers
 (let [tup [:keyword "string" @"buffer"]
       tab @{true (table/setproto @{:bar tup
                                    :baz 42}
                                  @{:_name "Foo"})}]
   (set (tab tup) tab)
-  (assert (= (string/format "%m" {tup @[tup tab]
+  (assert (= (string/format "%67m" {tup @[tup tab] 'symbol tup})
-                                  'symbol tup})
           `
-{symbol (:keyword
+{symbol (:keyword "string" @"buffer")
-         "string"
-         @"buffer")
  (:keyword
   "string"
-  @"buffer") @[(:keyword
+  @"buffer") @[(:keyword "string" @"buffer")
-                "string"
+               @{true @Foo{:bar (:keyword "string" @"buffer")
-                @"buffer")
-               @{true @Foo{:bar (:keyword
-                                 "string"
-                                 @"buffer")
                            :baz 42}
-                 (:keyword
+                 (:keyword "string" @"buffer") <cycle 2>}]}`))
-                  "string"
+  (assert (= (string/format "%67p" {(freeze (zipcoll (range 42)
-                  @"buffer") <cycle 2>}]}`))
+                                                     (range -42 0))) tab})
-  (assert (= (string/format "%p" {(freeze (zipcoll (range 42)
-                                                   (range -42 0))) tab})
              `
 {{0 -42
   1 -41
@@ -118,11 +123,24 @@
   27 -15
   28 -14
   29 -13
-  ...} @{true @Foo{:bar (:keyword
+  ...} @{true @Foo{:bar (:keyword "string" @"buffer") :baz 42}
-                         "string"
+         (:keyword "string" @"buffer") <cycle 1>}}`)))
-                         @"buffer")
+
-                   :baz 42}
+# Issue 1737
-         (:keyword
+(def capture-buf @"")
-          "string"
+(with-dyns [*err* capture-buf]
-          @"buffer") <cycle 1>}}`)))
+  (peg/match ~(* (constant @[]) (??)) "a"))
+(assert (deep= ```
+               ?? at [a] (index 0)
+               stack [1]:
+                 [0]: @[]
+
+               ```
+               (string capture-buf)))
+
+(assert (=
+         (string/format "?? at [bc] (index 2)\nstack [5]:\n  [0]: %m\n  [1]: %m\n  [2]: %m\n  [3]: %m\n  [4]: %m\n" "a" 1 true {} @[])
+         "?? at [bc] (index 2)\nstack [5]:\n  [0]: \"a\"\n  [1]: 1\n  [2]: true\n  [3]: {}\n  [4]: @[]\n")
+        "pretty format should not eat explicit newlines")
+
 (end-suite)

tools/afl/pp_runner.janet

@@ -0,0 +1,7 @@
+(def p (parser/new))
+(parser/consume p (slurp ((dyn :args) 1)))
+(while (parser/has-more p)
+  (def x (parser/produce p))
+  (printf "%m\n%99M\n%1m\n%0M" x x x x)
+  (printf "%q\n%99Q\n%1p\n%P" x x x x)
+  (protect (printf "%j" x)))