Add some more test cases for hex floats.

Update CodeQL actions to latest version

.github/workflows/codeql.yml

@@ -27,15 +27,16 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           queries: +security-and-quality
+          tools: linked
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
         with:
           category: "/language:${{ matrix.language }}"

CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## ??? - Unreleased
+- Add `struct/rawget`
+- Fix `deep=` and `deep-not=` to better handle degenerate cases with mutable table keys
+- Long strings will now dedent on `\r\n` instead of just `\n`.
+- Add `ev/to-file` for synchronous resource operations
+
 ## 1.37.1 - 2024-12-05
 - Fix meson cross compilation
 - Update timeout documentation for networking APIs: timeouts raise errors and do not return nil.

README.md

@@ -207,7 +207,7 @@ Alternatively, install the package directly with `pkgin install janet`.
 
 To build an `.msi` installer executable, in addition to the above steps, you will have to:
 
-5. Install, or otherwise add to your PATH the [WiX 3.11 Toolset](https://github.com/wixtoolset/wix3/releases).
+5. Install, or otherwise add to your PATH the [WiX 3.14 Toolset](https://github.com/wixtoolset/wix3/releases).
 6. Run `build_win dist`.
 
 Now you should have an `.msi`. You can run `build_win install` to install the `.msi`, or execute the file itself.

build_win.bat

@@ -91,7 +91,7 @@ exit /b 0
 
 @rem Clean build artifacts
 :CLEAN
-del *.exe *.lib *.exp
+del *.exe *.lib *.exp *.msi *.wixpdb
 rd /s /q build
 if exist dist (
     rd /s /q dist
@@ -143,7 +143,13 @@ if defined CI (
 ) else (
     set WIXBIN=
 )
-%WIXBIN%candle.exe tools\msi\janet.wxs -arch %BUILDARCH% -out build\
+
+set WIXARCH=%BUILDARCH%
+if "%WIXARCH%"=="aarch64" (
+    set WIXARCH=arm64
+)
+
+%WIXBIN%candle.exe tools\msi\janet.wxs -arch %WIXARCH% -out build\
 %WIXBIN%light.exe "-sice:ICE38" -b tools\msi -ext WixUIExtension build\janet.wixobj -out janet-%RELEASE_VERSION%-windows-%BUILDARCH%-installer.msi
 exit /b 0
 

src/boot/boot.janet

@@ -1311,7 +1311,7 @@
 (defdyn *redef* "When set, allow dynamically rebinding top level defs. Will slow generated code and is intended to be used for development.")
 (defdyn *debug* "Enables a built in debugger on errors and other useful features for debugging in a repl.")
 (defdyn *exit* "When set, will cause the current context to complete. Can be set to exit from repl (or file), for example.")
-(defdyn *exit-value* "Set the return value from `run-context` upon an exit. By default, `run-context` will return nil.")
+(defdyn *exit-value* "Set the return value from `run-context` upon an exit.")
 (defdyn *task-id* "When spawning a thread or fiber, the task-id can be assigned for concurrency control.")
 
 (defdyn *current-file*
@@ -2219,56 +2219,31 @@
   (map-template :some res pred ind inds)
   res)
 
-(defn deep-not=
-  ``Like `not=`, but mutable types (arrays, tables, buffers) are considered
-  equal if they have identical structure. Much slower than `not=`.``
-  [x y]
-  (def tx (type x))
-  (or
-    (not= tx (type y))
-    (case tx
-      :tuple (or (not= (length x) (length y))
-                 (do
-                   (var ret false)
-                   (forv i 0 (length x)
-                     (def xx (in x i))
-                     (def yy (in y i))
-                     (if (deep-not= xx yy)
-                       (break (set ret true))))
-                   ret))
-      :array (or (not= (length x) (length y))
-                 (do
-                   (var ret false)
-                   (forv i 0 (length x)
-                     (def xx (in x i))
-                     (def yy (in y i))
-                     (if (deep-not= xx yy)
-                       (break (set ret true))))
-                   ret))
-      :struct (deep-not= (kvs x) (kvs y))
-      :table (deep-not= (table/to-struct x) (table/to-struct y))
-      :buffer (not= (string x) (string y))
-      (not= x y))))
-
-(defn deep=
-  ``Like `=`, but mutable types (arrays, tables, buffers) are considered
-  equal if they have identical structure. Much slower than `=`.``
-  [x y]
-  (not (deep-not= x y)))
-
 (defn freeze
   `Freeze an object (make it immutable) and do a deep copy, making
   child values also immutable. Closures, fibers, and abstract types
   will not be recursively frozen, but all other types will.`
   [x]
-  (case (type x)
-    :array (tuple/slice (map freeze x))
-    :tuple (tuple/slice (map freeze x))
-    :table (if-let [p (table/getproto x)]
-             (freeze (merge (table/clone p) x))
-             (struct ;(map freeze (kvs x))))
-    :struct (struct ;(map freeze (kvs x)))
-    :buffer (string x)
+  (def tx (type x))
+  (cond
+    (or (= tx :array) (= tx :tuple))
+    (tuple/slice (map freeze x))
+
+    (or (= tx :table) (= tx :struct))
+    (let [temp-tab @{}]
+      # Handle multiple unique keys that freeze. Result should
+      # be independent of iteration order.
+      (eachp [k v] x
+        (def kk (freeze k))
+        (def vv (freeze v))
+        (def old (get temp-tab kk))
+        (def new (if (= nil old) vv (max vv old)))
+        (put temp-tab kk new))
+      (table/to-struct temp-tab (freeze (getproto x))))
+
+    (= tx :buffer)
+    (string x)
+
     x))
 
 (defn thaw
@@ -2284,6 +2259,41 @@
     :string (buffer ds)
     ds))
 
+(defn deep-not=
+  ``Like `not=`, but mutable types (arrays, tables, buffers) are considered
+  equal if they have identical structure. Much slower than `not=`.``
+  [x y]
+  (def tx (type x))
+  (or
+    (not= tx (type y))
+    (cond
+      (or (= tx :tuple) (= tx :array))
+      (or (not= (length x) (length y))
+          (do
+            (var ret false)
+            (forv i 0 (length x)
+              (def xx (in x i))
+              (def yy (in y i))
+              (if (deep-not= xx yy)
+                (break (set ret true))))
+            ret))
+      (or (= tx :struct) (= tx :table))
+      (or (not= (length x) (length y))
+          (do
+            (def rawget (if (= tx :struct) struct/rawget table/rawget))
+            (var ret false)
+            (eachp [k v] x
+                (if (deep-not= (rawget y k) v) (break (set ret true))))
+            ret))
+      (= tx :buffer) (not= 0 (- (length x) (length y)) (memcmp x y))
+      (not= x y))))
+
+(defn deep=
+  ``Like `=`, but mutable types (arrays, tables, buffers) are considered
+  equal if they have identical structure. Much slower than `=`.``
+  [x y]
+  (not (deep-not= x y)))
+
 (defn macex
   ``Expand macros completely.
   `on-binding` is an optional callback for whenever a normal symbolic binding

src/core/capi.c

@@ -62,6 +62,13 @@ JANET_NO_RETURN static void janet_top_level_signal(const char *msg) {
 
 void janet_signalv(JanetSignal sig, Janet message) {
     if (janet_vm.return_reg != NULL) {
+        /* Should match logic in janet_call for coercing everything not ok to an error (no awaits, yields, etc.) */
+        if (janet_vm.coerce_error && sig != JANET_SIGNAL_OK) {
+            if (sig != JANET_SIGNAL_ERROR) {
+                message = janet_wrap_string(janet_formatc("%v coerced from %s to error", message, janet_signal_names[sig]));
+            }
+            sig = JANET_SIGNAL_ERROR;
+        }
         *janet_vm.return_reg = message;
         if (NULL != janet_vm.fiber) {
             janet_vm.fiber->flags |= JANET_FIBER_DID_LONGJUMP;

src/core/corelib.c

@@ -449,8 +449,9 @@ JANET_CORE_FN(janet_core_range,
     }
     count = (count > 0) ? count : 0;
     int32_t int_count;
+    janet_assert(count >= 0, "bad range code");
     if (count > (double) INT32_MAX) {
-        int_count = INT32_MAX;
+        janet_panicf("range is too large, %f elements", count);
     } else {
         int_count = (int32_t) ceil(count);
     }

src/core/ev.c

@@ -32,9 +32,11 @@
 #ifdef JANET_EV
 
 #include <math.h>
+#include <fcntl.h>
 #ifdef JANET_WINDOWS
 #include <winsock2.h>
 #include <windows.h>
+#include <io.h>
 #else
 #include <pthread.h>
 #include <limits.h>
@@ -43,7 +45,6 @@
 #include <signal.h>
 #include <sys/ioctl.h>
 #include <sys/types.h>
-#include <fcntl.h>
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 #include <netdb.h>
@@ -3275,6 +3276,64 @@ JANET_CORE_FN(janet_cfun_rwlock_write_release,
     return argv[0];
 }
 
+static JanetFile *get_file_for_stream(JanetStream *stream) {
+    int32_t flags = 0;
+    char fmt[4] = {0};
+    int index = 0;
+    if (stream->flags & JANET_STREAM_READABLE) {
+        flags |= JANET_FILE_READ;
+        janet_sandbox_assert(JANET_SANDBOX_FS_READ);
+        fmt[index++] = 'r';
+    }
+    if (stream->flags & JANET_STREAM_WRITABLE) {
+        flags |= JANET_FILE_WRITE;
+        janet_sandbox_assert(JANET_SANDBOX_FS_WRITE);
+        int currindex = index;
+        fmt[index++] = (currindex == 0) ? 'w' : '+';
+    }
+    if (index == 0) return NULL;
+    /* duplicate handle when converting stream to file */
+#ifdef JANET_WINDOWS
+    int htype = 0;
+    if (fmt[0] == 'r' && fmt[1] == '+') {
+        htype = _O_RDWR;
+    } else if (fmt[0] == 'r') {
+        htype = _O_RDONLY;
+    } else if (fmt[0] == 'w') {
+        htype = _O_WRONLY;
+    }
+    int fd = _open_osfhandle((intptr_t) stream->handle, htype);
+    if (fd < 0) return NULL;
+    int fd_dup = _dup(fd);
+    if (fd_dup < 0) return NULL;
+    FILE *f = _fdopen(fd_dup, fmt);
+    if (NULL == f) {
+        _close(fd_dup);
+        return NULL;
+    }
+#else
+    int fd_dup = dup(stream->handle);
+    if (fd_dup < 0) return NULL;
+    FILE *f = fdopen(fd_dup, fmt);
+    if (NULL == f) {
+        close(fd_dup);
+        return NULL;
+    }
+#endif
+    return janet_makejfile(f, flags);
+}
+
+JANET_CORE_FN(janet_cfun_to_file,
+              "(ev/to-file)",
+              "Create core/file copy of the stream. This value can be used "
+              "when blocking IO behavior is needed.") {
+    janet_fixarity(argc, 1);
+    JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type);
+    JanetFile *iof = get_file_for_stream(stream);
+    if (iof == NULL) janet_panic("cannot make file from stream");
+    return janet_wrap_abstract(iof);
+}
+
 JANET_CORE_FN(janet_cfun_ev_all_tasks,
               "(ev/all-tasks)",
               "Get an array of all active fibers that are being used by the scheduler.") {
@@ -3319,6 +3378,7 @@ void janet_lib_ev(JanetTable *env) {
         JANET_CORE_REG("ev/acquire-wlock", janet_cfun_rwlock_write_lock),
         JANET_CORE_REG("ev/release-rlock", janet_cfun_rwlock_read_release),
         JANET_CORE_REG("ev/release-wlock", janet_cfun_rwlock_write_release),
+        JANET_CORE_REG("ev/to-file", janet_cfun_to_file),
         JANET_CORE_REG("ev/all-tasks", janet_cfun_ev_all_tasks),
         JANET_REG_END
     };

src/core/inttypes.c

@@ -191,21 +191,21 @@ Janet janet_wrap_u64(uint64_t x) {
 
 JANET_CORE_FN(cfun_it_s64_new,
               "(int/s64 value)",
-              "Create a boxed signed 64 bit integer from a string value.") {
+              "Create a boxed signed 64 bit integer from a string value or a number.") {
     janet_fixarity(argc, 1);
     return janet_wrap_s64(janet_unwrap_s64(argv[0]));
 }
 
 JANET_CORE_FN(cfun_it_u64_new,
               "(int/u64 value)",
-              "Create a boxed unsigned 64 bit integer from a string value.") {
+              "Create a boxed unsigned 64 bit integer from a string value or a number.") {
     janet_fixarity(argc, 1);
     return janet_wrap_u64(janet_unwrap_u64(argv[0]));
 }
 
 JANET_CORE_FN(cfun_to_number,
               "(int/to-number value)",
-              "Convert an int/u64 or int/s64 to a number. Fails if the number is out of range for an int32.") {
+              "Convert an int/u64 or int/s64 to a number. Fails if the number is out of range for an int64.") {
     janet_fixarity(argc, 1);
     if (janet_type(argv[0]) == JANET_ABSTRACT) {
         void *abst = janet_unwrap_abstract(argv[0]);

src/core/io.c

@@ -31,6 +31,7 @@
 
 #ifndef JANET_WINDOWS
 #include <fcntl.h>
+#include <sys/stat.h>
 #include <sys/wait.h>
 #include <unistd.h>
 #endif
@@ -164,6 +165,14 @@ JANET_CORE_FN(cfun_io_fopen,
     }
     FILE *f = fopen((const char *)fname, (const char *)fmode);
     if (f != NULL) {
+#ifndef JANET_WINDOWS
+        struct stat st;
+        fstat(fileno(f), &st);
+        if (S_ISDIR(st.st_mode)) {
+            fclose(f);
+            janet_panicf("cannot open directory: %s", fname);
+        }
+#endif
         size_t bufsize = janet_optsize(argv, argc, 2, BUFSIZ);
         if (bufsize != BUFSIZ) {
             int result = setvbuf(f, NULL, bufsize ? _IOFBF : _IONBF, bufsize);

src/core/net.c

@@ -578,9 +578,10 @@ JANET_CORE_FN(cfun_net_connect,
     net_sched_connect(stream);
 }
 
-static const char *serverify_socket(JSock sfd) {
+static const char *serverify_socket(JSock sfd, int reuse) {
     /* Set various socket options */
     int enable = 1;
+    if (reuse) {
         if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (char *) &enable, sizeof(int)) < 0) {
             return "setsockopt(SO_REUSEADDR) failed";
         }
@@ -589,6 +590,7 @@ static const char *serverify_socket(JSock sfd) {
             return "setsockopt(SO_REUSEPORT) failed";
         }
 #endif
+    }
     janet_net_socknoblock(sfd);
     return NULL;
 }
@@ -642,19 +644,21 @@ JANET_CORE_FN(cfun_net_shutdown,
 }
 
 JANET_CORE_FN(cfun_net_listen,
-              "(net/listen host port &opt type)",
+              "(net/listen host port &opt type no-reuse)",
               "Creates a server. Returns a new stream that is neither readable nor "
               "writeable. Use net/accept or net/accept-loop be to handle connections and start the server. "
               "The type parameter specifies the type of network connection, either "
               "a :stream (usually tcp), or :datagram (usually udp). If not specified, the default is "
-              ":stream. The host and port arguments are the same as in net/address.") {
+              ":stream. The host and port arguments are the same as in net/address. The last boolean parameter `no-reuse` will "
+              "disable the use of SO_REUSEADDR and SO_REUSEPORT when creating a server on some operating systems.") {
     janet_sandbox_assert(JANET_SANDBOX_NET_LISTEN);
-    janet_arity(argc, 2, 3);
+    janet_arity(argc, 2, 4);
 
     /* Get host, port, and handler*/
     int socktype = janet_get_sockettype(argv, argc, 2);
     int is_unix = 0;
     struct addrinfo *ai = janet_get_addrinfo(argv, 0, socktype, 1, &is_unix);
+    int reuse = !(argc >= 4 && janet_truthy(argv[3]));
 
     JSock sfd = JSOCKDEFAULT;
 #ifndef JANET_WINDOWS
@@ -664,7 +668,7 @@ JANET_CORE_FN(cfun_net_listen,
             janet_free(ai);
             janet_panicf("could not create socket: %V", janet_ev_lasterr());
         }
-        const char *err = serverify_socket(sfd);
+        const char *err = serverify_socket(sfd, reuse);
         if (NULL != err || bind(sfd, (struct sockaddr *)ai, sizeof(struct sockaddr_un))) {
             JSOCKCLOSE(sfd);
             janet_free(ai);
@@ -687,7 +691,7 @@ JANET_CORE_FN(cfun_net_listen,
             sfd = socket(rp->ai_family, rp->ai_socktype | JSOCKFLAGS, rp->ai_protocol);
 #endif
             if (!JSOCKVALID(sfd)) continue;
-            const char *err = serverify_socket(sfd);
+            const char *err = serverify_socket(sfd, reuse);
             if (NULL != err) {
                 JSOCKCLOSE(sfd);
                 continue;

src/core/parse.c

@@ -363,8 +363,7 @@ static int stringend(JanetParser *p, JanetParseState *state) {
         JanetParseState top = p->states[p->statecount - 1];
         int32_t indent_col = (int32_t) top.column - 1;
         uint8_t *r = bufstart, *end = r + buflen;
-        /* Check if there are any characters before the start column -
-         * if so, do not reindent. */
+        /* Unless there are only spaces before EOLs, disable reindenting */
         int reindent = 1;
         while (reindent && (r < end)) {
             if (*r++ == '\n') {
@@ -374,34 +373,36 @@ static int stringend(JanetParser *p, JanetParseState *state) {
                         break;
                     }
                 }
+                if ((r + 1) < end && *r == '\r' && *(r + 1) == '\n') reindent = 1;
             }
         }
-        /* Now reindent if able to, otherwise just drop leading newline. */
-        if (!reindent) {
-            if (buflen > 0 && bufstart[0] == '\n') {
-                buflen--;
-                bufstart++;
-            }
-        } else {
+        /* Now reindent if able */
+        if (reindent) {
             uint8_t *w = bufstart;
             r = bufstart;
             while (r < end) {
                 if (*r == '\n') {
-                    if (r == bufstart) {
-                        /* Skip leading newline */
-                        r++;
-                    } else {
                     *w++ = *r++;
-                    }
                     for (int32_t j = 0; (r < end) && (*r != '\n') && (j < indent_col); j++, r++);
+                    if ((r + 1) < end && *r == '\r' && *(r + 1) == '\n') *w++ = *r++;
                 } else {
                     *w++ = *r++;
                 }
             }
             buflen = (int32_t)(w - bufstart);
         }
-        /* Check for trailing newline character so we can remove it */
-        if (buflen > 0 && bufstart[buflen - 1] == '\n') {
+        /* Check for leading EOL so we can remove it */
+        if (buflen > 1 && bufstart[0] == '\r' && bufstart[1] == '\n') { /* Windows EOL */
+            buflen = buflen - 2;
+            bufstart = bufstart + 2;
+        } else if (buflen > 0 && bufstart[0] == '\n') { /* Unix EOL */
+            buflen--;
+            bufstart++;
+        }
+        /* Check for trailing EOL so we can remove it */
+        if (buflen > 1 && bufstart[buflen - 2] == '\r' && bufstart[buflen - 1] == '\n') { /* Windows EOL */
+            buflen = buflen - 2;
+        } else if (buflen > 0 && bufstart[buflen - 1] == '\n') { /* Unix EOL */
             buflen--;
         }
     }

src/core/peg.c

@@ -1,5 +1,5 @@
 /*
-* Copyright (c) 2024 Calvin Rose
+* Copyright (c) 2025 Calvin Rose
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
@@ -549,36 +549,39 @@ tail:
             const uint32_t *rule_separator = s->bytecode + rule[1];
             const uint32_t *rule_subpattern = s->bytecode + rule[2];
 
-            const uint8_t *separator_end = NULL;
-            do {
-                const uint8_t *text_start = text;
+            const uint8_t *chunk_start = text;
+            const uint8_t *chunk_end = NULL;
+
+            while (text <= saved_end) {
+                /* Find next split (or end of text) */
                 CapState cs = cap_save(s);
                 down1(s);
-                while (text <= s->text_end) {
-                    separator_end = peg_rule(s, rule_separator, text);
+                while (text <= saved_end) {
+                    chunk_end = text;
+                    const uint8_t *check = peg_rule(s, rule_separator, text);
                     cap_load(s, cs);
-                    if (separator_end) {
+                    if (check) {
+                        text = check;
                         break;
                     }
                     text++;
                 }
                 up1(s);
 
-                if (separator_end) {
-                    s->text_end = text;
-                    text = separator_end;
-                }
-
+                /* Match between splits */
+                s->text_end = chunk_end;
                 down1(s);
-                const uint8_t *subpattern_end = peg_rule(s, rule_subpattern, text_start);
+                const uint8_t *subpattern_end = peg_rule(s, rule_subpattern, chunk_start);
                 up1(s);
                 s->text_end = saved_end;
+                if (!subpattern_end) return NULL; /* Don't match anything */
 
-                if (!subpattern_end) {
-                    return NULL;
+                /* Ensure forward progress */
+                if (text == chunk_start) return NULL;
+                chunk_start = text;
             }
-            } while (separator_end);
 
+            s->text_end = saved_end;
             return s->text_end;
         }
 
@@ -1416,6 +1419,11 @@ static uint32_t peg_compile1(Builder *b, Janet peg) {
             emit_bytes(b, RULE_LITERAL, len, str);
             break;
         }
+        case JANET_BUFFER: {
+            const JanetBuffer *buf = janet_unwrap_buffer(peg);
+            emit_bytes(b, RULE_LITERAL, buf->count, buf->data);
+            break;
+        }
         case JANET_TABLE: {
             /* Build grammar table */
             JanetTable *new_grammar = janet_table_clone(janet_unwrap_table(peg));

src/core/run.c

@@ -28,7 +28,7 @@
 
 /* Run a string */
 int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char *sourcePath, Janet *out) {
-    JanetParser parser;
+    JanetParser *parser;
     int errflags = 0, done = 0;
     int32_t index = 0;
     Janet ret = janet_wrap_nil();
@@ -37,14 +37,16 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
 
     if (where) janet_gcroot(janet_wrap_string(where));
     if (NULL == sourcePath) sourcePath = "<unknown>";
-    janet_parser_init(&parser);
+    parser = janet_abstract(&janet_parser_type, sizeof(JanetParser));
+    janet_parser_init(parser);
+    janet_gcroot(janet_wrap_abstract(parser));
 
     /* While we haven't seen an error */
     while (!done) {
 
         /* Evaluate parsed values */
-        while (janet_parser_has_more(&parser)) {
-            Janet form = janet_parser_produce(&parser);
+        while (janet_parser_has_more(parser)) {
+            Janet form = janet_parser_produce(parser);
             JanetCompileResult cres = janet_compile(form, env, where);
             if (cres.status == JANET_COMPILE_OK) {
                 JanetFunction *f = janet_thunk(cres.funcdef);
@@ -58,8 +60,8 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
                 }
             } else {
                 ret = janet_wrap_string(cres.error);
-                int32_t line = (int32_t) parser.line;
-                int32_t col = (int32_t) parser.column;
+                int32_t line = (int32_t) parser->line;
+                int32_t col = (int32_t) parser->column;
                 if ((cres.error_mapping.line > 0) &&
                         (cres.error_mapping.column > 0)) {
                     line = cres.error_mapping.line;
@@ -81,16 +83,16 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
         if (done) break;
 
         /* Dispatch based on parse state */
-        switch (janet_parser_status(&parser)) {
+        switch (janet_parser_status(parser)) {
             case JANET_PARSE_DEAD:
                 done = 1;
                 break;
             case JANET_PARSE_ERROR: {
-                const char *e = janet_parser_error(&parser);
+                const char *e = janet_parser_error(parser);
                 errflags |= 0x04;
                 ret = janet_cstringv(e);
-                int32_t line = (int32_t) parser.line;
-                int32_t col = (int32_t) parser.column;
+                int32_t line = (int32_t) parser->line;
+                int32_t col = (int32_t) parser->column;
                 janet_eprintf("%s:%d:%d: parse error: %s\n", sourcePath, line, col, e);
                 done = 1;
                 break;
@@ -98,9 +100,9 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
             case JANET_PARSE_ROOT:
             case JANET_PARSE_PENDING:
                 if (index >= len) {
-                    janet_parser_eof(&parser);
+                    janet_parser_eof(parser);
                 } else {
-                    janet_parser_consume(&parser, bytes[index++]);
+                    janet_parser_consume(parser, bytes[index++]);
                 }
                 break;
         }
@@ -108,7 +110,7 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char
     }
 
     /* Clean up and return errors */
-    janet_parser_deinit(&parser);
+    janet_gcunroot(janet_wrap_abstract(parser));
     if (where) janet_gcunroot(janet_wrap_string(where));
 #ifdef JANET_EV
     /* Enter the event loop if we are not already in it */

src/core/state.h

@@ -100,6 +100,7 @@ struct JanetVM {
      * return point for panics. */
     jmp_buf *signal_buf;
     Janet *return_reg;
+    int coerce_error;
 
     /* The global registry for c functions. Used to store meta-data
      * along with otherwise bare c function pointers. */

src/core/strtod.c

@@ -301,6 +301,7 @@ int janet_scan_number_base(
     if (base == 0) {
         base = 10;
     }
+    int exp_base = base;
 
     /* Skip leading zeros */
     while (str < end && (*str == '0' || *str == '.')) {
@@ -322,6 +323,12 @@ int janet_scan_number_base(
         } else if (*str == '&') {
             foundexp = 1;
             break;
+        } else if (base == 16 && (*str == 'P' || *str == 'p')) { /* IEEE hex float */
+            foundexp = 1;
+            exp_base = 10;
+            base = 2;
+            ex *= 4; /* We need to correct the current exponent after we change the base */
+            break;
         } else if (base == 10 && (*str == 'E' || *str == 'e')) {
             foundexp = 1;
             break;
@@ -360,9 +367,9 @@ int janet_scan_number_base(
         }
         while (str < end) {
             int digit = digit_lookup[*str & 0x7F];
-            if (*str > 127 || digit >= base) goto error;
+            if (*str > 127 || digit >= exp_base) goto error;
             if (ee < (INT32_MAX / 40)) {
-                ee = base * ee + digit;
+                ee = exp_base * ee + digit;
             }
             str++;
             seenadigit = 1;

src/core/struct.c

@@ -294,6 +294,16 @@ JANET_CORE_FN(cfun_struct_to_table,
     return janet_wrap_table(tab);
 }
 
+JANET_CORE_FN(cfun_struct_rawget,
+              "(struct/rawget st key)",
+              "Gets a value from a struct `st` without looking at the prototype struct. "
+              "If `st` does not contain the key directly, the function will return "
+              "nil without checking the prototype. Returns the value in the struct.") {
+    janet_fixarity(argc, 2);
+    JanetStruct st = janet_getstruct(argv, 0);
+    return janet_struct_rawget(st, argv[1]);
+}
+
 /* Load the struct module */
 void janet_lib_struct(JanetTable *env) {
     JanetRegExt struct_cfuns[] = {
@@ -301,6 +311,7 @@ void janet_lib_struct(JanetTable *env) {
         JANET_CORE_REG("struct/getproto", cfun_struct_getproto),
         JANET_CORE_REG("struct/proto-flatten", cfun_struct_flatten),
         JANET_CORE_REG("struct/to-table", cfun_struct_to_table),
+        JANET_CORE_REG("struct/rawget", cfun_struct_rawget),
         JANET_REG_END
     };
     janet_core_cfuns_ext(env, NULL, struct_cfuns);

src/core/table.c

@@ -372,12 +372,14 @@ JANET_CORE_FN(cfun_table_setproto,
 }
 
 JANET_CORE_FN(cfun_table_tostruct,
-              "(table/to-struct tab)",
-              "Convert a table to a struct. Returns a new struct. This function "
-              "does not take into account prototype tables.") {
-    janet_fixarity(argc, 1);
+              "(table/to-struct tab &opt proto)",
+              "Convert a table to a struct. Returns a new struct.") {
+    janet_arity(argc, 1, 2);
     JanetTable *t = janet_gettable(argv, 0);
-    return janet_wrap_struct(janet_table_to_struct(t));
+    JanetStruct proto = janet_optstruct(argv, argc, 1, NULL);
+    JanetStruct st = janet_table_to_struct(t);
+    janet_struct_proto(st) = proto;
+    return janet_wrap_struct(st);
 }
 
 JANET_CORE_FN(cfun_table_rawget,

src/core/vm.c

@@ -1373,7 +1373,10 @@ Janet janet_call(JanetFunction *fun, int32_t argc, const Janet *argv) {
 
     /* Run vm */
     janet_vm.fiber->flags |= JANET_FIBER_RESUME_NO_USEVAL | JANET_FIBER_RESUME_NO_SKIP;
+    int old_coerce_error = janet_vm.coerce_error;
+    janet_vm.coerce_error = 1;
     JanetSignal signal = run_vm(janet_vm.fiber, janet_wrap_nil());
+    janet_vm.coerce_error = old_coerce_error;
 
     /* Teardown */
     janet_vm.stackn = oldn;
@@ -1384,6 +1387,10 @@ Janet janet_call(JanetFunction *fun, int32_t argc, const Janet *argv) {
     }
 
     if (signal != JANET_SIGNAL_OK) {
+        /* Should match logic in janet_signalv */
+        if (signal != JANET_SIGNAL_ERROR) {
+            *janet_vm.return_reg = janet_wrap_string(janet_formatc("%v coerced from %s to error", *janet_vm.return_reg, janet_signal_names[signal]));
+        }
         janet_panicv(*janet_vm.return_reg);
     }
 
@@ -1430,8 +1437,10 @@ void janet_try_init(JanetTryState *state) {
     state->vm_fiber = janet_vm.fiber;
     state->vm_jmp_buf = janet_vm.signal_buf;
     state->vm_return_reg = janet_vm.return_reg;
+    state->coerce_error = janet_vm.coerce_error;
     janet_vm.return_reg = &(state->payload);
     janet_vm.signal_buf = &(state->buf);
+    janet_vm.coerce_error = 0;
 }
 
 void janet_restore(JanetTryState *state) {
@@ -1440,6 +1449,7 @@ void janet_restore(JanetTryState *state) {
     janet_vm.fiber = state->vm_fiber;
     janet_vm.signal_buf = state->vm_jmp_buf;
     janet_vm.return_reg = state->vm_return_reg;
+    janet_vm.coerce_error = state->coerce_error;
 }
 
 static JanetSignal janet_continue_no_check(JanetFiber *fiber, Janet in, Janet *out) {

src/include/janet.h

@@ -1261,6 +1261,7 @@ typedef struct {
     /* new state */
     jmp_buf buf;
     Janet payload;
+    int coerce_error;
 } JanetTryState;
 
 /***** END SECTION TYPES *****/

test/helper.janet

@@ -39,7 +39,7 @@
 (defmacro assert
   [x &opt e]
   (def xx (gensym))
-  (default e ~',x)
+  (default e (string/format "%j" x))
   ~(do
      (def ,xx ,x)
      (,assert-no-tail ,xx ,e)

test/suite-boot.janet

@@ -896,11 +896,18 @@
   (struct/with-proto {:a [1 2 3]} :c 22 :b [1 2 3 4] :d "test" :e "test2"))
 (table/setproto table-to-freeze @{:a @[1 2 3]})
 
-(assert (deep= {:a [1 2 3] :b [1 2 3 4] :c 22 :d "test" :e "test2"}
-               (freeze table-to-freeze)))
+(assert (deep= struct-to-thaw (freeze table-to-freeze)))
 (assert (deep= table-to-freeze-with-inline-proto (thaw table-to-freeze)))
 (assert (deep= table-to-freeze-with-inline-proto (thaw struct-to-thaw)))
 
+# Check that freezing mutable keys is deterministic
+# for issue #1535
+(def hashes @{})
+(repeat 200
+  (def x (freeze {@"" 1 @"" 2 @"" 3 @"" 4 @"" 5}))
+  (put hashes (hash x) true))
+(assert (= 1 (length hashes)) "freeze mutable keys is deterministic")
+
 # Make sure Carriage Returns don't end up in doc strings
 # e528b86
 (assert (not (string/find "\r"
@@ -995,4 +1002,18 @@
 (assert-error "assertf error 3" (assertf false "%s message" "mystery"))
 (assert-error "assertf error 4" (assertf nil "%s %s" "alice" "bob"))
 
+# issue #1535
+(loop [i :range [1 1000]]
+  (assert (deep-not= @{:key1 "value1" @"key" "value2"}
+                     @{:key1 "value1" @"key" "value2"}) "deep= mutable keys"))
+(assert (deep-not= {"abc" 123} {@"abc" 123}) "deep= mutable keys vs immutable key")
+(assert (deep-not= {@"" 1 @"" 2 @"" 3} {@"" 1 @"" 2 @"" 3}) "deep= duplicate mutable keys")
+(assert (deep-not= {@"" @"" @"" @"" @"" 3} {@"" @"" @"" @"" @"" 3}) "deep= duplicate mutable keys 2")
+(assert (deep-not= {@[] @"" @[] @"" @[] 3} {@[] @"" @[] @"" @[] 3}) "deep= duplicate mutable keys 3")
+(assert (deep-not= {@{} @"" @{} @"" @{} 3} {@{} @"" @{} @"" @{} 3}) "deep= duplicate mutable keys 4")
+(assert (deep-not= @{:key1 "value1" @"key2" @"value2"}
+                   @{:key1 "value1" @"key2" "value2"}) "deep= mutable keys")
+(assert (deep-not= @{:key1 "value1" [@"key2"] @"value2"}
+                   @{:key1 "value1" [@"key2"] @"value2"}) "deep= mutable keys")
+
 (end-suite)

test/suite-corelib.janet

@@ -174,6 +174,7 @@
 (assert (deep= (range 0 17 4) @[0 4 8 12 16]) "(range 0 17 4)")
 (assert (deep= (range 16 0 -4) @[16 12 8 4]) "(range 16 0 -4)")
 (assert (deep= (range 17 0 -4) @[17 13 9 5 1]) "(range 17 0 -4)")
+(assert-error "large range" (range 0xFFFFFFFFFF))
 
 (assert (= (length (range 10)) 10) "(range 10)")
 (assert (= (length (range -10)) 0) "(range -10)")

test/suite-ev.janet

@@ -410,6 +410,10 @@
         (ev/call handler connection)
         (break))))
 
+# Make sure we can't bind again with no-reuse
+(assert-error "no-reuse"
+              (net/listen test-host test-port :stream true))
+
 # Read from socket
 
 (defn expect-read
@@ -418,9 +422,15 @@
   (assert (= result text) (string/format "expected %v, got %v" text result)))
 
 # Now do our telnet chat
-(def bob (net/connect test-host test-port))
+(def bob (net/connect test-host test-port :stream))
 (expect-read bob "Whats your name?\n")
+(if (= :mingw (os/which))
   (net/write bob "bob")
+  (do
+    (def fbob (ev/to-file bob))
+    (file/write fbob "bob")
+    (file/flush fbob)
+    (:close fbob)))
 (expect-read bob "Welcome bob\n")
 (def alice (net/connect test-host test-port))
 (expect-read alice "Whats your name?\n")
@@ -465,4 +475,13 @@
 # Close chat server
 (:close chat-server)
 
+# Issue #1531
+(def c (ev/chan 0))
+(ev/spawn (while (def x (ev/take c))))
+(defn print-to-chan [x] (ev/give c x))
+(assert-error "coerce await inside janet_call to error"
+              (with-dyns [*out* print-to-chan]
+                (pp :foo)))
+(ev/chan-close c)
+
 (end-suite)

test/suite-marsh.janet

@@ -207,7 +207,7 @@ neldb\0\0\0\xD8\x05printG\x01\0\xDE\xDE\xDE'\x03\0marshal_tes/\x02
 (assert (= 2 (length tclone)) "table/weak-values marsh 2")
 (gccollect)
 (assert (= 1 (length t)) "table/weak-value marsh 3")
-(assert (deep= t tclone) "table/weak-values marsh 4")
+(assert (deep= (freeze t) (freeze tclone)) "table/weak-values marsh 4")
 
 # tables with prototypes
 (def t (table/weak-values 1))
@@ -219,7 +219,7 @@ neldb\0\0\0\xD8\x05printG\x01\0\xDE\xDE\xDE'\x03\0marshal_tes/\x02
 (assert (= 2 (length tclone)) "marsh weak tables with prototypes 2")
 (gccollect)
 (assert (= 1 (length t)) "marsh weak tables with prototypes 3")
-(assert (deep= t tclone) "marsh weak tables with prototypes 4")
+(assert (deep= (freeze t) (freeze tclone)) "marsh weak tables with prototypes 4")
 (assert (deep= (getproto t) (getproto tclone)) "marsh weak tables with prototypes 5")
 
 (end-suite)

test/suite-parse.janet

@@ -57,6 +57,8 @@
     (for i (+ index 1) (+ index indent 1)
       (case (get text i)
         nil (break)
+        (chr "\r") (if-not (= (chr "\n") (get text (inc i)))
+                     (set rewrite false))
         (chr "\n") (break)
         (chr " ") nil
         (set rewrite false))))
@@ -64,12 +66,17 @@
   # Only re-indent if no dedented characters.
   (def str
     (if rewrite
-      (peg/replace-all ~(* "\n" (between 0 ,indent " ")) "\n" text)
+      (peg/replace-all ~(* '(* (? "\r") "\n") (between 0 ,indent " "))
+                      (fn [mtch eol] eol) text)
       text))
 
-  (def first-nl (= (chr "\n") (first str)))
-  (def last-nl (= (chr "\n") (last str)))
-  (string/slice str (if first-nl 1 0) (if last-nl -2)))
+  (def first-eol (cond
+                   (string/has-prefix? "\r\n" str) :crlf
+                   (string/has-prefix? "\n" str) :lf))
+  (def last-eol (cond
+                  (string/has-suffix? "\r\n" str) :crlf
+                  (string/has-suffix? "\n" str) :lf))
+  (string/slice str (case first-eol :crlf 2 :lf 1 0) (case last-eol :crlf -3 :lf -2)))
 
 (defn reindent-reference
   "Same as reindent but use parser functionality. Useful for
@@ -89,8 +96,10 @@
   (let [a (reindent text indent)
         b (reindent-reference text indent)]
     (assert (= a b)
-            (string "indent " indent-counter " (indent=" indent ")"))))
+            (string/format "reindent: %q, parse: %q (indent-test #%d with indent of %d)" a b indent-counter indent)
+            )))
 
+# Unix EOLs
 (check-indent "" 0)
 (check-indent "\n" 0)
 (check-indent "\n" 1)
@@ -106,6 +115,17 @@
 (check-indent "\n    Hello, world!\n    " 4)
 (check-indent "\n    Hello, world!\n   dedented text\n    " 4)
 (check-indent "\n    Hello, world!\n    indented text\n    " 4)
+# Windows EOLs
+(check-indent "\r\n" 0)
+(check-indent "\r\n" 1)
+(check-indent "\r\n\r\n" 0)
+(check-indent "\r\n\r\n" 1)
+(check-indent "\r\nHello, world!" 0)
+(check-indent "\r\nHello, world!" 1)
+(check-indent "\r\n    Hello, world!\r\n   " 4)
+(check-indent "\r\n    Hello, world!\r\n    " 4)
+(check-indent "\r\n    Hello, world!\r\n   dedented text\r\n    " 4)
+(check-indent "\r\n    Hello, world!\r\n    indented text\r\n    " 4)
 
 # Symbols with @ character
 # d68eae9
@@ -188,5 +208,14 @@
 (parser/consume p `")`)
 (assert (= (parser/produce p) ["hello"]))
 
+# Hex floats
+(assert (= math/pi +0x1.921fb54442d18p+0001))
+(assert (= math/int-max +0x1.ffff_ffff_ffff_ffp+0052))
+(assert (= math/int-min -0x1.ffff_ffff_ffff_ffp+0052))
+(assert (= 1 0x1P0))
+(assert (= 2 0x1P1))
+(assert (= -2 -0x1p1))
+(assert (= -0.5 -0x1p-1))
+
 (end-suite)
 

test/suite-peg.janet

@@ -772,5 +772,22 @@
   "5:apple6:banana6:cherry"
   @["apple" "banana" "cherry"])
 
+# Issue #1539 - make sure split with "" doesn't infinite loop/oom
+(test "issue 1539"
+      ~(split "" (capture (to -1)))
+      "hello there friends"
+      nil)
+
+(test "issue 1539 pt. 2"
+  ~(split "," (capture 0))
+  "abc123,,,,"
+  @["" "" "" "" ""])
+
+# Issue #1549 - allow buffers as peg literals
+(test "issue 1549"
+      ''@"abc123"
+      "abc123"
+      @["abc123"])
+
 (end-suite)
 

tools/msi/janet.wxs

@@ -19,6 +19,11 @@
     <?define ProgramFilesFolder="ProgramFilesFolder" ?>
     <?define Win64="no" ?>
     <?define Arch="(x86)" ?>
+<?elseif $(sys.BUILDARCH)="arm64" ?>
+    <?define UpgradeCode="0bd4bab6-c838-4c2a-b9e6-56ea8064863c" ?>
+    <?define ProgramFilesFolder="ProgramFiles64Folder" ?>
+    <?define Win64="yes" ?>
+    <?define Arch="(Arm)" ?>
 <?else ?>
     <?error Unsupported value of sys.BUILDARCH=$(sys.BUILDARCH)?>
 <?endif?>