From a7d424bc81923adbe7dfbb065ec8ae0a98de21fa Mon Sep 17 00:00:00 2001 From: sogaiu <983021772@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:39:53 +0900 Subject: [PATCH 01/47] Remove unused var pstatus --- src/boot/boot.janet | 1 - 1 file changed, 1 deletion(-) diff --git a/src/boot/boot.janet b/src/boot/boot.janet index 8e306f8b..34631138 100644 --- a/src/boot/boot.janet +++ b/src/boot/boot.janet @@ -2665,7 +2665,6 @@ (do (var pindex 0) - (var pstatus nil) (def len (length buf)) (when (= len 0) (:eof p) From a0eeb630e70a0271d4cd72e153dbb39078ffbcbb Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Sun, 1 Dec 2024 09:04:03 -0600 Subject: [PATCH 02/47] Correct documentation for issue #1523 net/* API documentation was not consistent with the implementation. The `ev/*` module documentation was, however. On timeout, all networking function calls raise an error and do not return nil. That was the old behavior. --- CHANGELOG.md | 2 ++ src/core/ev.c | 12 ++++++++++++ src/core/net.c | 10 +++++----- src/include/janet.h | 1 + 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 308f32f7..3f5d0640 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. ## ??? - Unreleased +- Update timeout documentation for networking APIs: timeouts raise errors and do not return nil. +- Add `janet_addtimeout_nil(double sec);` to the C API. - Change string hashing. - Fix string equality bug. - Add `assertf` diff --git a/src/core/ev.c b/src/core/ev.c index 5437a8de..9c6b0634 100644 --- a/src/core/ev.c +++ b/src/core/ev.c @@ -625,6 +625,18 @@ void janet_addtimeout(double sec) { add_timeout(to); } +/* Set timeout for the current root fiber but resume with nil instead of raising an error */ +void janet_addtimeout_nil(double sec) { + JanetFiber *fiber = janet_vm.root_fiber; + JanetTimeout to; + to.when = ts_delta(ts_now(), sec); + to.fiber = fiber; + to.curr_fiber = NULL; + to.sched_id = fiber->sched_id; + to.is_error = 0; + add_timeout(to); +} + void janet_ev_inc_refcount(void) { janet_atomic_inc(&janet_vm.listener_count); } diff --git a/src/core/net.c b/src/core/net.c index a01a7495..e59b1f54 100644 --- a/src/core/net.c +++ b/src/core/net.c @@ -829,7 +829,7 @@ JANET_CORE_FN(cfun_stream_accept_loop, JANET_CORE_FN(cfun_stream_accept, "(net/accept stream &opt timeout)", "Get the next connection on a server stream. This would usually be called in a loop in a dedicated fiber. " - "Takes an optional timeout in seconds, after which will return nil. " + "Takes an optional timeout in seconds, after which will raise an error. " "Returns a new duplex stream which represents a connection to the client.") { janet_arity(argc, 1, 2); JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type); @@ -864,7 +864,7 @@ JANET_CORE_FN(cfun_stream_read, JANET_CORE_FN(cfun_stream_chunk, "(net/chunk stream nbytes &opt buf timeout)", "Same a net/read, but will wait for all n bytes to arrive rather than return early. " - "Takes an optional timeout in seconds, after which will return nil.") { + "Takes an optional timeout in seconds, after which will raise an error.") { janet_arity(argc, 2, 4); JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type); janet_stream_flags(stream, JANET_STREAM_READABLE | JANET_STREAM_SOCKET); @@ -878,7 +878,7 @@ JANET_CORE_FN(cfun_stream_chunk, JANET_CORE_FN(cfun_stream_recv_from, "(net/recv-from stream nbytes buf &opt timeout)", "Receives data from a server stream and puts it into a buffer. Returns the socket-address the " - "packet came from. Takes an optional timeout in seconds, after which will return nil.") { + "packet came from. Takes an optional timeout in seconds, after which will raise an error.") { janet_arity(argc, 3, 4); JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type); janet_stream_flags(stream, JANET_STREAM_UDPSERVER | JANET_STREAM_SOCKET); @@ -892,7 +892,7 @@ JANET_CORE_FN(cfun_stream_recv_from, JANET_CORE_FN(cfun_stream_write, "(net/write stream data &opt timeout)", "Write data to a stream, suspending the current fiber until the write " - "completes. Takes an optional timeout in seconds, after which will return nil. " + "completes. Takes an optional timeout in seconds, after which will raise an error. " "Returns nil, or raises an error if the write failed.") { janet_arity(argc, 2, 3); JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type); @@ -911,7 +911,7 @@ JANET_CORE_FN(cfun_stream_write, JANET_CORE_FN(cfun_stream_send_to, "(net/send-to stream dest data &opt timeout)", "Writes a datagram to a server stream. dest is a the destination address of the packet. " - "Takes an optional timeout in seconds, after which will return nil. " + "Takes an optional timeout in seconds, after which will raise an error. " "Returns stream.") { janet_arity(argc, 3, 4); JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type); diff --git a/src/include/janet.h b/src/include/janet.h index 8ec75e4f..e9c9429f 100644 --- a/src/include/janet.h +++ b/src/include/janet.h @@ -1442,6 +1442,7 @@ JANET_NO_RETURN JANET_API void janet_sleep_await(double sec); /* For use inside listeners - adds a timeout to the current fiber, such that * it will be resumed after sec seconds if no other event schedules the current fiber. */ JANET_API void janet_addtimeout(double sec); +JANET_API void janet_addtimeout_nil(double sec); JANET_API void janet_ev_inc_refcount(void); JANET_API void janet_ev_dec_refcount(void); From 9338312103422cb5e3e8973055e72ede0b1e34b5 Mon Sep 17 00:00:00 2001 From: sogaiu <983021772@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:21:56 +0900 Subject: [PATCH 03/47] Additional tweak to address #1523 --- src/core/net.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/net.c b/src/core/net.c index e59b1f54..69e891fc 100644 --- a/src/core/net.c +++ b/src/core/net.c @@ -844,7 +844,7 @@ JANET_CORE_FN(cfun_stream_read, "Read up to n bytes from a stream, suspending the current fiber until the bytes are available. " "`n` can also be the keyword `:all` to read into the buffer until end of stream. " "If less than n bytes are available (and more than 0), will push those bytes and return early. " - "Takes an optional timeout in seconds, after which will return nil. " + "Takes an optional timeout in seconds, after which will raise an error. " "Returns a buffer with up to n more bytes in it, or raises an error if the read failed.") { janet_arity(argc, 2, 4); JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type); From 5b79b48ae02f5a4c95a8fa671a13bf37d63e37ed Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Tue, 3 Dec 2024 21:03:38 -0600 Subject: [PATCH 04/47] Address #1524 - fix meson cross compilation linking. In the cross compilation case, we need to resolve our dependencies on libc twice, once for the build machine and once for the target machine. This includes pthreads, -libc, and android-spawn. --- CHANGELOG.md | 1 + meson.build | 27 +++++++++++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f5d0640..f128eee1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ All notable changes to this project will be documented in this file. ## ??? - Unreleased +- Fix meson cross compilation - Update timeout documentation for networking APIs: timeouts raise errors and do not return nil. - Add `janet_addtimeout_nil(double sec);` to the C API. - Change string hashing. diff --git a/meson.build b/meson.build index f17e8a59..0e829057 100644 --- a/meson.build +++ b/meson.build @@ -26,8 +26,17 @@ project('janet', 'c', janet_path = join_paths(get_option('prefix'), get_option('libdir'), 'janet') header_path = join_paths(get_option('prefix'), get_option('includedir'), 'janet') -# Link math library on all systems +# Compilers cc = meson.get_compiler('c') +native_cc = meson.get_compiler('c', native : true) + +# Native deps +native_m_dep = native_cc.find_library('m', required : false) +native_dl_dep = native_cc.find_library('dl', required : false) +native_android_spawn_dep = native_cc.find_library('android-spawn', required : false) +native_thread_dep = dependency('threads', native : true) + +# Deps m_dep = cc.find_library('m', required : false) dl_dep = cc.find_library('dl', required : false) android_spawn_dep = cc.find_library('android-spawn', required : false) @@ -164,11 +173,18 @@ mainclient_src = [ 'src/mainclient/shell.c' ] +janet_dependencies = [m_dep, dl_dep, android_spawn_dep] +janet_native_dependencies = [native_m_dep, native_dl_dep, native_android_spawn_dep] +if not get_option('single_threaded') + janet_dependencies += thread_dep + janet_native_dependencies += native_thread_dep +endif + # Build boot binary janet_boot = executable('janet-boot', core_src, boot_src, include_directories : incdir, c_args : '-DJANET_BOOTSTRAP', - dependencies : [m_dep, dl_dep, thread_dep, android_spawn_dep], + dependencies : janet_native_dependencies, native : true) # Build janet.c @@ -181,11 +197,6 @@ janetc = custom_target('janetc', 'JANET_PATH', janet_path ]) -janet_dependencies = [m_dep, dl_dep, android_spawn_dep] -if not get_option('single_threaded') - janet_dependencies += thread_dep -endif - # Allow building with no shared library if cc.has_argument('-fvisibility=hidden') lib_cflags = ['-fvisibility=hidden'] @@ -231,7 +242,7 @@ if meson.is_cross_build() endif janet_nativeclient = executable('janet-native', janetc, mainclient_src, include_directories : incdir, - dependencies : janet_dependencies, + dependencies : janet_native_dependencies, c_args : extra_native_cflags, native : true) else From 952906279c8ad7370c11a363955fa1a852adea63 Mon Sep 17 00:00:00 2001 From: Ian Henry Date: Wed, 4 Dec 2024 21:17:10 -0800 Subject: [PATCH 05/47] add (til) PEG special (til sep subpattern) is a specialized (sub) that behaves like (sub (to sep) subpattern), but advances over the input like (thru sep). --- src/core/peg.c | 46 ++++++++++++++++++++++++++++++++++++++++++++ src/include/janet.h | 1 + test/suite-peg.janet | 35 +++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) diff --git a/src/core/peg.c b/src/core/peg.c index 24d1d320..2f2ebe0f 100644 --- a/src/core/peg.c +++ b/src/core/peg.c @@ -544,6 +544,42 @@ tail: return window_end; } + case RULE_TIL: { + const uint32_t *rule_terminus = s->bytecode + rule[1]; + const uint32_t *rule_subpattern = s->bytecode + rule[2]; + + const uint8_t *terminus_start = text; + const uint8_t *terminus_end = NULL; + down1(s); + while (terminus_start <= s->text_end) { + CapState cs2 = cap_save(s); + terminus_end = peg_rule(s, rule_terminus, terminus_start); + cap_load(s, cs2); + if (terminus_end) { + break; + } + terminus_start++; + } + up1(s); + + if (!terminus_end) { + return NULL; + } + + const uint8_t *saved_end = s->text_end; + s->text_end = terminus_start; + down1(s); + const uint8_t *matched = peg_rule(s, rule_subpattern, text); + up1(s); + s->text_end = saved_end; + + if (!matched) { + return NULL; + } + + return terminus_end; + } + case RULE_SPLIT: { const uint8_t *saved_end = s->text_end; const uint32_t *rule_separator = s->bytecode + rule[1]; @@ -1227,6 +1263,14 @@ static void spec_sub(Builder *b, int32_t argc, const Janet *argv) { emit_2(r, RULE_SUB, subrule1, subrule2); } +static void spec_til(Builder *b, int32_t argc, const Janet *argv) { + peg_fixarity(b, argc, 2); + Reserve r = reserve(b, 3); + uint32_t subrule1 = peg_compile1(b, argv[0]); + uint32_t subrule2 = peg_compile1(b, argv[1]); + emit_2(r, RULE_TIL, subrule1, subrule2); +} + static void spec_split(Builder *b, int32_t argc, const Janet *argv) { peg_fixarity(b, argc, 2); Reserve r = reserve(b, 3); @@ -1323,6 +1367,7 @@ static const SpecialPair peg_specials[] = { {"split", spec_split}, {"sub", spec_sub}, {"thru", spec_thru}, + {"til", spec_til}, {"to", spec_to}, {"uint", spec_uint_le}, {"uint-be", spec_uint_be}, @@ -1657,6 +1702,7 @@ static void *peg_unmarshal(JanetMarshalContext *ctx) { i += 4; break; case RULE_SUB: + case RULE_TIL: case RULE_SPLIT: /* [rule, rule] */ if (rule[1] >= blen) goto bad; diff --git a/src/include/janet.h b/src/include/janet.h index 8ec75e4f..974ee837 100644 --- a/src/include/janet.h +++ b/src/include/janet.h @@ -2180,6 +2180,7 @@ typedef enum { RULE_UNREF, /* [rule, tag] */ RULE_CAPTURE_NUM, /* [rule, tag] */ RULE_SUB, /* [rule, rule] */ + RULE_TIL, /* [rule, rule] */ RULE_SPLIT, /* [rule, rule] */ RULE_NTH, /* [nth, rule, tag] */ RULE_ONLY_TAGS, /* [rule] */ diff --git a/test/suite-peg.janet b/test/suite-peg.janet index ac426cfc..3f97125d 100644 --- a/test/suite-peg.janet +++ b/test/suite-peg.janet @@ -713,6 +713,41 @@ "abcdef" @[]) +(test "til: basic matching" + ~(til "d" "abc") + "abcdef" + @[]) + +(test "til: second pattern can't see past the first occurrence of first pattern" + ~(til "d" (* "abc" -1)) + "abcdef" + @[]) + +(test "til: fails if first pattern fails" + ~(til "x" "abc") + "abcdef" + nil) + +(test "til: fails if second pattern fails" + ~(til "abc" "x") + "abcdef" + nil) + +(test "til: discards captures from initial pattern" + ~(til '"d" '"abc") + "abcdef" + @["abc"]) + +(test "til: positions inside second match are still relative to the entire input" + ~(* "one\ntw" (til 0 (* ($) (line) (column)))) + "one\ntwo\nthree\n" + @[6 2 3]) + +(test "til: advances to the end of the first pattern's first occurrence" + ~(* (til "d" "ab") "e") + "abcdef" + @[]) + (test "split: basic functionality" ~(split "," '1) "a,b,c" From 85cb35e68f9d29912d3a2a24d8d6dc90d706ad6d Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Thu, 5 Dec 2024 17:51:06 -0600 Subject: [PATCH 06/47] Prepare for 1.37.0 release. --- CHANGELOG.md | 2 +- src/conf/janetconf.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f128eee1..48c5e80b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog All notable changes to this project will be documented in this file. -## ??? - Unreleased +## 1.37.0 - 2024-12-05 - Fix meson cross compilation - Update timeout documentation for networking APIs: timeouts raise errors and do not return nil. - Add `janet_addtimeout_nil(double sec);` to the C API. diff --git a/src/conf/janetconf.h b/src/conf/janetconf.h index 0211ee22..4ac3f0cf 100644 --- a/src/conf/janetconf.h +++ b/src/conf/janetconf.h @@ -6,8 +6,8 @@ #define JANET_VERSION_MAJOR 1 #define JANET_VERSION_MINOR 37 #define JANET_VERSION_PATCH 0 -#define JANET_VERSION_EXTRA "-dev" -#define JANET_VERSION "1.37.0-dev" +#define JANET_VERSION_EXTRA "" +#define JANET_VERSION "1.37.0" /* #define JANET_BUILD "local" */ From 83e8aab2897aa2982db48668e85757eef680ea5e Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Thu, 5 Dec 2024 20:18:16 -0600 Subject: [PATCH 07/47] Prepare for 1.37.1 release and fix CI. --- .github/workflows/test.yml | 3 +++ CHANGELOG.md | 2 +- build_win.bat | 3 ++- meson.build | 2 +- src/conf/janetconf.h | 4 ++-- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e832e7fc..4a385ab3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,6 +38,9 @@ jobs: - name: Test the project shell: cmd run: build_win test + - name: Test installer build + shell: cmd + run: build_win dist test-windows-min: name: Build and test on Windows Minimal build diff --git a/CHANGELOG.md b/CHANGELOG.md index 48c5e80b..7c36f2f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog All notable changes to this project will be documented in this file. -## 1.37.0 - 2024-12-05 +## 1.37.1 - 2024-12-05 - Fix meson cross compilation - Update timeout documentation for networking APIs: timeouts raise errors and do not return nil. - Add `janet_addtimeout_nil(double sec);` to the C API. diff --git a/build_win.bat b/build_win.bat index 2a3d06cb..a53d4931 100644 --- a/build_win.bat +++ b/build_win.bat @@ -138,7 +138,8 @@ if defined APPVEYOR_REPO_TAG_NAME ( set RELEASE_VERSION=%JANET_VERSION% ) if defined CI ( - set WIXBIN="c:\Program Files (x86)\WiX Toolset v3.11\bin\" + set WIXBIN="%WIX%bin\" + echo WIXBIN = %WIXBIN% ) else ( set WIXBIN= ) diff --git a/meson.build b/meson.build index 0e829057..d06102e4 100644 --- a/meson.build +++ b/meson.build @@ -20,7 +20,7 @@ project('janet', 'c', default_options : ['c_std=c99', 'build.c_std=c99', 'b_lundef=false', 'default_library=both'], - version : '1.37.0') + version : '1.37.1') # Global settings janet_path = join_paths(get_option('prefix'), get_option('libdir'), 'janet') diff --git a/src/conf/janetconf.h b/src/conf/janetconf.h index 4ac3f0cf..7fc77bc0 100644 --- a/src/conf/janetconf.h +++ b/src/conf/janetconf.h @@ -5,9 +5,9 @@ #define JANET_VERSION_MAJOR 1 #define JANET_VERSION_MINOR 37 -#define JANET_VERSION_PATCH 0 +#define JANET_VERSION_PATCH 1 #define JANET_VERSION_EXTRA "" -#define JANET_VERSION "1.37.0" +#define JANET_VERSION "1.37.1" /* #define JANET_BUILD "local" */ From ff173047f45680b9281a476f8678be422ed1b253 Mon Sep 17 00:00:00 2001 From: peteee Date: Fri, 13 Dec 2024 00:20:44 -0500 Subject: [PATCH 08/47] file/open: check if directory Adds fstat() directory test after fopen(), which can return non-NULL when passed a directory name on Linux --- src/core/io.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/core/io.c b/src/core/io.c index adbc9385..2b8e06af 100644 --- a/src/core/io.c +++ b/src/core/io.c @@ -31,6 +31,7 @@ #ifndef JANET_WINDOWS #include +#include #include #include #endif @@ -164,6 +165,14 @@ JANET_CORE_FN(cfun_io_fopen, } FILE *f = fopen((const char *)fname, (const char *)fmode); if (f != NULL) { +#ifndef JANET_WINDOWS + struct stat st; + fstat(f->_fileno, &st); + if (S_ISDIR(st.st_mode)) { + fclose(f); + janet_panicf("cannot open directory: %s", fname); + } +#endif size_t bufsize = janet_optsize(argv, argc, 2, BUFSIZ); if (bufsize != BUFSIZ) { int result = setvbuf(f, NULL, bufsize ? _IOFBF : _IONBF, bufsize); From ed63987fd1b52a1d283bdbdd80407e3f4f2e0671 Mon Sep 17 00:00:00 2001 From: peteee Date: Sat, 14 Dec 2024 07:17:28 -0500 Subject: [PATCH 09/47] use fileno() Should use fileno() instead of direct --- src/core/io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/io.c b/src/core/io.c index 2b8e06af..0f6703e9 100644 --- a/src/core/io.c +++ b/src/core/io.c @@ -167,7 +167,7 @@ JANET_CORE_FN(cfun_io_fopen, if (f != NULL) { #ifndef JANET_WINDOWS struct stat st; - fstat(f->_fileno, &st); + fstat(fileno(f), &st); if (S_ISDIR(st.st_mode)) { fclose(f); janet_panicf("cannot open directory: %s", fname); From 4daecc9a4113aa0198ddf30c90e59cc256a748d1 Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Sat, 14 Dec 2024 10:34:36 -0600 Subject: [PATCH 10/47] Prevent await inside janet_call - address #1531 This was partially implemented before, but not in the case where the await or other signal itself was created by a C function. We have to separate code paths for generating signals - one via normal returns in janet_vm_continue, and the other via longjump. This adds handling for the longjump case, as well as improved messaging. --- src/core/capi.c | 7 +++++++ src/core/state.h | 1 + src/core/vm.c | 10 ++++++++++ src/include/janet.h | 1 + test/suite-ev.janet | 9 +++++++++ 5 files changed, 28 insertions(+) diff --git a/src/core/capi.c b/src/core/capi.c index 9d741332..d4169dc0 100644 --- a/src/core/capi.c +++ b/src/core/capi.c @@ -62,6 +62,13 @@ JANET_NO_RETURN static void janet_top_level_signal(const char *msg) { void janet_signalv(JanetSignal sig, Janet message) { if (janet_vm.return_reg != NULL) { + /* Should match logic in janet_call for coercing everything not ok to an error (no awaits, yields, etc.) */ + if (janet_vm.coerce_error && sig != JANET_SIGNAL_OK) { + if (sig != JANET_SIGNAL_ERROR) { + message = janet_wrap_string(janet_formatc("%v coerced from %s to error", message, janet_signal_names[sig])); + } + sig = JANET_SIGNAL_ERROR; + } *janet_vm.return_reg = message; if (NULL != janet_vm.fiber) { janet_vm.fiber->flags |= JANET_FIBER_DID_LONGJUMP; diff --git a/src/core/state.h b/src/core/state.h index 5d9192c4..d121f559 100644 --- a/src/core/state.h +++ b/src/core/state.h @@ -100,6 +100,7 @@ struct JanetVM { * return point for panics. */ jmp_buf *signal_buf; Janet *return_reg; + int coerce_error; /* The global registry for c functions. Used to store meta-data * along with otherwise bare c function pointers. */ diff --git a/src/core/vm.c b/src/core/vm.c index ccbc87e2..0e599aa3 100644 --- a/src/core/vm.c +++ b/src/core/vm.c @@ -1373,7 +1373,10 @@ Janet janet_call(JanetFunction *fun, int32_t argc, const Janet *argv) { /* Run vm */ janet_vm.fiber->flags |= JANET_FIBER_RESUME_NO_USEVAL | JANET_FIBER_RESUME_NO_SKIP; + int old_coerce_error = janet_vm.coerce_error; + janet_vm.coerce_error = 1; JanetSignal signal = run_vm(janet_vm.fiber, janet_wrap_nil()); + janet_vm.coerce_error = old_coerce_error; /* Teardown */ janet_vm.stackn = oldn; @@ -1384,6 +1387,10 @@ Janet janet_call(JanetFunction *fun, int32_t argc, const Janet *argv) { } if (signal != JANET_SIGNAL_OK) { + /* Should match logic in janet_signalv */ + if (signal != JANET_SIGNAL_ERROR) { + *janet_vm.return_reg = janet_wrap_string(janet_formatc("%v coerced from %s to error", *janet_vm.return_reg, janet_signal_names[signal])); + } janet_panicv(*janet_vm.return_reg); } @@ -1430,8 +1437,10 @@ void janet_try_init(JanetTryState *state) { state->vm_fiber = janet_vm.fiber; state->vm_jmp_buf = janet_vm.signal_buf; state->vm_return_reg = janet_vm.return_reg; + state->coerce_error = janet_vm.coerce_error; janet_vm.return_reg = &(state->payload); janet_vm.signal_buf = &(state->buf); + janet_vm.coerce_error = 0; } void janet_restore(JanetTryState *state) { @@ -1440,6 +1449,7 @@ void janet_restore(JanetTryState *state) { janet_vm.fiber = state->vm_fiber; janet_vm.signal_buf = state->vm_jmp_buf; janet_vm.return_reg = state->vm_return_reg; + janet_vm.coerce_error = state->coerce_error; } static JanetSignal janet_continue_no_check(JanetFiber *fiber, Janet in, Janet *out) { diff --git a/src/include/janet.h b/src/include/janet.h index e9c9429f..c91e5e60 100644 --- a/src/include/janet.h +++ b/src/include/janet.h @@ -1261,6 +1261,7 @@ typedef struct { /* new state */ jmp_buf buf; Janet payload; + int coerce_error; } JanetTryState; /***** END SECTION TYPES *****/ diff --git a/test/suite-ev.janet b/test/suite-ev.janet index f0e859bf..2b9ef2d9 100644 --- a/test/suite-ev.janet +++ b/test/suite-ev.janet @@ -465,4 +465,13 @@ # Close chat server (:close chat-server) +# Issue #1531 +(def c (ev/chan 0)) +(ev/spawn (while (def x (ev/take c)))) +(defn print-to-chan [x] (ev/give c x)) +(assert-error "coerce await inside janet_call to error" + (with-dyns [*out* print-to-chan] + (pp :foo))) +(ev/chan-close c) + (end-suite) From 687b987f7e7a6352701eea68725ad861a32a77fa Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 15 Dec 2024 17:36:54 +0900 Subject: [PATCH 11/47] Add `ev/to-file` for synchronous resource operations --- CHANGELOG.md | 3 +++ src/core/ev.c | 53 +++++++++++++++++++++++++++++++++++++++++++++ test/suite-ev.janet | 7 ++++-- 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c36f2f1..b4e600bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog All notable changes to this project will be documented in this file. +## ??? - Unreleased +- Add `ev/to-file` for synchronous resource operations + ## 1.37.1 - 2024-12-05 - Fix meson cross compilation - Update timeout documentation for networking APIs: timeouts raise errors and do not return nil. diff --git a/src/core/ev.c b/src/core/ev.c index 9c6b0634..47e972b3 100644 --- a/src/core/ev.c +++ b/src/core/ev.c @@ -3275,6 +3275,58 @@ JANET_CORE_FN(janet_cfun_rwlock_write_release, return argv[0]; } +static JanetFile *get_file_for_stream(JanetStream *stream) { + int32_t flags = 0; + char fmt[4] = {0}; + int index = 0; + if (stream->flags & JANET_STREAM_READABLE) { + flags |= JANET_FILE_READ; + janet_sandbox_assert(JANET_SANDBOX_FS_READ); + fmt[index++] = 'r'; + } + if (stream->flags & JANET_STREAM_WRITABLE) { + flags |= JANET_FILE_WRITE; + janet_sandbox_assert(JANET_SANDBOX_FS_WRITE); + int currindex = index; + fmt[index++] = (currindex == 0) ? 'w' : '+'; + } + /* duplicate handle when converting stream to file */ + #ifdef JANET_WINDOWS + HANDLE prochandle = GetCurrentProcess(); + HANDLE newHandle = INVALID_HANDLE_VALUE; + if (!DuplicateHandle(prochandle, handle, prochandle, &newHandle, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + return NULL; + } + FILE *f = _fdopen(newHandle, fmt); + if (NULL == f) { + _close(newHandle); + return NULL; + } + #else + int newHandle = dup(stream->handle); + if (newHandle < 0) { + return NULL; + } + FILE *f = fdopen(newHandle, fmt); + if (NULL == f) { + close(newHandle); + return NULL; + } + #endif + return janet_makejfile(f, flags); +} + +JANET_CORE_FN(janet_cfun_to_file, + "(ev/to-file)", + "Create core/file copy of the stream. This value can be used " + "when blocking IO behavior is needed.") { + janet_fixarity(argc, 1); + JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type); + JanetFile *iof = get_file_for_stream(stream); + if (iof == NULL) janet_panic("cannot make file from stream"); + return janet_wrap_abstract(iof); +} + JANET_CORE_FN(janet_cfun_ev_all_tasks, "(ev/all-tasks)", "Get an array of all active fibers that are being used by the scheduler.") { @@ -3319,6 +3371,7 @@ void janet_lib_ev(JanetTable *env) { JANET_CORE_REG("ev/acquire-wlock", janet_cfun_rwlock_write_lock), JANET_CORE_REG("ev/release-rlock", janet_cfun_rwlock_read_release), JANET_CORE_REG("ev/release-wlock", janet_cfun_rwlock_write_release), + JANET_CORE_REG("ev/to-file", janet_cfun_to_file), JANET_CORE_REG("ev/all-tasks", janet_cfun_ev_all_tasks), JANET_REG_END }; diff --git a/test/suite-ev.janet b/test/suite-ev.janet index 2b9ef2d9..1bbbe623 100644 --- a/test/suite-ev.janet +++ b/test/suite-ev.janet @@ -418,9 +418,12 @@ (assert (= result text) (string/format "expected %v, got %v" text result))) # Now do our telnet chat -(def bob (net/connect test-host test-port)) +(def bob (net/connect test-host test-port :stream)) (expect-read bob "Whats your name?\n") -(net/write bob "bob") +(def fbob (ev/to-file bob)) +(file/write fbob "bob") +(file/flush fbob) +(:close fbob) (expect-read bob "Welcome bob\n") (def alice (net/connect test-host test-port)) (expect-read alice "Whats your name?\n") From 17d5fb32103cdaa7e7f9229a2239f52fee65862a Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 15 Dec 2024 18:56:35 +0900 Subject: [PATCH 12/47] Fix `ev/to-file` on Windows --- src/core/ev.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/core/ev.c b/src/core/ev.c index 47e972b3..93cc8679 100644 --- a/src/core/ev.c +++ b/src/core/ev.c @@ -3290,23 +3290,30 @@ static JanetFile *get_file_for_stream(JanetStream *stream) { int currindex = index; fmt[index++] = (currindex == 0) ? 'w' : '+'; } + if (index == 0) return NULL; /* duplicate handle when converting stream to file */ #ifdef JANET_WINDOWS - HANDLE prochandle = GetCurrentProcess(); - HANDLE newHandle = INVALID_HANDLE_VALUE; - if (!DuplicateHandle(prochandle, handle, prochandle, &newHandle, 0, FALSE, DUPLICATE_SAME_ACCESS)) { - return NULL; + int htype = 0; + if (fmt[0] == 'r' && fmt[1] == '+') { + htype = _O_RDWR; + } else if (fmt[0] == 'r') { + htype = _O_RDONLY; + } else if (fmt[0] == 'w') { + htype = _O_WRONLY; } - FILE *f = _fdopen(newHandle, fmt); + int fd = _open_osfhandle((intptr_t) stream->handle, htype); + if (fd < 0) return NULL; + int fd_dup = _dup(fd); + if (fd_dup < 0) return NULL; + FILE *f = _fdopen(fd_dup, fmt); if (NULL == f) { - _close(newHandle); + /* the stream isn't duplicated so this would close the stream + /* _close(fd); */ return NULL; } #else int newHandle = dup(stream->handle); - if (newHandle < 0) { - return NULL; - } + if (newHandle < 0) return NULL; FILE *f = fdopen(newHandle, fmt); if (NULL == f) { close(newHandle); From 91bb34c3bf46de4e72b3b783afdd22f2fb6aeee5 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 15 Dec 2024 19:17:48 +0900 Subject: [PATCH 13/47] Add missing header for Windows --- src/core/ev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/ev.c b/src/core/ev.c index 93cc8679..3d4964e8 100644 --- a/src/core/ev.c +++ b/src/core/ev.c @@ -35,6 +35,7 @@ #ifdef JANET_WINDOWS #include #include +#include #else #include #include From 268ff666d2fa13b72a66938f29dd10a35455c96b Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 15 Dec 2024 19:50:36 +0900 Subject: [PATCH 14/47] Move header to general imports --- src/core/ev.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/ev.c b/src/core/ev.c index 3d4964e8..f6d46a8d 100644 --- a/src/core/ev.c +++ b/src/core/ev.c @@ -32,6 +32,7 @@ #ifdef JANET_EV #include +#include #ifdef JANET_WINDOWS #include #include @@ -44,7 +45,6 @@ #include #include #include -#include #include #include #include @@ -3293,7 +3293,7 @@ static JanetFile *get_file_for_stream(JanetStream *stream) { } if (index == 0) return NULL; /* duplicate handle when converting stream to file */ - #ifdef JANET_WINDOWS +#ifdef JANET_WINDOWS int htype = 0; if (fmt[0] == 'r' && fmt[1] == '+') { htype = _O_RDWR; @@ -3312,7 +3312,7 @@ static JanetFile *get_file_for_stream(JanetStream *stream) { /* _close(fd); */ return NULL; } - #else +#else int newHandle = dup(stream->handle); if (newHandle < 0) return NULL; FILE *f = fdopen(newHandle, fmt); @@ -3320,7 +3320,7 @@ static JanetFile *get_file_for_stream(JanetStream *stream) { close(newHandle); return NULL; } - #endif +#endif return janet_makejfile(f, flags); } From 6ee05785d11cd7b93bd7256a58f750ada885d18b Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 15 Dec 2024 20:37:58 +0900 Subject: [PATCH 15/47] Disable buffering for files created with `ev/to-file` --- src/core/ev.c | 10 +++++++++- test/suite-ev.janet | 1 - 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/core/ev.c b/src/core/ev.c index f6d46a8d..ce27c838 100644 --- a/src/core/ev.c +++ b/src/core/ev.c @@ -3312,6 +3312,9 @@ static JanetFile *get_file_for_stream(JanetStream *stream) { /* _close(fd); */ return NULL; } + if (setvbuf(f, NULL, _IONBF, 0)) { + return NULL; + } #else int newHandle = dup(stream->handle); if (newHandle < 0) return NULL; @@ -3320,6 +3323,10 @@ static JanetFile *get_file_for_stream(JanetStream *stream) { close(newHandle); return NULL; } + if (setvbuf(f, NULL, _IONBF, 0)) { + close(newHandle); + return NULL; + } #endif return janet_makejfile(f, flags); } @@ -3327,7 +3334,8 @@ static JanetFile *get_file_for_stream(JanetStream *stream) { JANET_CORE_FN(janet_cfun_to_file, "(ev/to-file)", "Create core/file copy of the stream. This value can be used " - "when blocking IO behavior is needed.") { + "when blocking IO behavior is needed. Buffering is turned off " + "for the file.") { janet_fixarity(argc, 1); JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type); JanetFile *iof = get_file_for_stream(stream); diff --git a/test/suite-ev.janet b/test/suite-ev.janet index 1bbbe623..46820953 100644 --- a/test/suite-ev.janet +++ b/test/suite-ev.janet @@ -422,7 +422,6 @@ (expect-read bob "Whats your name?\n") (def fbob (ev/to-file bob)) (file/write fbob "bob") -(file/flush fbob) (:close fbob) (expect-read bob "Welcome bob\n") (def alice (net/connect test-host test-port)) From 1a24d4fc86617de3faa3bd920d31931da9e0f877 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 15 Dec 2024 21:00:52 +0900 Subject: [PATCH 16/47] Raise error if using `ev/to-file` on MinGW --- src/core/ev.c | 19 +++++++++---------- test/suite-ev.janet | 10 +++++++--- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/core/ev.c b/src/core/ev.c index ce27c838..5d4e04c5 100644 --- a/src/core/ev.c +++ b/src/core/ev.c @@ -3309,10 +3309,7 @@ static JanetFile *get_file_for_stream(JanetStream *stream) { FILE *f = _fdopen(fd_dup, fmt); if (NULL == f) { /* the stream isn't duplicated so this would close the stream - /* _close(fd); */ - return NULL; - } - if (setvbuf(f, NULL, _IONBF, 0)) { + * _close(fd); */ return NULL; } #else @@ -3323,10 +3320,6 @@ static JanetFile *get_file_for_stream(JanetStream *stream) { close(newHandle); return NULL; } - if (setvbuf(f, NULL, _IONBF, 0)) { - close(newHandle); - return NULL; - } #endif return janet_makejfile(f, flags); } @@ -3334,13 +3327,19 @@ static JanetFile *get_file_for_stream(JanetStream *stream) { JANET_CORE_FN(janet_cfun_to_file, "(ev/to-file)", "Create core/file copy of the stream. This value can be used " - "when blocking IO behavior is needed. Buffering is turned off " - "for the file.") { + "when blocking IO behavior is needed.") { janet_fixarity(argc, 1); +#ifdef JANET_MINGW + (void) argc; + (void) argv; + janet_panic("ev/to-file not supported on MinGW"); + return janet_wrap_nil(); +#else JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type); JanetFile *iof = get_file_for_stream(stream); if (iof == NULL) janet_panic("cannot make file from stream"); return janet_wrap_abstract(iof); +#endif } JANET_CORE_FN(janet_cfun_ev_all_tasks, diff --git a/test/suite-ev.janet b/test/suite-ev.janet index 46820953..724bf24a 100644 --- a/test/suite-ev.janet +++ b/test/suite-ev.janet @@ -420,9 +420,13 @@ # Now do our telnet chat (def bob (net/connect test-host test-port :stream)) (expect-read bob "Whats your name?\n") -(def fbob (ev/to-file bob)) -(file/write fbob "bob") -(:close fbob) +(if (= :mingw (os/which)) + (net/write bob "bob") + (do + (def fbob (ev/to-file bob)) + (file/write fbob "bob") + (file/flush fbob) + (:close fbob))) (expect-read bob "Welcome bob\n") (def alice (net/connect test-host test-port)) (expect-read alice "Whats your name?\n") From e94e8dc4845e76caacc62ca72ed48d73071eed8c Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 16 Dec 2024 08:12:14 +0900 Subject: [PATCH 17/47] Remove special casing for MinGW --- src/core/ev.c | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/core/ev.c b/src/core/ev.c index 5d4e04c5..0303a36e 100644 --- a/src/core/ev.c +++ b/src/core/ev.c @@ -3308,16 +3308,15 @@ static JanetFile *get_file_for_stream(JanetStream *stream) { if (fd_dup < 0) return NULL; FILE *f = _fdopen(fd_dup, fmt); if (NULL == f) { - /* the stream isn't duplicated so this would close the stream - * _close(fd); */ + _close(fd_dup); return NULL; } #else - int newHandle = dup(stream->handle); - if (newHandle < 0) return NULL; - FILE *f = fdopen(newHandle, fmt); + int fd_dup = dup(stream->handle); + if (fd_dup < 0) return NULL; + FILE *f = fdopen(fd_dup, fmt); if (NULL == f) { - close(newHandle); + close(fd_dup); return NULL; } #endif @@ -3329,17 +3328,10 @@ JANET_CORE_FN(janet_cfun_to_file, "Create core/file copy of the stream. This value can be used " "when blocking IO behavior is needed.") { janet_fixarity(argc, 1); -#ifdef JANET_MINGW - (void) argc; - (void) argv; - janet_panic("ev/to-file not supported on MinGW"); - return janet_wrap_nil(); -#else JanetStream *stream = janet_getabstract(argv, 0, &janet_stream_type); JanetFile *iof = get_file_for_stream(stream); if (iof == NULL) janet_panic("cannot make file from stream"); return janet_wrap_abstract(iof); -#endif } JANET_CORE_FN(janet_cfun_ev_all_tasks, From 67e8518ba6dc548fabb44cdad968dff748d2d15b Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Tue, 17 Dec 2024 05:14:59 +0900 Subject: [PATCH 18/47] Support dedenting longstrings with Windows EOLs --- src/core/parse.c | 35 ++++++++++++++++++----------------- test/suite-parse.janet | 30 +++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/src/core/parse.c b/src/core/parse.c index 6faa7948..c9177316 100644 --- a/src/core/parse.c +++ b/src/core/parse.c @@ -363,8 +363,7 @@ static int stringend(JanetParser *p, JanetParseState *state) { JanetParseState top = p->states[p->statecount - 1]; int32_t indent_col = (int32_t) top.column - 1; uint8_t *r = bufstart, *end = r + buflen; - /* Check if there are any characters before the start column - - * if so, do not reindent. */ + /* Unless there are only spaces before EOLs, disable reindenting */ int reindent = 1; while (reindent && (r < end)) { if (*r++ == '\n') { @@ -374,34 +373,36 @@ static int stringend(JanetParser *p, JanetParseState *state) { break; } } + if ((r + 1) < end && *r == '\r' && *(r + 1) == '\n') reindent = 1; } } - /* Now reindent if able to, otherwise just drop leading newline. */ - if (!reindent) { - if (buflen > 0 && bufstart[0] == '\n') { - buflen--; - bufstart++; - } - } else { + /* Now reindent if able */ + if (reindent) { uint8_t *w = bufstart; r = bufstart; while (r < end) { if (*r == '\n') { - if (r == bufstart) { - /* Skip leading newline */ - r++; - } else { - *w++ = *r++; - } + *w++ = *r++; for (int32_t j = 0; (r < end) && (*r != '\n') && (j < indent_col); j++, r++); + if ((r + 1) < end && *r == '\r' && *(r + 1) == '\n') *w++ = *r++; } else { *w++ = *r++; } } buflen = (int32_t)(w - bufstart); } - /* Check for trailing newline character so we can remove it */ - if (buflen > 0 && bufstart[buflen - 1] == '\n') { + /* Check for leading EOL so we can remove it */ + if (buflen > 1 && bufstart[0] == '\r' && bufstart[1] == '\n') { /* Windows EOL */ + buflen = buflen - 2; + bufstart = bufstart + 2; + } else if (buflen > 0 && bufstart[0] == '\n') { /* Unix EOL */ + buflen--; + bufstart++; + } + /* Check for trailing EOL so we can remove it */ + if (buflen > 1 && bufstart[buflen - 2] == '\r' && bufstart[buflen - 1] == '\n') { /* Windows EOL */ + buflen = buflen - 2; + } else if (buflen > 0 && bufstart[buflen - 1] == '\n') { /* Unix EOL */ buflen--; } } diff --git a/test/suite-parse.janet b/test/suite-parse.janet index 94154120..05b34292 100644 --- a/test/suite-parse.janet +++ b/test/suite-parse.janet @@ -57,6 +57,8 @@ (for i (+ index 1) (+ index indent 1) (case (get text i) nil (break) + (chr "\r") (if-not (= (chr "\n") (get text (inc i))) + (set rewrite false)) (chr "\n") (break) (chr " ") nil (set rewrite false)))) @@ -64,12 +66,17 @@ # Only re-indent if no dedented characters. (def str (if rewrite - (peg/replace-all ~(* "\n" (between 0 ,indent " ")) "\n" text) + (peg/replace-all ~(* '(* (? "\r") "\n") (between 0 ,indent " ")) + (fn [mtch eol] eol) text) text)) - (def first-nl (= (chr "\n") (first str))) - (def last-nl (= (chr "\n") (last str))) - (string/slice str (if first-nl 1 0) (if last-nl -2))) + (def first-eol (cond + (string/has-prefix? "\r\n" str) :crlf + (string/has-prefix? "\n" str) :lf)) + (def last-eol (cond + (string/has-suffix? "\r\n" str) :crlf + (string/has-suffix? "\n" str) :lf)) + (string/slice str (case first-eol :crlf 2 :lf 1 0) (case last-eol :crlf -3 :lf -2))) (defn reindent-reference "Same as reindent but use parser functionality. Useful for @@ -89,8 +96,10 @@ (let [a (reindent text indent) b (reindent-reference text indent)] (assert (= a b) - (string "indent " indent-counter " (indent=" indent ")")))) + (string/format "reindent: %q, parse: %q (indent-test #%d with indent of %d)" a b indent-counter indent) + ))) +# Unix EOLs (check-indent "" 0) (check-indent "\n" 0) (check-indent "\n" 1) @@ -106,6 +115,17 @@ (check-indent "\n Hello, world!\n " 4) (check-indent "\n Hello, world!\n dedented text\n " 4) (check-indent "\n Hello, world!\n indented text\n " 4) +# Windows EOLs +(check-indent "\r\n" 0) +(check-indent "\r\n" 1) +(check-indent "\r\n\r\n" 0) +(check-indent "\r\n\r\n" 1) +(check-indent "\r\nHello, world!" 0) +(check-indent "\r\nHello, world!" 1) +(check-indent "\r\n Hello, world!\r\n " 4) +(check-indent "\r\n Hello, world!\r\n " 4) +(check-indent "\r\n Hello, world!\r\n dedented text\r\n " 4) +(check-indent "\r\n Hello, world!\r\n indented text\r\n " 4) # Symbols with @ character # d68eae9 From 7f745a34c38fa6711a0846e3ccd7e8696e57df7b Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Thu, 19 Dec 2024 18:17:59 -0600 Subject: [PATCH 19/47] Allow for mutable keys correctly in deep= --- src/boot/boot.janet | 90 ++++++++++++++++++++++++------------------- test/suite-boot.janet | 7 ++++ 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/src/boot/boot.janet b/src/boot/boot.janet index 34631138..dc16fec4 100644 --- a/src/boot/boot.janet +++ b/src/boot/boot.janet @@ -2219,43 +2219,6 @@ (map-template :some res pred ind inds) res) -(defn deep-not= - ``Like `not=`, but mutable types (arrays, tables, buffers) are considered - equal if they have identical structure. Much slower than `not=`.`` - [x y] - (def tx (type x)) - (or - (not= tx (type y)) - (case tx - :tuple (or (not= (length x) (length y)) - (do - (var ret false) - (forv i 0 (length x) - (def xx (in x i)) - (def yy (in y i)) - (if (deep-not= xx yy) - (break (set ret true)))) - ret)) - :array (or (not= (length x) (length y)) - (do - (var ret false) - (forv i 0 (length x) - (def xx (in x i)) - (def yy (in y i)) - (if (deep-not= xx yy) - (break (set ret true)))) - ret)) - :struct (deep-not= (kvs x) (kvs y)) - :table (deep-not= (table/to-struct x) (table/to-struct y)) - :buffer (not= (string x) (string y)) - (not= x y)))) - -(defn deep= - ``Like `=`, but mutable types (arrays, tables, buffers) are considered - equal if they have identical structure. Much slower than `=`.`` - [x y] - (not (deep-not= x y))) - (defn freeze `Freeze an object (make it immutable) and do a deep copy, making child values also immutable. Closures, fibers, and abstract types @@ -2284,6 +2247,53 @@ :string (buffer ds) ds)) +(def- mutable-types {:table true :array true :buffer true}) + +(defn deep-not= + ``Like `not=`, but mutable types (arrays, tables, buffers) are considered + equal if they have identical structure. Much slower than `not=`.`` + [x y] + (def tx (type x)) + (or + (not= tx (type y)) + (cond + (or (= tx :tuple) (= tx :array)) + (or (not= (length x) (length y)) + (do + (var ret false) + (forv i 0 (length x) + (def xx (in x i)) + (def yy (in y i)) + (if (deep-not= xx yy) + (break (set ret true)))) + ret)) + (or (= tx :struct) (= tx :table)) + (or (not= (length x) (length y)) + (do + (var ret false) + (def mut-keys-x @{}) + (eachp [k v] x + (if (get mutable-types (type k)) + (let [kk (freeze k)] + (put mut-keys-x kk (put (get mut-keys-x kk @{}) (freeze v) true))) + (if (deep-not= (get y k) v) (break (set ret true))))) + (when (next mut-keys-x) # handle case when we have mutable keys separately + (def mut-keys-y @{}) + (eachp [k v] y + (if (get mutable-types (type k)) + (let [kk (freeze k)] + (put mut-keys-y kk (put (get mut-keys-y kk @{}) (freeze v) true))))) + (set ret (deep-not= mut-keys-x mut-keys-y))) + ret)) + (= tx :buffer) (not= 0 (- (length x) (length y)) (memcmp x y)) + (not= x y)))) + +(defn deep= + ``Like `=`, but mutable types (arrays, tables, buffers) are considered + equal if they have identical structure. Much slower than `=`.`` + [x y] + (not (deep-not= x y))) + (defn macex ``Expand macros completely. `on-binding` is an optional callback for whenever a normal symbolic binding @@ -2854,8 +2864,8 @@ (when (and (string? pattern) (string/has-prefix? ":sys:/" pattern)) (set last-index index) (array/push copies [(string/replace ":sys:" path pattern) ;(drop 1 entry)]))) - (array/insert mp (+ 1 last-index) ;copies) - mp) + (array/insert mp (+ 1 last-index) ;copies) + mp) (module/add-paths ":native:" :native) (module/add-paths "/init.janet" :source) @@ -4096,7 +4106,7 @@ (when (empty? b) (buffer/trim b) (os/chmod to perm) (break)) (file/write fto b) (buffer/clear b))) - (errorf "destination file %s cannot be opened for writing" to)) + (errorf "destination file %s cannot be opened for writing" to)) (errorf "source file %s cannot be opened for reading" from))) (defn- copyrf diff --git a/test/suite-boot.janet b/test/suite-boot.janet index 98049ae6..bcaa1f9d 100644 --- a/test/suite-boot.janet +++ b/test/suite-boot.janet @@ -995,4 +995,11 @@ (assert-error "assertf error 3" (assertf false "%s message" "mystery")) (assert-error "assertf error 4" (assertf nil "%s %s" "alice" "bob")) +# issue #1535 +(loop [i :range [1 1000]] + (assert (deep= @{:key1 "value1" @"key" "value2"} + @{:key1 "value1" @"key" "value2"}) "deep= mutable keys")) +(assert (deep-not= {"abc" 123} {@"abc" 123}) "deep= mutable keys vs immutable key") +(assert (deep= {@"" 1 @"" 2 @"" 3} {@"" 1 @"" 2 @"" 3}) "deep= duplicate mutable keys") + (end-suite) From 8043caf5815a1c3c36d8a033cade0e648da175a4 Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Thu, 19 Dec 2024 18:31:05 -0600 Subject: [PATCH 20/47] Update CHANGELOG. --- CHANGELOG.md | 2 + src/core/corelib.c | 160 ++++++++++++++++++++++----------------------- 2 files changed, 82 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4e600bb..13145525 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. ## ??? - Unreleased +- Fix `deep=` and `deep-not=` to better handle degenerate cases with mutable table keys +- Long strings will now dedent on `\r\n` instead of just `\n`. - Add `ev/to-file` for synchronous resource operations ## 1.37.1 - 2024-12-05 diff --git a/src/core/corelib.c b/src/core/corelib.c index e333f079..01775d19 100644 --- a/src/core/corelib.c +++ b/src/core/corelib.c @@ -1001,12 +1001,12 @@ static void make_apply(JanetTable *env) { janet_quick_asm(env, JANET_FUN_APPLY | JANET_FUNCDEF_FLAG_VARARG, "apply", 1, 1, INT32_MAX, 6, apply_asm, sizeof(apply_asm), JDOC("(apply f & args)\n\n" - "Applies a function to a variable number of arguments. Each element in args " - "is used as an argument to f, except the last element in args, which is expected to " - "be an array-like. Each element in this last argument is then also pushed as an argument to " - "f. For example:\n\n" - "\t(apply + 1000 (range 10))\n\n" - "sums the first 10 integers and 1000.")); + "Applies a function to a variable number of arguments. Each element in args " + "is used as an argument to f, except the last element in args, which is expected to " + "be an array-like. Each element in this last argument is then also pushed as an argument to " + "f. For example:\n\n" + "\t(apply + 1000 (range 10))\n\n" + "sums the first 10 integers and 1000.")); } static const uint32_t error_asm[] = { @@ -1159,82 +1159,82 @@ JanetTable *janet_core_env(JanetTable *replacements) { janet_quick_asm(env, JANET_FUN_CMP, "cmp", 2, 2, 2, 2, cmp_asm, sizeof(cmp_asm), JDOC("(cmp x y)\n\n" - "Returns -1 if x is strictly less than y, 1 if y is strictly greater " - "than x, and 0 otherwise. To return 0, x and y must be the exact same type.")); + "Returns -1 if x is strictly less than y, 1 if y is strictly greater " + "than x, and 0 otherwise. To return 0, x and y must be the exact same type.")); janet_quick_asm(env, JANET_FUN_NEXT, "next", 2, 1, 2, 2, next_asm, sizeof(next_asm), JDOC("(next ds &opt key)\n\n" - "Gets the next key in a data structure. Can be used to iterate through " - "the keys of a data structure in an unspecified order. Keys are guaranteed " - "to be seen only once per iteration if the data structure is not mutated " - "during iteration. If key is nil, next returns the first key. If next " - "returns nil, there are no more keys to iterate through.")); + "Gets the next key in a data structure. Can be used to iterate through " + "the keys of a data structure in an unspecified order. Keys are guaranteed " + "to be seen only once per iteration if the data structure is not mutated " + "during iteration. If key is nil, next returns the first key. If next " + "returns nil, there are no more keys to iterate through.")); janet_quick_asm(env, JANET_FUN_PROP, "propagate", 2, 2, 2, 2, propagate_asm, sizeof(propagate_asm), JDOC("(propagate x fiber)\n\n" - "Propagate a signal from a fiber to the current fiber and " - "set the last value of the current fiber to `x`. The signal " - "value is then available as the status of the current fiber. " - "The resulting stack trace from the current fiber will include " - "frames from fiber. If fiber is in a state that can be resumed, " - "resuming the current fiber will first resume `fiber`. " - "This function can be used to re-raise an error without losing " - "the original stack trace.")); + "Propagate a signal from a fiber to the current fiber and " + "set the last value of the current fiber to `x`. The signal " + "value is then available as the status of the current fiber. " + "The resulting stack trace from the current fiber will include " + "frames from fiber. If fiber is in a state that can be resumed, " + "resuming the current fiber will first resume `fiber`. " + "This function can be used to re-raise an error without losing " + "the original stack trace.")); janet_quick_asm(env, JANET_FUN_DEBUG, "debug", 1, 0, 1, 1, debug_asm, sizeof(debug_asm), JDOC("(debug &opt x)\n\n" - "Throws a debug signal that can be caught by a parent fiber and used to inspect " - "the running state of the current fiber. Returns the value passed in by resume.")); + "Throws a debug signal that can be caught by a parent fiber and used to inspect " + "the running state of the current fiber. Returns the value passed in by resume.")); janet_quick_asm(env, JANET_FUN_ERROR, "error", 1, 1, 1, 1, error_asm, sizeof(error_asm), JDOC("(error e)\n\n" - "Throws an error e that can be caught and handled by a parent fiber.")); + "Throws an error e that can be caught and handled by a parent fiber.")); janet_quick_asm(env, JANET_FUN_YIELD, "yield", 1, 0, 1, 2, yield_asm, sizeof(yield_asm), JDOC("(yield &opt x)\n\n" - "Yield a value to a parent fiber. When a fiber yields, its execution is paused until " - "another thread resumes it. The fiber will then resume, and the last yield call will " - "return the value that was passed to resume.")); + "Yield a value to a parent fiber. When a fiber yields, its execution is paused until " + "another thread resumes it. The fiber will then resume, and the last yield call will " + "return the value that was passed to resume.")); janet_quick_asm(env, JANET_FUN_CANCEL, "cancel", 2, 2, 2, 2, cancel_asm, sizeof(cancel_asm), JDOC("(cancel fiber err)\n\n" - "Resume a fiber but have it immediately raise an error. This lets a programmer unwind a pending fiber. " - "Returns the same result as resume.")); + "Resume a fiber but have it immediately raise an error. This lets a programmer unwind a pending fiber. " + "Returns the same result as resume.")); janet_quick_asm(env, JANET_FUN_RESUME, "resume", 2, 1, 2, 2, resume_asm, sizeof(resume_asm), JDOC("(resume fiber &opt x)\n\n" - "Resume a new or suspended fiber and optionally pass in a value to the fiber that " - "will be returned to the last yield in the case of a pending fiber, or the argument to " - "the dispatch function in the case of a new fiber. Returns either the return result of " - "the fiber's dispatch function, or the value from the next yield call in fiber.")); + "Resume a new or suspended fiber and optionally pass in a value to the fiber that " + "will be returned to the last yield in the case of a pending fiber, or the argument to " + "the dispatch function in the case of a new fiber. Returns either the return result of " + "the fiber's dispatch function, or the value from the next yield call in fiber.")); janet_quick_asm(env, JANET_FUN_IN, "in", 3, 2, 3, 4, in_asm, sizeof(in_asm), JDOC("(in ds key &opt dflt)\n\n" - "Get value in ds at key, works on associative data structures. Arrays, tuples, tables, structs, " - "strings, symbols, and buffers are all associative and can be used. Arrays, tuples, strings, buffers, " - "and symbols must use integer keys that are in bounds or an error is raised. Structs and tables can " - "take any value as a key except nil and will return nil or dflt if not found.")); + "Get value in ds at key, works on associative data structures. Arrays, tuples, tables, structs, " + "strings, symbols, and buffers are all associative and can be used. Arrays, tuples, strings, buffers, " + "and symbols must use integer keys that are in bounds or an error is raised. Structs and tables can " + "take any value as a key except nil and will return nil or dflt if not found.")); janet_quick_asm(env, JANET_FUN_GET, "get", 3, 2, 3, 4, get_asm, sizeof(in_asm), JDOC("(get ds key &opt dflt)\n\n" - "Get the value mapped to key in data structure ds, and return dflt or nil if not found. " - "Similar to in, but will not throw an error if the key is invalid for the data structure " - "unless the data structure is an abstract type. In that case, the abstract type getter may throw " - "an error.")); + "Get the value mapped to key in data structure ds, and return dflt or nil if not found. " + "Similar to in, but will not throw an error if the key is invalid for the data structure " + "unless the data structure is an abstract type. In that case, the abstract type getter may throw " + "an error.")); janet_quick_asm(env, JANET_FUN_PUT, "put", 3, 3, 3, 3, put_asm, sizeof(put_asm), JDOC("(put ds key value)\n\n" - "Associate a key with a value in any mutable associative data structure. Indexed data structures " - "(arrays and buffers) only accept non-negative integer keys, and will expand if an out of bounds " - "value is provided. In an array, extra space will be filled with nils, and in a buffer, extra " - "space will be filled with 0 bytes. In a table, putting a key that is contained in the table prototype " - "will hide the association defined by the prototype, but will not mutate the prototype table. Putting " - "a value nil into a table will remove the key from the table. Returns the data structure ds.")); + "Associate a key with a value in any mutable associative data structure. Indexed data structures " + "(arrays and buffers) only accept non-negative integer keys, and will expand if an out of bounds " + "value is provided. In an array, extra space will be filled with nils, and in a buffer, extra " + "space will be filled with 0 bytes. In a table, putting a key that is contained in the table prototype " + "will hide the association defined by the prototype, but will not mutate the prototype table. Putting " + "a value nil into a table will remove the key from the table. Returns the data structure ds.")); janet_quick_asm(env, JANET_FUN_LENGTH, "length", 1, 1, 1, 1, length_asm, sizeof(length_asm), JDOC("(length ds)\n\n" - "Returns the length or count of a data structure in constant time as an integer. For " - "structs and tables, returns the number of key-value pairs in the data structure.")); + "Returns the length or count of a data structure in constant time as an integer. For " + "structs and tables, returns the number of key-value pairs in the data structure.")); janet_quick_asm(env, JANET_FUN_BNOT, "bnot", 1, 1, 1, 1, bnot_asm, sizeof(bnot_asm), JDOC("(bnot x)\n\nReturns the bit-wise inverse of integer x.")); @@ -1243,74 +1243,74 @@ JanetTable *janet_core_env(JanetTable *replacements) { /* Variadic ops */ templatize_varop(env, JANET_FUN_ADD, "+", 0, 0, JOP_ADD, JDOC("(+ & xs)\n\n" - "Returns the sum of all xs. xs must be integers or real numbers only. If xs is empty, return 0.")); + "Returns the sum of all xs. xs must be integers or real numbers only. If xs is empty, return 0.")); templatize_varop(env, JANET_FUN_SUBTRACT, "-", 0, 0, JOP_SUBTRACT, JDOC("(- & xs)\n\n" - "Returns the difference of xs. If xs is empty, returns 0. If xs has one element, returns the " - "negative value of that element. Otherwise, returns the first element in xs minus the sum of " - "the rest of the elements.")); + "Returns the difference of xs. If xs is empty, returns 0. If xs has one element, returns the " + "negative value of that element. Otherwise, returns the first element in xs minus the sum of " + "the rest of the elements.")); templatize_varop(env, JANET_FUN_MULTIPLY, "*", 1, 1, JOP_MULTIPLY, JDOC("(* & xs)\n\n" - "Returns the product of all elements in xs. If xs is empty, returns 1.")); + "Returns the product of all elements in xs. If xs is empty, returns 1.")); templatize_varop(env, JANET_FUN_DIVIDE, "/", 1, 1, JOP_DIVIDE, JDOC("(/ & xs)\n\n" - "Returns the quotient of xs. If xs is empty, returns 1. If xs has one value x, returns " - "the reciprocal of x. Otherwise return the first value of xs repeatedly divided by the remaining " - "values.")); + "Returns the quotient of xs. If xs is empty, returns 1. If xs has one value x, returns " + "the reciprocal of x. Otherwise return the first value of xs repeatedly divided by the remaining " + "values.")); templatize_varop(env, JANET_FUN_DIVIDE_FLOOR, "div", 1, 1, JOP_DIVIDE_FLOOR, JDOC("(div & xs)\n\n" - "Returns the floored division of xs. If xs is empty, returns 1. If xs has one value x, returns " - "the reciprocal of x. Otherwise return the first value of xs repeatedly divided by the remaining " - "values.")); + "Returns the floored division of xs. If xs is empty, returns 1. If xs has one value x, returns " + "the reciprocal of x. Otherwise return the first value of xs repeatedly divided by the remaining " + "values.")); templatize_varop(env, JANET_FUN_MODULO, "mod", 0, 1, JOP_MODULO, JDOC("(mod & xs)\n\n" - "Returns the result of applying the modulo operator on the first value of xs with each remaining value. " - "`(mod x 0)` is defined to be `x`.")); + "Returns the result of applying the modulo operator on the first value of xs with each remaining value. " + "`(mod x 0)` is defined to be `x`.")); templatize_varop(env, JANET_FUN_REMAINDER, "%", 0, 1, JOP_REMAINDER, JDOC("(% & xs)\n\n" - "Returns the remainder of dividing the first value of xs by each remaining value.")); + "Returns the remainder of dividing the first value of xs by each remaining value.")); templatize_varop(env, JANET_FUN_BAND, "band", -1, -1, JOP_BAND, JDOC("(band & xs)\n\n" - "Returns the bit-wise and of all values in xs. Each x in xs must be an integer.")); + "Returns the bit-wise and of all values in xs. Each x in xs must be an integer.")); templatize_varop(env, JANET_FUN_BOR, "bor", 0, 0, JOP_BOR, JDOC("(bor & xs)\n\n" - "Returns the bit-wise or of all values in xs. Each x in xs must be an integer.")); + "Returns the bit-wise or of all values in xs. Each x in xs must be an integer.")); templatize_varop(env, JANET_FUN_BXOR, "bxor", 0, 0, JOP_BXOR, JDOC("(bxor & xs)\n\n" - "Returns the bit-wise xor of all values in xs. Each in xs must be an integer.")); + "Returns the bit-wise xor of all values in xs. Each in xs must be an integer.")); templatize_varop(env, JANET_FUN_LSHIFT, "blshift", 1, 1, JOP_SHIFT_LEFT, JDOC("(blshift x & shifts)\n\n" - "Returns the value of x bit shifted left by the sum of all values in shifts. x " - "and each element in shift must be an integer.")); + "Returns the value of x bit shifted left by the sum of all values in shifts. x " + "and each element in shift must be an integer.")); templatize_varop(env, JANET_FUN_RSHIFT, "brshift", 1, 1, JOP_SHIFT_RIGHT, JDOC("(brshift x & shifts)\n\n" - "Returns the value of x bit shifted right by the sum of all values in shifts. x " - "and each element in shift must be an integer.")); + "Returns the value of x bit shifted right by the sum of all values in shifts. x " + "and each element in shift must be an integer.")); templatize_varop(env, JANET_FUN_RSHIFTU, "brushift", 1, 1, JOP_SHIFT_RIGHT_UNSIGNED, JDOC("(brushift x & shifts)\n\n" - "Returns the value of x bit shifted right by the sum of all values in shifts. x " - "and each element in shift must be an integer. The sign of x is not preserved, so " - "for positive shifts the return value will always be positive.")); + "Returns the value of x bit shifted right by the sum of all values in shifts. x " + "and each element in shift must be an integer. The sign of x is not preserved, so " + "for positive shifts the return value will always be positive.")); /* Variadic comparators */ templatize_comparator(env, JANET_FUN_GT, ">", 0, JOP_GREATER_THAN, JDOC("(> & xs)\n\n" - "Check if xs is in descending order. Returns a boolean.")); + "Check if xs is in descending order. Returns a boolean.")); templatize_comparator(env, JANET_FUN_LT, "<", 0, JOP_LESS_THAN, JDOC("(< & xs)\n\n" - "Check if xs is in ascending order. Returns a boolean.")); + "Check if xs is in ascending order. Returns a boolean.")); templatize_comparator(env, JANET_FUN_GTE, ">=", 0, JOP_GREATER_THAN_EQUAL, JDOC("(>= & xs)\n\n" - "Check if xs is in non-ascending order. Returns a boolean.")); + "Check if xs is in non-ascending order. Returns a boolean.")); templatize_comparator(env, JANET_FUN_LTE, "<=", 0, JOP_LESS_THAN_EQUAL, JDOC("(<= & xs)\n\n" - "Check if xs is in non-descending order. Returns a boolean.")); + "Check if xs is in non-descending order. Returns a boolean.")); templatize_comparator(env, JANET_FUN_EQ, "=", 0, JOP_EQUALS, JDOC("(= & xs)\n\n" - "Check if all values in xs are equal. Returns a boolean.")); + "Check if all values in xs are equal. Returns a boolean.")); templatize_comparator(env, JANET_FUN_NEQ, "not=", 1, JOP_EQUALS, JDOC("(not= & xs)\n\n" - "Check if any values in xs are not equal. Returns a boolean.")); + "Check if any values in xs are not equal. Returns a boolean.")); /* Platform detection */ janet_def(env, "janet/version", janet_cstringv(JANET_VERSION), @@ -1319,7 +1319,7 @@ JanetTable *janet_core_env(JanetTable *replacements) { JDOC("The build identifier of the running janet program.")); janet_def(env, "janet/config-bits", janet_wrap_integer(JANET_CURRENT_CONFIG_BITS), JDOC("The flag set of config options from janetconf.h which is used to check " - "if native modules are compatible with the host program.")); + "if native modules are compatible with the host program.")); /* Allow references to the environment */ janet_def(env, "root-env", janet_wrap_table(env), From 611b2a6c3a84532e69d44ccff24cc5bc0b3c72fd Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Thu, 19 Dec 2024 18:37:51 -0600 Subject: [PATCH 21/47] Add more test cases for #1535 --- test/suite-boot.janet | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/suite-boot.janet b/test/suite-boot.janet index bcaa1f9d..d77611a6 100644 --- a/test/suite-boot.janet +++ b/test/suite-boot.janet @@ -1001,5 +1001,8 @@ @{:key1 "value1" @"key" "value2"}) "deep= mutable keys")) (assert (deep-not= {"abc" 123} {@"abc" 123}) "deep= mutable keys vs immutable key") (assert (deep= {@"" 1 @"" 2 @"" 3} {@"" 1 @"" 2 @"" 3}) "deep= duplicate mutable keys") +(assert (deep= {@"" @"" @"" @"" @"" 3} {@"" @"" @"" @"" @"" 3}) "deep= duplicate mutable keys 2") +(assert (deep= {@[] @"" @[] @"" @[] 3} {@[] @"" @[] @"" @[] 3}) "deep= duplicate mutable keys 3") +(assert (deep= {@{} @"" @{} @"" @{} 3} {@{} @"" @{} @"" @{} 3}) "deep= duplicate mutable keys 4") (end-suite) From 682f0f584f0af96d79f27cfb191835eae7ece2ed Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Thu, 19 Dec 2024 19:31:01 -0600 Subject: [PATCH 22/47] freeze with mutable keys should be determinsic help address #1535 --- src/boot/boot.janet | 20 ++++++++++++-------- test/suite-boot.janet | 11 +++++++++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/boot/boot.janet b/src/boot/boot.janet index dc16fec4..78c009f2 100644 --- a/src/boot/boot.janet +++ b/src/boot/boot.janet @@ -2224,14 +2224,18 @@ child values also immutable. Closures, fibers, and abstract types will not be recursively frozen, but all other types will.` [x] - (case (type x) - :array (tuple/slice (map freeze x)) - :tuple (tuple/slice (map freeze x)) - :table (if-let [p (table/getproto x)] - (freeze (merge (table/clone p) x)) - (struct ;(map freeze (kvs x)))) - :struct (struct ;(map freeze (kvs x))) - :buffer (string x) + (def tx (type x)) + (cond + (or (= tx :array) (= tx :tuple)) + (tuple/slice (map freeze x)) + + (or (= tx :table) (= tx :struct)) + (let [sorted-kvs (array/join @[] ;(sort (map freeze (pairs x))))] + (struct/with-proto (freeze (getproto x)) ;sorted-kvs)) + + (= tx :buffer) + (string x) + x)) (defn thaw diff --git a/test/suite-boot.janet b/test/suite-boot.janet index d77611a6..045f0368 100644 --- a/test/suite-boot.janet +++ b/test/suite-boot.janet @@ -896,11 +896,18 @@ (struct/with-proto {:a [1 2 3]} :c 22 :b [1 2 3 4] :d "test" :e "test2")) (table/setproto table-to-freeze @{:a @[1 2 3]}) -(assert (deep= {:a [1 2 3] :b [1 2 3 4] :c 22 :d "test" :e "test2"} - (freeze table-to-freeze))) +(assert (deep= struct-to-thaw (freeze table-to-freeze))) (assert (deep= table-to-freeze-with-inline-proto (thaw table-to-freeze))) (assert (deep= table-to-freeze-with-inline-proto (thaw struct-to-thaw))) +# Check that freezing mutable keys is deterministic +# for issue #1535 +(def hashes @{}) +(repeat 200 + (def x (freeze {@"" 1 @"" 2 @"" 3 @"" 4 @"" 5})) + (put hashes (hash x) true)) +(assert (= 1 (length hashes)) "freeze mutable keys is deterministic") + # Make sure Carriage Returns don't end up in doc strings # e528b86 (assert (not (string/find "\r" From 1b49934e4fd6b53e1116d5964fa639dbe28aa065 Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Thu, 19 Dec 2024 19:41:19 -0600 Subject: [PATCH 23/47] Allow table/to-struct to take a prototype. Use this prototype struct in freeze. --- src/boot/boot.janet | 12 ++++++++++-- src/core/table.c | 12 +++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/boot/boot.janet b/src/boot/boot.janet index 78c009f2..f0b58631 100644 --- a/src/boot/boot.janet +++ b/src/boot/boot.janet @@ -2230,8 +2230,16 @@ (tuple/slice (map freeze x)) (or (= tx :table) (= tx :struct)) - (let [sorted-kvs (array/join @[] ;(sort (map freeze (pairs x))))] - (struct/with-proto (freeze (getproto x)) ;sorted-kvs)) + (let [temp-tab @{}] + # Handle multiple unique keys that freeze. Result should + # be independent of iteration order. + (eachp [k v] x + (def kk (freeze k)) + (def vv (freeze v)) + (def old (get temp-tab kk)) + (def new (if (= nil old) vv (max vv old))) + (put temp-tab kk new)) + (table/to-struct temp-tab (freeze (getproto x)))) (= tx :buffer) (string x) diff --git a/src/core/table.c b/src/core/table.c index 2e82b1ab..229d8fa3 100644 --- a/src/core/table.c +++ b/src/core/table.c @@ -372,12 +372,14 @@ JANET_CORE_FN(cfun_table_setproto, } JANET_CORE_FN(cfun_table_tostruct, - "(table/to-struct tab)", - "Convert a table to a struct. Returns a new struct. This function " - "does not take into account prototype tables.") { - janet_fixarity(argc, 1); + "(table/to-struct tab &opt proto)", + "Convert a table to a struct. Returns a new struct.") { + janet_arity(argc, 1, 2); JanetTable *t = janet_gettable(argv, 0); - return janet_wrap_struct(janet_table_to_struct(t)); + JanetStruct proto = janet_optstruct(argv, argc, 1, NULL); + JanetStruct st = janet_table_to_struct(t); + janet_struct_proto(st) = proto; + return janet_wrap_struct(st); } JANET_CORE_FN(cfun_table_rawget, From 746ced55019d07f3de25e40ce6efb20dbd4ae95b Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Sat, 21 Dec 2024 08:58:05 -0600 Subject: [PATCH 24/47] Revert behavior of deep= on mutable keys. Mutable keys are a minefield for comparisons, as resolving equality require re-implementing a lot of the internal structures, as well as dealing with multiple mutable keys that are in the same equivalency class by deep=. Simplifying the implementation to not resole mutable keys is much simpler, faster, and has the benefit that deep= and deep-not= do not need to allocate. --- CHANGELOG.md | 1 + src/boot/boot.janet | 16 ++-------------- src/core/struct.c | 11 +++++++++++ test/suite-boot.janet | 16 ++++++++++------ test/suite-marsh.janet | 4 ++-- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13145525..a83876c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ All notable changes to this project will be documented in this file. ## ??? - Unreleased +- Add `struct/rawget` - Fix `deep=` and `deep-not=` to better handle degenerate cases with mutable table keys - Long strings will now dedent on `\r\n` instead of just `\n`. - Add `ev/to-file` for synchronous resource operations diff --git a/src/boot/boot.janet b/src/boot/boot.janet index f0b58631..577a67fb 100644 --- a/src/boot/boot.janet +++ b/src/boot/boot.janet @@ -2259,8 +2259,6 @@ :string (buffer ds) ds)) -(def- mutable-types {:table true :array true :buffer true}) - (defn deep-not= ``Like `not=`, but mutable types (arrays, tables, buffers) are considered equal if they have identical structure. Much slower than `not=`.`` @@ -2282,20 +2280,10 @@ (or (= tx :struct) (= tx :table)) (or (not= (length x) (length y)) (do + (def rawget (if (= tx :struct) struct/rawget table/rawget)) (var ret false) - (def mut-keys-x @{}) (eachp [k v] x - (if (get mutable-types (type k)) - (let [kk (freeze k)] - (put mut-keys-x kk (put (get mut-keys-x kk @{}) (freeze v) true))) - (if (deep-not= (get y k) v) (break (set ret true))))) - (when (next mut-keys-x) # handle case when we have mutable keys separately - (def mut-keys-y @{}) - (eachp [k v] y - (if (get mutable-types (type k)) - (let [kk (freeze k)] - (put mut-keys-y kk (put (get mut-keys-y kk @{}) (freeze v) true))))) - (set ret (deep-not= mut-keys-x mut-keys-y))) + (if (deep-not= (rawget y k) v) (break (set ret true)))) ret)) (= tx :buffer) (not= 0 (- (length x) (length y)) (memcmp x y)) (not= x y)))) diff --git a/src/core/struct.c b/src/core/struct.c index acc9a921..2e4be627 100644 --- a/src/core/struct.c +++ b/src/core/struct.c @@ -294,6 +294,16 @@ JANET_CORE_FN(cfun_struct_to_table, return janet_wrap_table(tab); } +JANET_CORE_FN(cfun_struct_rawget, + "(struct/rawget st key)", + "Gets a value from a struct `st` without looking at the prototype struct. " + "If `st` does not contain the key directly, the function will return " + "nil without checking the prototype. Returns the value in the struct.") { + janet_fixarity(argc, 2); + JanetStruct st = janet_getstruct(argv, 0); + return janet_struct_rawget(st, argv[1]); +} + /* Load the struct module */ void janet_lib_struct(JanetTable *env) { JanetRegExt struct_cfuns[] = { @@ -301,6 +311,7 @@ void janet_lib_struct(JanetTable *env) { JANET_CORE_REG("struct/getproto", cfun_struct_getproto), JANET_CORE_REG("struct/proto-flatten", cfun_struct_flatten), JANET_CORE_REG("struct/to-table", cfun_struct_to_table), + JANET_CORE_REG("struct/rawget", cfun_struct_rawget), JANET_REG_END }; janet_core_cfuns_ext(env, NULL, struct_cfuns); diff --git a/test/suite-boot.janet b/test/suite-boot.janet index 045f0368..b8f22625 100644 --- a/test/suite-boot.janet +++ b/test/suite-boot.janet @@ -1004,12 +1004,16 @@ # issue #1535 (loop [i :range [1 1000]] - (assert (deep= @{:key1 "value1" @"key" "value2"} - @{:key1 "value1" @"key" "value2"}) "deep= mutable keys")) + (assert (deep-not= @{:key1 "value1" @"key" "value2"} + @{:key1 "value1" @"key" "value2"}) "deep= mutable keys")) (assert (deep-not= {"abc" 123} {@"abc" 123}) "deep= mutable keys vs immutable key") -(assert (deep= {@"" 1 @"" 2 @"" 3} {@"" 1 @"" 2 @"" 3}) "deep= duplicate mutable keys") -(assert (deep= {@"" @"" @"" @"" @"" 3} {@"" @"" @"" @"" @"" 3}) "deep= duplicate mutable keys 2") -(assert (deep= {@[] @"" @[] @"" @[] 3} {@[] @"" @[] @"" @[] 3}) "deep= duplicate mutable keys 3") -(assert (deep= {@{} @"" @{} @"" @{} 3} {@{} @"" @{} @"" @{} 3}) "deep= duplicate mutable keys 4") +(assert (deep-not= {@"" 1 @"" 2 @"" 3} {@"" 1 @"" 2 @"" 3}) "deep= duplicate mutable keys") +(assert (deep-not= {@"" @"" @"" @"" @"" 3} {@"" @"" @"" @"" @"" 3}) "deep= duplicate mutable keys 2") +(assert (deep-not= {@[] @"" @[] @"" @[] 3} {@[] @"" @[] @"" @[] 3}) "deep= duplicate mutable keys 3") +(assert (deep-not= {@{} @"" @{} @"" @{} 3} {@{} @"" @{} @"" @{} 3}) "deep= duplicate mutable keys 4") +(assert (deep-not= @{:key1 "value1" @"key2" @"value2"} + @{:key1 "value1" @"key2" "value2"}) "deep= mutable keys") +(assert (deep-not= @{:key1 "value1" [@"key2"] @"value2"} + @{:key1 "value1" [@"key2"] @"value2"}) "deep= mutable keys") (end-suite) diff --git a/test/suite-marsh.janet b/test/suite-marsh.janet index b9f4d277..1bcfb0d5 100644 --- a/test/suite-marsh.janet +++ b/test/suite-marsh.janet @@ -207,7 +207,7 @@ neldb\0\0\0\xD8\x05printG\x01\0\xDE\xDE\xDE'\x03\0marshal_tes/\x02 (assert (= 2 (length tclone)) "table/weak-values marsh 2") (gccollect) (assert (= 1 (length t)) "table/weak-value marsh 3") -(assert (deep= t tclone) "table/weak-values marsh 4") +(assert (deep= (freeze t) (freeze tclone)) "table/weak-values marsh 4") # tables with prototypes (def t (table/weak-values 1)) @@ -219,7 +219,7 @@ neldb\0\0\0\xD8\x05printG\x01\0\xDE\xDE\xDE'\x03\0marshal_tes/\x02 (assert (= 2 (length tclone)) "marsh weak tables with prototypes 2") (gccollect) (assert (= 1 (length t)) "marsh weak tables with prototypes 3") -(assert (deep= t tclone) "marsh weak tables with prototypes 4") +(assert (deep= (freeze t) (freeze tclone)) "marsh weak tables with prototypes 4") (assert (deep= (getproto t) (getproto tclone)) "marsh weak tables with prototypes 5") (end-suite) From 753911fe2df6c4fd42cd6e824551f9c6716851a6 Mon Sep 17 00:00:00 2001 From: sogaiu <983021772@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:02:45 +0900 Subject: [PATCH 25/47] Tweak *exit-value* doc - address #1537 --- src/boot/boot.janet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/boot/boot.janet b/src/boot/boot.janet index 577a67fb..16e7f48a 100644 --- a/src/boot/boot.janet +++ b/src/boot/boot.janet @@ -1311,7 +1311,7 @@ (defdyn *redef* "When set, allow dynamically rebinding top level defs. Will slow generated code and is intended to be used for development.") (defdyn *debug* "Enables a built in debugger on errors and other useful features for debugging in a repl.") (defdyn *exit* "When set, will cause the current context to complete. Can be set to exit from repl (or file), for example.") -(defdyn *exit-value* "Set the return value from `run-context` upon an exit. By default, `run-context` will return nil.") +(defdyn *exit-value* "Set the return value from `run-context` upon an exit.") (defdyn *task-id* "When spawning a thread or fiber, the task-id can be assigned for concurrency control.") (defdyn *current-file* From ba5990ef2106de2d633609dd7bb0f913be0b6c43 Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Tue, 31 Dec 2024 08:52:37 -0600 Subject: [PATCH 26/47] Add switch to turn off "reuse" behavior for server sockets. --- src/core/net.c | 28 ++++++++++++++++------------ test/suite-ev.janet | 4 ++++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/core/net.c b/src/core/net.c index 69e891fc..f38a20f6 100644 --- a/src/core/net.c +++ b/src/core/net.c @@ -578,17 +578,19 @@ JANET_CORE_FN(cfun_net_connect, net_sched_connect(stream); } -static const char *serverify_socket(JSock sfd) { +static const char *serverify_socket(JSock sfd, int reuse) { /* Set various socket options */ int enable = 1; - if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (char *) &enable, sizeof(int)) < 0) { - return "setsockopt(SO_REUSEADDR) failed"; - } + if (reuse) { + if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (char *) &enable, sizeof(int)) < 0) { + return "setsockopt(SO_REUSEADDR) failed"; + } #ifdef SO_REUSEPORT - if (setsockopt(sfd, SOL_SOCKET, SO_REUSEPORT, &enable, sizeof(int)) < 0) { - return "setsockopt(SO_REUSEPORT) failed"; - } + if (setsockopt(sfd, SOL_SOCKET, SO_REUSEPORT, &enable, sizeof(int)) < 0) { + return "setsockopt(SO_REUSEPORT) failed"; + } #endif + } janet_net_socknoblock(sfd); return NULL; } @@ -642,19 +644,21 @@ JANET_CORE_FN(cfun_net_shutdown, } JANET_CORE_FN(cfun_net_listen, - "(net/listen host port &opt type)", + "(net/listen host port &opt type no-reuse)", "Creates a server. Returns a new stream that is neither readable nor " "writeable. Use net/accept or net/accept-loop be to handle connections and start the server. " "The type parameter specifies the type of network connection, either " "a :stream (usually tcp), or :datagram (usually udp). If not specified, the default is " - ":stream. The host and port arguments are the same as in net/address.") { + ":stream. The host and port arguments are the same as in net/address. The last boolean parameter `no-reuse` will " + "disable the use of SO_REUSEADDR and SO_REUSEPORT when creating a server on some operating systems.") { janet_sandbox_assert(JANET_SANDBOX_NET_LISTEN); - janet_arity(argc, 2, 3); + janet_arity(argc, 2, 4); /* Get host, port, and handler*/ int socktype = janet_get_sockettype(argv, argc, 2); int is_unix = 0; struct addrinfo *ai = janet_get_addrinfo(argv, 0, socktype, 1, &is_unix); + int reuse = !(argc >= 4 && janet_truthy(argv[3])); JSock sfd = JSOCKDEFAULT; #ifndef JANET_WINDOWS @@ -664,7 +668,7 @@ JANET_CORE_FN(cfun_net_listen, janet_free(ai); janet_panicf("could not create socket: %V", janet_ev_lasterr()); } - const char *err = serverify_socket(sfd); + const char *err = serverify_socket(sfd, reuse); if (NULL != err || bind(sfd, (struct sockaddr *)ai, sizeof(struct sockaddr_un))) { JSOCKCLOSE(sfd); janet_free(ai); @@ -687,7 +691,7 @@ JANET_CORE_FN(cfun_net_listen, sfd = socket(rp->ai_family, rp->ai_socktype | JSOCKFLAGS, rp->ai_protocol); #endif if (!JSOCKVALID(sfd)) continue; - const char *err = serverify_socket(sfd); + const char *err = serverify_socket(sfd, reuse); if (NULL != err) { JSOCKCLOSE(sfd); continue; diff --git a/test/suite-ev.janet b/test/suite-ev.janet index 724bf24a..0ed19413 100644 --- a/test/suite-ev.janet +++ b/test/suite-ev.janet @@ -410,6 +410,10 @@ (ev/call handler connection) (break)))) +# Make sure we can't bind again with no-reuse +(assert-error "no-reuse" + (net/listen test-host test-port :stream true)) + # Read from socket (defn expect-read From a47eb847fbd5e7586019b9a796a7ed01e6912a4d Mon Sep 17 00:00:00 2001 From: Bob Tolbert Date: Tue, 31 Dec 2024 08:55:25 -0600 Subject: [PATCH 27/47] Brief: Add Arm64 .msi support on Windows Summary: Small update to add Windows on Arm64 support. Also requires the latest version 3.14 of the WiX toolset. --- README.md | 2 +- build_win.bat | 10 ++++++++-- tools/msi/janet.wxs | 5 +++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7376b433..5a37618d 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ Alternatively, install the package directly with `pkgin install janet`. To build an `.msi` installer executable, in addition to the above steps, you will have to: -5. Install, or otherwise add to your PATH the [WiX 3.11 Toolset](https://github.com/wixtoolset/wix3/releases). +5. Install, or otherwise add to your PATH the [WiX 3.14 Toolset](https://github.com/wixtoolset/wix3/releases). 6. Run `build_win dist`. Now you should have an `.msi`. You can run `build_win install` to install the `.msi`, or execute the file itself. diff --git a/build_win.bat b/build_win.bat index a53d4931..03829667 100644 --- a/build_win.bat +++ b/build_win.bat @@ -91,7 +91,7 @@ exit /b 0 @rem Clean build artifacts :CLEAN -del *.exe *.lib *.exp +del *.exe *.lib *.exp *.msi *.wixpdb rd /s /q build if exist dist ( rd /s /q dist @@ -143,7 +143,13 @@ if defined CI ( ) else ( set WIXBIN= ) -%WIXBIN%candle.exe tools\msi\janet.wxs -arch %BUILDARCH% -out build\ + +set WIXARCH=%BUILDARCH% +if "%WIXARCH%"=="aarch64" ( + set WIXARCH=arm64 +) + +%WIXBIN%candle.exe tools\msi\janet.wxs -arch %WIXARCH% -out build\ %WIXBIN%light.exe "-sice:ICE38" -b tools\msi -ext WixUIExtension build\janet.wixobj -out janet-%RELEASE_VERSION%-windows-%BUILDARCH%-installer.msi exit /b 0 diff --git a/tools/msi/janet.wxs b/tools/msi/janet.wxs index 9ea2038d..1d2f60d5 100644 --- a/tools/msi/janet.wxs +++ b/tools/msi/janet.wxs @@ -19,6 +19,11 @@ + + + + + From 60d9f97750fff592d7d8444c4443dca263dd6eca Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Wed, 1 Jan 2025 11:26:43 -0600 Subject: [PATCH 28/47] Address issue #1539 - infinite loop in peg split Other looping rules ensure forward progress by terminating if an iteration is at the same location as the previous iteration. Do the same for split. --- src/core/peg.c | 37 ++++++++++++++++++++----------------- test/suite-peg.janet | 11 +++++++++++ 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/core/peg.c b/src/core/peg.c index 24d1d320..9a10b733 100644 --- a/src/core/peg.c +++ b/src/core/peg.c @@ -1,5 +1,5 @@ /* -* Copyright (c) 2024 Calvin Rose +* Copyright (c) 2025 Calvin Rose * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -549,36 +549,39 @@ tail: const uint32_t *rule_separator = s->bytecode + rule[1]; const uint32_t *rule_subpattern = s->bytecode + rule[2]; - const uint8_t *separator_end = NULL; - do { - const uint8_t *text_start = text; + const uint8_t *chunk_start = text; + const uint8_t *chunk_end = NULL; + + while (text <= saved_end) { + /* Find next split (or end of text) */ CapState cs = cap_save(s); down1(s); - while (text <= s->text_end) { - separator_end = peg_rule(s, rule_separator, text); + while (text <= saved_end) { + chunk_end = text; + const uint8_t *check = peg_rule(s, rule_separator, text); cap_load(s, cs); - if (separator_end) { + if (check) { + text = check; break; } text++; } up1(s); - if (separator_end) { - s->text_end = text; - text = separator_end; - } - + /* Match between splits */ + s->text_end = chunk_end; down1(s); - const uint8_t *subpattern_end = peg_rule(s, rule_subpattern, text_start); + const uint8_t *subpattern_end = peg_rule(s, rule_subpattern, chunk_start); up1(s); s->text_end = saved_end; + if (!subpattern_end) return NULL; /* Don't match anything */ - if (!subpattern_end) { - return NULL; - } - } while (separator_end); + /* Ensure forward progress */ + if (text == chunk_start) return NULL; + chunk_start = text; + } + s->text_end = saved_end; return s->text_end; } diff --git a/test/suite-peg.janet b/test/suite-peg.janet index ac426cfc..7f7a83d6 100644 --- a/test/suite-peg.janet +++ b/test/suite-peg.janet @@ -772,5 +772,16 @@ "5:apple6:banana6:cherry" @["apple" "banana" "cherry"]) +# Issue #1539 - make sure split with "" doesn't infinite loop/oom +(test "issue 1539" + ~(split "" (capture (to -1))) + "hello there friends" + nil) + +(test "issue 1539 pt. 2" + ~(split "," (capture 0)) + "abc123,,,," + @["" "" "" "" ""]) + (end-suite) From 8a6b44cb4e0f59d4752482c92f2cd4cda1aebab1 Mon Sep 17 00:00:00 2001 From: Florian Beeres Date: Sat, 11 Jan 2025 22:42:58 +0100 Subject: [PATCH 29/47] docstring int/to-number: supports int64, not int32 Update the cfun_to_number docstring to indicate that it handles values up to JANET_INTMAX_INT64 (75845c028364b8a25f06d620a61851cfa4505e64), rather than up to int32, what the current docstring says. --- src/core/inttypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/inttypes.c b/src/core/inttypes.c index f26dbdf6..39ef4ea6 100644 --- a/src/core/inttypes.c +++ b/src/core/inttypes.c @@ -205,7 +205,7 @@ JANET_CORE_FN(cfun_it_u64_new, JANET_CORE_FN(cfun_to_number, "(int/to-number value)", - "Convert an int/u64 or int/s64 to a number. Fails if the number is out of range for an int32.") { + "Convert an int/u64 or int/s64 to a number. Fails if the number is out of range for an int64.") { janet_fixarity(argc, 1); if (janet_type(argv[0]) == JANET_ABSTRACT) { void *abst = janet_unwrap_abstract(argv[0]); From a17ae977a57ff45e38a7ffee71ea1e612dde55ac Mon Sep 17 00:00:00 2001 From: Florian Beeres Date: Sat, 11 Jan 2025 22:51:21 +0100 Subject: [PATCH 30/47] docstring int/u64 int/s64: supports number as well Update the docstrings of the u64 and s64 functions to indicate that they work on numbers as well strings. Previously the docstring only mentioned string support. --- src/core/inttypes.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/inttypes.c b/src/core/inttypes.c index 39ef4ea6..855793d7 100644 --- a/src/core/inttypes.c +++ b/src/core/inttypes.c @@ -191,14 +191,14 @@ Janet janet_wrap_u64(uint64_t x) { JANET_CORE_FN(cfun_it_s64_new, "(int/s64 value)", - "Create a boxed signed 64 bit integer from a string value.") { + "Create a boxed signed 64 bit integer from a string value or a number.") { janet_fixarity(argc, 1); return janet_wrap_s64(janet_unwrap_s64(argv[0])); } JANET_CORE_FN(cfun_it_u64_new, "(int/u64 value)", - "Create a boxed unsigned 64 bit integer from a string value.") { + "Create a boxed unsigned 64 bit integer from a string value or a number.") { janet_fixarity(argc, 1); return janet_wrap_u64(janet_unwrap_u64(argv[0])); } From 06d581dde3595f372ef2ac94fcbcc1bc34bd1bae Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Mon, 20 Jan 2025 09:02:22 -0600 Subject: [PATCH 31/47] Fix #1546 - large ranges. Raise an error for very large ranges instead of internal assert. --- src/core/corelib.c | 3 ++- test/suite-corelib.janet | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/corelib.c b/src/core/corelib.c index 01775d19..e6715962 100644 --- a/src/core/corelib.c +++ b/src/core/corelib.c @@ -449,8 +449,9 @@ JANET_CORE_FN(janet_core_range, } count = (count > 0) ? count : 0; int32_t int_count; + janet_assert(count >= 0, "bad range code"); if (count > (double) INT32_MAX) { - int_count = INT32_MAX; + janet_panicf("range is too large, %f elements", count); } else { int_count = (int32_t) ceil(count); } diff --git a/test/suite-corelib.janet b/test/suite-corelib.janet index 3c209418..119ec768 100644 --- a/test/suite-corelib.janet +++ b/test/suite-corelib.janet @@ -174,6 +174,7 @@ (assert (deep= (range 0 17 4) @[0 4 8 12 16]) "(range 0 17 4)") (assert (deep= (range 16 0 -4) @[16 12 8 4]) "(range 16 0 -4)") (assert (deep= (range 17 0 -4) @[17 13 9 5 1]) "(range 17 0 -4)") +(assert-error "large range" (range 0xFFFFFFFFFF)) (assert (= (length (range 10)) 10) "(range 10)") (assert (= (length (range -10)) 0) "(range -10)") From 2b73a15ad82715dea5ad632ecc8ae4d0fc6c04f6 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Wed, 22 Jan 2025 23:47:44 +0900 Subject: [PATCH 32/47] Update CodeQL actions to latest version --- .github/workflows/codeql.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 5a603c48..38cba164 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -27,15 +27,15 @@ jobs: uses: actions/checkout@v3 - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} queries: +security-and-quality - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{ matrix.language }}" From 49f151e265b54838e1580a0f1b807c27a6ccece9 Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Wed, 22 Jan 2025 09:21:56 -0600 Subject: [PATCH 33/47] Allocate parser with GC. This fixes janet_dobytes, which manually allocated a parser on the stack, and then possibly ran garbage collection before it was deallocated. --- src/core/run.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/core/run.c b/src/core/run.c index bdd7205d..9942f07b 100644 --- a/src/core/run.c +++ b/src/core/run.c @@ -28,7 +28,7 @@ /* Run a string */ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char *sourcePath, Janet *out) { - JanetParser parser; + JanetParser *parser; int errflags = 0, done = 0; int32_t index = 0; Janet ret = janet_wrap_nil(); @@ -37,14 +37,16 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char if (where) janet_gcroot(janet_wrap_string(where)); if (NULL == sourcePath) sourcePath = ""; - janet_parser_init(&parser); + parser = janet_abstract(&janet_parser_type, sizeof(JanetParser)); + janet_parser_init(parser); + janet_gcroot(janet_wrap_abstract(parser)); /* While we haven't seen an error */ while (!done) { /* Evaluate parsed values */ - while (janet_parser_has_more(&parser)) { - Janet form = janet_parser_produce(&parser); + while (janet_parser_has_more(parser)) { + Janet form = janet_parser_produce(parser); JanetCompileResult cres = janet_compile(form, env, where); if (cres.status == JANET_COMPILE_OK) { JanetFunction *f = janet_thunk(cres.funcdef); @@ -58,8 +60,8 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char } } else { ret = janet_wrap_string(cres.error); - int32_t line = (int32_t) parser.line; - int32_t col = (int32_t) parser.column; + int32_t line = (int32_t) parser->line; + int32_t col = (int32_t) parser->column; if ((cres.error_mapping.line > 0) && (cres.error_mapping.column > 0)) { line = cres.error_mapping.line; @@ -81,16 +83,16 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char if (done) break; /* Dispatch based on parse state */ - switch (janet_parser_status(&parser)) { + switch (janet_parser_status(parser)) { case JANET_PARSE_DEAD: done = 1; break; case JANET_PARSE_ERROR: { - const char *e = janet_parser_error(&parser); + const char *e = janet_parser_error(parser); errflags |= 0x04; ret = janet_cstringv(e); - int32_t line = (int32_t) parser.line; - int32_t col = (int32_t) parser.column; + int32_t line = (int32_t) parser->line; + int32_t col = (int32_t) parser->column; janet_eprintf("%s:%d:%d: parse error: %s\n", sourcePath, line, col, e); done = 1; break; @@ -98,9 +100,9 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char case JANET_PARSE_ROOT: case JANET_PARSE_PENDING: if (index >= len) { - janet_parser_eof(&parser); + janet_parser_eof(parser); } else { - janet_parser_consume(&parser, bytes[index++]); + janet_parser_consume(parser, bytes[index++]); } break; } @@ -108,7 +110,7 @@ int janet_dobytes(JanetTable *env, const uint8_t *bytes, int32_t len, const char } /* Clean up and return errors */ - janet_parser_deinit(&parser); + janet_gcunroot(janet_wrap_abstract(parser)); if (where) janet_gcunroot(janet_wrap_string(where)); #ifdef JANET_EV /* Enter the event loop if we are not already in it */ From 5e93f0e34bda727dca80da04d0861b023d3b879b Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Fri, 24 Jan 2025 03:02:15 +0900 Subject: [PATCH 34/47] Trigger workflow to run again From f75c08a78eabd1c1727bb71f8139197528546298 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Fri, 24 Jan 2025 04:00:52 +0900 Subject: [PATCH 35/47] Add 'tools: linked' to CodeQL workflow --- .github/workflows/codeql.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 38cba164..0bbaa526 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -31,6 +31,7 @@ jobs: with: languages: ${{ matrix.language }} queries: +security-and-quality + tools: linked - name: Autobuild uses: github/codeql-action/autobuild@v3 From fa75a395cb2b48511ed2ad5bcda4e60c1ec2cfe7 Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Sun, 26 Jan 2025 09:48:48 -0600 Subject: [PATCH 36/47] Add support for buffer peg literals - address #1549 --- src/core/peg.c | 5 +++++ test/suite-peg.janet | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/src/core/peg.c b/src/core/peg.c index 9a10b733..a1e4f0d0 100644 --- a/src/core/peg.c +++ b/src/core/peg.c @@ -1419,6 +1419,11 @@ static uint32_t peg_compile1(Builder *b, Janet peg) { emit_bytes(b, RULE_LITERAL, len, str); break; } + case JANET_BUFFER: { + const JanetBuffer *buf = janet_unwrap_buffer(peg); + emit_bytes(b, RULE_LITERAL, buf->count, buf->data); + break; + } case JANET_TABLE: { /* Build grammar table */ JanetTable *new_grammar = janet_table_clone(janet_unwrap_table(peg)); diff --git a/test/suite-peg.janet b/test/suite-peg.janet index 7f7a83d6..2472374b 100644 --- a/test/suite-peg.janet +++ b/test/suite-peg.janet @@ -783,5 +783,11 @@ "abc123,,,," @["" "" "" "" ""]) +# Issue #1549 - allow buffers as peg literals +(test "issue 1549" + ''@"abc123" + "abc123" + @["abc123"]) + (end-suite) From f63a33884f411e312d7ad59195f3887dff87990e Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Thu, 30 Jan 2025 07:36:18 -0600 Subject: [PATCH 37/47] Add hexfloats - Address #1550 --- src/core/strtod.c | 12 ++++++++++-- test/suite-parse.janet | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/core/strtod.c b/src/core/strtod.c index 67247aa6..17fbc450 100644 --- a/src/core/strtod.c +++ b/src/core/strtod.c @@ -298,8 +298,10 @@ int janet_scan_number_base( } /* If still base is 0, set to default (10) */ + int exp_base = base; if (base == 0) { base = 10; + exp_base = 10; } /* Skip leading zeros */ @@ -322,6 +324,12 @@ int janet_scan_number_base( } else if (*str == '&') { foundexp = 1; break; + } else if (base == 16 && (*str == 'P' || *str == 'p')) { /* IEEE hex float */ + foundexp = 1; + exp_base = 10; + base = 2; + ex *= 4; /* We need to correct the current exponent after we change the base */ + break; } else if (base == 10 && (*str == 'E' || *str == 'e')) { foundexp = 1; break; @@ -360,9 +368,9 @@ int janet_scan_number_base( } while (str < end) { int digit = digit_lookup[*str & 0x7F]; - if (*str > 127 || digit >= base) goto error; + if (*str > 127 || digit >= exp_base) goto error; if (ee < (INT32_MAX / 40)) { - ee = base * ee + digit; + ee = exp_base * ee + digit; } str++; seenadigit = 1; diff --git a/test/suite-parse.janet b/test/suite-parse.janet index 05b34292..08e8a993 100644 --- a/test/suite-parse.janet +++ b/test/suite-parse.janet @@ -208,5 +208,8 @@ (parser/consume p `")`) (assert (= (parser/produce p) ["hello"])) +# Hex floats +(assert (= math/pi +0x1.921fb54442d18p+0001)) + (end-suite) From eecffe01a56dce9ae9e59a222f371f0faadc90cd Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Sat, 1 Feb 2025 08:09:57 -0600 Subject: [PATCH 38/47] Add some more test cases for hex floats. --- src/core/strtod.c | 3 +-- test/helper.janet | 2 +- test/suite-parse.janet | 6 ++++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/core/strtod.c b/src/core/strtod.c index 17fbc450..33857027 100644 --- a/src/core/strtod.c +++ b/src/core/strtod.c @@ -298,11 +298,10 @@ int janet_scan_number_base( } /* If still base is 0, set to default (10) */ - int exp_base = base; if (base == 0) { base = 10; - exp_base = 10; } + int exp_base = base; /* Skip leading zeros */ while (str < end && (*str == '0' || *str == '.')) { diff --git a/test/helper.janet b/test/helper.janet index 288638a9..4a6c8596 100644 --- a/test/helper.janet +++ b/test/helper.janet @@ -39,7 +39,7 @@ (defmacro assert [x &opt e] (def xx (gensym)) - (default e ~',x) + (default e (string/format "%j" x)) ~(do (def ,xx ,x) (,assert-no-tail ,xx ,e) diff --git a/test/suite-parse.janet b/test/suite-parse.janet index 08e8a993..3cf8b595 100644 --- a/test/suite-parse.janet +++ b/test/suite-parse.janet @@ -210,6 +210,12 @@ # Hex floats (assert (= math/pi +0x1.921fb54442d18p+0001)) +(assert (= math/int-max +0x1.ffff_ffff_ffff_ffp+0052)) +(assert (= math/int-min -0x1.ffff_ffff_ffff_ffp+0052)) +(assert (= 1 0x1P0)) +(assert (= 2 0x1P1)) +(assert (= -2 -0x1p1)) +(assert (= -0.5 -0x1p-1)) (end-suite) From 1b278fc657e1ecf258c2ed8778e8529a3861fa7e Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Sat, 1 Feb 2025 17:31:55 -0600 Subject: [PATCH 39/47] Update reduce2 documentation to fix issue #1545 --- CHANGELOG.md | 1 + src/boot/boot.janet | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a83876c8..395b3188 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ All notable changes to this project will be documented in this file. ## ??? - Unreleased +- Add IEEE hex floats to grammar. - Add `struct/rawget` - Fix `deep=` and `deep-not=` to better handle degenerate cases with mutable table keys - Long strings will now dedent on `\r\n` instead of just `\n`. diff --git a/src/boot/boot.janet b/src/boot/boot.janet index 16e7f48a..2198e74b 100644 --- a/src/boot/boot.janet +++ b/src/boot/boot.janet @@ -996,7 +996,7 @@ (defn reduce2 ``The 2-argument version of `reduce` that does not take an initialization value. - Instead, the first element of the array is used for initialization.`` + Instead, the first element of the array is used for initialization. If `ind` is empty, will evaluate to nil.`` [f ind] (var k (next ind)) (if (= nil k) (break nil)) From d30fd2757555ea38c1ae5693be1766ef5bb23919 Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Sun, 2 Feb 2025 20:34:36 -0600 Subject: [PATCH 40/47] Address #1554 - capture first 0-width match in repeating rules and optionals. This fixes some undesirable interplay between lookaheads and looping (repeating) rules where lookaheads caused the loop to immediately terminate _and_ discard any captures. This change adds an exception if there is a 0-width match at the first iteration of the loop, in which case captures will be kept, although the loop will still be terminated (any further iteration of the loop would possibly cause an infinite loop). This allows optionals to work as expected when combined with lookahead. There is also a question of whether or not optional should be implemented as separate rule rather than as `(between 0 1 subrule)`. --- src/core/peg.c | 2 +- test/suite-peg.janet | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/core/peg.c b/src/core/peg.c index a1e4f0d0..59de5bca 100644 --- a/src/core/peg.c +++ b/src/core/peg.c @@ -343,7 +343,7 @@ tail: CapState cs2 = cap_save(s); next_text = peg_rule(s, rule_a, text); if (!next_text || next_text == text) { - cap_load(s, cs2); + if (!next_text || captured > 0) cap_load(s, cs2); break; } captured++; diff --git a/test/suite-peg.janet b/test/suite-peg.janet index 2472374b..faf8a533 100644 --- a/test/suite-peg.janet +++ b/test/suite-peg.janet @@ -789,5 +789,13 @@ "abc123" @["abc123"]) +# Issue 1554 +(test "issue 1554 case 1" '(any (> '1)) "abc" @["a"]) +(test "issue 1554 case 2" '(any (? (> '1))) "abc" @["a"]) +(test "issue 1554 case 3" '(any (> (? '1))) "abc" @["a"]) +(test "issue 1554 case 4" '(* "a" (> '1)) "abc" @["b"]) +(test "issue 1554 case 5" '(* "a" (? (> '1))) "abc" @["b"]) +(test "issue 1554 case 6" '(* "a" (> (? '1))) "abc" @["b"]) + (end-suite) From 6da44bdb6a9504950caa2fc0a0d15d5edae33e6b Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Mon, 3 Feb 2025 07:36:00 -0600 Subject: [PATCH 41/47] Get rid of early termination rule in all finite loops. --- src/core/peg.c | 4 ++-- test/suite-peg.janet | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/core/peg.c b/src/core/peg.c index 59de5bca..bb30dd0e 100644 --- a/src/core/peg.c +++ b/src/core/peg.c @@ -342,8 +342,8 @@ tail: while (captured < hi) { CapState cs2 = cap_save(s); next_text = peg_rule(s, rule_a, text); - if (!next_text || next_text == text) { - if (!next_text || captured > 0) cap_load(s, cs2); + if (!next_text || ((next_text == text) && (hi == UINT32_MAX))) { + cap_load(s, cs2); break; } captured++; diff --git a/test/suite-peg.janet b/test/suite-peg.janet index faf8a533..6ef67b19 100644 --- a/test/suite-peg.janet +++ b/test/suite-peg.janet @@ -789,13 +789,16 @@ "abc123" @["abc123"]) -# Issue 1554 -(test "issue 1554 case 1" '(any (> '1)) "abc" @["a"]) -(test "issue 1554 case 2" '(any (? (> '1))) "abc" @["a"]) -(test "issue 1554 case 3" '(any (> (? '1))) "abc" @["a"]) +# Issue 1554 - 0-width match termination behavior +(test "issue 1554 case 1" '(any (> '1)) "abc" @[]) +(test "issue 1554 case 2" '(any (? (> '1))) "abc" @[]) +(test "issue 1554 case 3" '(any (> (? '1))) "abc" @[]) (test "issue 1554 case 4" '(* "a" (> '1)) "abc" @["b"]) (test "issue 1554 case 5" '(* "a" (? (> '1))) "abc" @["b"]) (test "issue 1554 case 6" '(* "a" (> (? '1))) "abc" @["b"]) +(test "issue 1554 case 7" '(between 0 2 (> '1)) "abc" @["a" "a"]) +(test "issue 1554 case 8" '(between 2 3 (? (> '1))) "abc" @["a" "a" "a"]) +(test "issue 1554 case 9" '(between 0 0 (> (? '1))) "abc" @[]) (end-suite) From 1b6cc023a5904ef2e1c6d21ba94dae49d2d722de Mon Sep 17 00:00:00 2001 From: sogaiu <983021772@users.noreply.github.com> Date: Thu, 6 Feb 2025 22:35:59 +0900 Subject: [PATCH 42/47] Tweak apply and short-fn docstrings --- src/boot/boot.janet | 16 +++++----------- src/core/corelib.c | 11 +++++------ 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/boot/boot.janet b/src/boot/boot.janet index 2198e74b..44a72de2 100644 --- a/src/boot/boot.janet +++ b/src/boot/boot.janet @@ -2345,17 +2345,11 @@ (defmacro short-fn ``` - Shorthand for `fn`. Arguments are given as `$n`, where `n` is the 0-indexed - argument of the function. `$` is also an alias for the first (index 0) argument. - The `$&` symbol will make the anonymous function variadic if it appears in the - body of the function, and can be combined with positional arguments. - - Example usage: - - (short-fn (+ $ $)) # A function that doubles its arguments. - (short-fn (string $0 $1)) # accepting multiple args. - |(+ $ $) # use pipe reader macro for terse function literals. - |(+ $&) # variadic functions + Shorthand for `fn`. Arguments are given as `$n`, where `n` is the + 0-indexed argument of the function. `$` is also an alias for the + first (index 0) argument. The `$&` symbol will make the anonymous + function variadic if it appears in the body of the function, and + can be combined with positional arguments. ``` [arg &opt name] (var max-param-seen -1) diff --git a/src/core/corelib.c b/src/core/corelib.c index e6715962..cf1d9ab5 100644 --- a/src/core/corelib.c +++ b/src/core/corelib.c @@ -1002,12 +1002,11 @@ static void make_apply(JanetTable *env) { janet_quick_asm(env, JANET_FUN_APPLY | JANET_FUNCDEF_FLAG_VARARG, "apply", 1, 1, INT32_MAX, 6, apply_asm, sizeof(apply_asm), JDOC("(apply f & args)\n\n" - "Applies a function to a variable number of arguments. Each element in args " - "is used as an argument to f, except the last element in args, which is expected to " - "be an array-like. Each element in this last argument is then also pushed as an argument to " - "f. For example:\n\n" - "\t(apply + 1000 (range 10))\n\n" - "sums the first 10 integers and 1000.")); + "Applies a function f to a variable number of arguments. Each " + "element in args is used as an argument to f, except the last " + "element in args, which is expected to be an array or a tuple. " + "Each element in this last argument is then also pushed as an " + "argument to f.")); } static const uint32_t error_asm[] = { From 9a892363a3aa87ad6683b4fcc8ea7dca10f37a18 Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Fri, 7 Feb 2025 21:04:17 -0600 Subject: [PATCH 43/47] Address issue #1558 Don't attempt SO_REUSEPORT on unix domain sockets since it doesn't make sense and may cause a failure to bind without extra effort by the programmer. --- src/boot/boot.janet | 2 +- src/core/net.c | 12 ++++++++---- test/suite-ev.janet | 6 ++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/boot/boot.janet b/src/boot/boot.janet index 44a72de2..fa417a07 100644 --- a/src/boot/boot.janet +++ b/src/boot/boot.janet @@ -2283,7 +2283,7 @@ (def rawget (if (= tx :struct) struct/rawget table/rawget)) (var ret false) (eachp [k v] x - (if (deep-not= (rawget y k) v) (break (set ret true)))) + (if (deep-not= (rawget y k) v) (break (set ret true)))) ret)) (= tx :buffer) (not= 0 (- (length x) (length y)) (memcmp x y)) (not= x y)))) diff --git a/src/core/net.c b/src/core/net.c index f38a20f6..799976fa 100644 --- a/src/core/net.c +++ b/src/core/net.c @@ -578,17 +578,21 @@ JANET_CORE_FN(cfun_net_connect, net_sched_connect(stream); } -static const char *serverify_socket(JSock sfd, int reuse) { +static const char *serverify_socket(JSock sfd, int reuse_addr, int reuse_port) { /* Set various socket options */ int enable = 1; - if (reuse) { + if (reuse_addr) { if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (char *) &enable, sizeof(int)) < 0) { return "setsockopt(SO_REUSEADDR) failed"; } + } + if (reuse_port) { #ifdef SO_REUSEPORT if (setsockopt(sfd, SOL_SOCKET, SO_REUSEPORT, &enable, sizeof(int)) < 0) { return "setsockopt(SO_REUSEPORT) failed"; } +#else + (void) reuse_port; #endif } janet_net_socknoblock(sfd); @@ -668,7 +672,7 @@ JANET_CORE_FN(cfun_net_listen, janet_free(ai); janet_panicf("could not create socket: %V", janet_ev_lasterr()); } - const char *err = serverify_socket(sfd, reuse); + const char *err = serverify_socket(sfd, reuse, 0); if (NULL != err || bind(sfd, (struct sockaddr *)ai, sizeof(struct sockaddr_un))) { JSOCKCLOSE(sfd); janet_free(ai); @@ -691,7 +695,7 @@ JANET_CORE_FN(cfun_net_listen, sfd = socket(rp->ai_family, rp->ai_socktype | JSOCKFLAGS, rp->ai_protocol); #endif if (!JSOCKVALID(sfd)) continue; - const char *err = serverify_socket(sfd, reuse); + const char *err = serverify_socket(sfd, reuse, reuse); if (NULL != err) { JSOCKCLOSE(sfd); continue; diff --git a/test/suite-ev.janet b/test/suite-ev.janet index 0ed19413..2e6f8833 100644 --- a/test/suite-ev.janet +++ b/test/suite-ev.janet @@ -484,4 +484,10 @@ (pp :foo))) (ev/chan-close c) +# soreuseport on unix domain sockets +(compwhen (= :linux (os/which)) + (assert-no-error "unix-domain socket reuseaddr" + (let [s (net/listen :unix "./unix-domain-socket" :stream)] + (:close s)))) + (end-suite) From 5c56c7fa915112fc81dc66331a7aac9bf7ae801a Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Fri, 7 Feb 2025 21:05:28 -0600 Subject: [PATCH 44/47] Formatting in peg.c --- src/core/peg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/peg.c b/src/core/peg.c index c8cd2b8c..454cc802 100644 --- a/src/core/peg.c +++ b/src/core/peg.c @@ -563,7 +563,7 @@ tail: up1(s); if (!terminus_end) { - return NULL; + return NULL; } const uint8_t *saved_end = s->text_end; From dd609bb1bba26e6eea2a636e5455231f7dfd2e1c Mon Sep 17 00:00:00 2001 From: sogaiu <983021772@users.noreply.github.com> Date: Fri, 7 Feb 2025 14:29:54 +0900 Subject: [PATCH 45/47] Update CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 395b3188..11eebf84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,17 @@ All notable changes to this project will be documented in this file. ## ??? - Unreleased +- Improve `?` peg special termination behavior - Add IEEE hex floats to grammar. +- Add buffer peg literal support +- Improve `split` peg special edge case behavior +- Add Arm64 .msi support +- Add `no-reuse` argument to `net/listen` to disable reusing server sockets - Add `struct/rawget` - Fix `deep=` and `deep-not=` to better handle degenerate cases with mutable table keys - Long strings will now dedent on `\r\n` instead of just `\n`. - Add `ev/to-file` for synchronous resource operations +- Improve `file/open` error message by including path ## 1.37.1 - 2024-12-05 - Fix meson cross compilation From 2e6001316a9873935c80bb1157168a07e02df25a Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Sat, 15 Feb 2025 23:22:12 -0600 Subject: [PATCH 46/47] Work on bug #1556 --- src/boot/boot.janet | 4 ++-- src/core/ev.c | 18 ++++++++++++++++- src/core/net.c | 5 ++++- test/suite-ev.janet | 48 +++++++++++++++++++++++++++++++++++---------- 4 files changed, 61 insertions(+), 14 deletions(-) diff --git a/src/boot/boot.janet b/src/boot/boot.janet index fa417a07..0d8a5f34 100644 --- a/src/boot/boot.janet +++ b/src/boot/boot.janet @@ -3877,8 +3877,8 @@ (compwhen (dyn 'net/listen) (defn net/server "Start a server asynchronously with `net/listen` and `net/accept-loop`. Returns the new server stream." - [host port &opt handler type] - (def s (net/listen host port type)) + [host port &opt handler type no-reuse] + (def s (net/listen host port type no-reuse)) (if handler (ev/go (fn [] (net/accept-loop s handler)))) s)) diff --git a/src/core/ev.c b/src/core/ev.c index 0303a36e..4cde9ff8 100644 --- a/src/core/ev.c +++ b/src/core/ev.c @@ -1,5 +1,5 @@ /* -* Copyright (c) 2024 Calvin Rose +* Copyright (c) 2025 Calvin Rose * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -1789,6 +1789,22 @@ void janet_stream_edge_triggered(JanetStream *stream) { } void janet_stream_level_triggered(JanetStream *stream) { + /* On macos, we seem to need to delete any registered events before re-registering without + * EV_CLEAR, otherwise the new event will still have EV_CLEAR set erroneously. This could be a + * kernel bug, but unfortunately the specification is vague here, esp. in regards to where and when + * EV_CLEAR is set automatically. */ + struct kevent kevs[2]; + int length = 0; + if (stream->flags & (JANET_STREAM_READABLE | JANET_STREAM_ACCEPTABLE)) { + EV_SETx(&kevs[length++], stream->handle, EVFILT_READ, EV_DELETE, 0, 0, stream); + } + if (stream->flags & JANET_STREAM_WRITABLE) { + EV_SETx(&kevs[length++], stream->handle, EVFILT_WRITE, EV_DELETE, 0, 0, stream); + } + int status; + do { + status = kevent(janet_vm.kq, kevs, length, NULL, 0, NULL); + } while (status == -1 && errno == EINTR); janet_register_stream_impl(stream, 0); } diff --git a/src/core/net.c b/src/core/net.c index 799976fa..be338b5f 100644 --- a/src/core/net.c +++ b/src/core/net.c @@ -554,7 +554,10 @@ JANET_CORE_FN(cfun_net_connect, int err = WSAGetLastError(); freeaddrinfo(ai); #else - int status = connect(sock, addr, addrlen); + int status; + do { + status = connect(sock, addr, addrlen); + } while (status == -1 && errno == EINTR); int err = errno; if (is_unix) { janet_free(ai); diff --git a/test/suite-ev.janet b/test/suite-ev.janet index 2e6f8833..3e3c2502 100644 --- a/test/suite-ev.janet +++ b/test/suite-ev.janet @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Calvin Rose & contributors +# Copyright (c) 2025 Calvin Rose & contributors # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to @@ -199,7 +199,7 @@ (assert s "made server 1") (defn test-echo [msg] - (with [conn (net/connect test-host test-port)] + (with [conn (assert (net/connect test-host test-port))] (net/write conn msg) (def res (net/read conn 1024)) (assert (= (string res) msg) (string "echo " msg)))) @@ -213,6 +213,7 @@ # Test on both server and client # 504411e +(var iterations 0) (defn names-handler [stream] (defer (:close stream) @@ -220,21 +221,26 @@ (ev/read stream 1) (def [host port] (net/localname stream)) (assert (= host test-host) "localname host server") - (assert (= port (scan-number test-port)) "localname port server"))) + (assert (= port (scan-number test-port)) "localname port server") + (++ iterations) + (ev/write stream " "))) # Test localname and peername # 077bf5eba (repeat 10 (with [s (net/server test-host test-port names-handler)] (repeat 10 - (with [conn (net/connect test-host test-port)] + (with [conn (assert (net/connect test-host test-port))] (def [host port] (net/peername conn)) (assert (= host test-host) "peername host client ") (assert (= port (scan-number test-port)) "peername port client") - # let server close - (ev/write conn " ")))) + (++ iterations) + (ev/write conn " ") + (ev/read conn 1)))) (gccollect)) +(assert (= iterations 200) "localname and peername not enough checks") + # Create pipe # 12f09ad2d (var pipe-counter 0) @@ -422,7 +428,7 @@ (assert (= result text) (string/format "expected %v, got %v" text result))) # Now do our telnet chat -(def bob (net/connect test-host test-port :stream)) +(def bob (assert (net/connect test-host test-port :stream))) (expect-read bob "Whats your name?\n") (if (= :mingw (os/which)) (net/write bob "bob") @@ -432,7 +438,7 @@ (file/flush fbob) (:close fbob))) (expect-read bob "Welcome bob\n") -(def alice (net/connect test-host test-port)) +(def alice (assert (net/connect test-host test-port))) (expect-read alice "Whats your name?\n") (net/write alice "alice") (expect-read alice "Welcome alice\n") @@ -446,7 +452,7 @@ (expect-read bob "[alice]:hi\n") # Ted joins the chat server -(def ted (net/connect test-host test-port)) +(def ted (assert (net/connect test-host test-port))) (expect-read ted "Whats your name?\n") (net/write ted "ted") (expect-read ted "Welcome ted\n") @@ -485,9 +491,31 @@ (ev/chan-close c) # soreuseport on unix domain sockets -(compwhen (= :linux (os/which)) +(compwhen (or (= :macos (os/which)) (= :linux (os/which))) (assert-no-error "unix-domain socket reuseaddr" (let [s (net/listen :unix "./unix-domain-socket" :stream)] (:close s)))) +# net/accept-loop level triggering +(gccollect) +(def maxconn 50) +(var connect-count 0) +(defn level-trigger-handling + [conn &] + (with [conn conn] + (ev/write conn (ev/read conn 4096)) + (++ connect-count))) +(def s (assert (net/server test-host test-port level-trigger-handling))) +(def cons @[]) +(repeat maxconn (array/push cons (assert (net/connect test-host test-port)))) +(assert (= maxconn (length cons))) +(defn do-connect [i] + (with [c (get cons i)] + (ev/write c "abc123") + (ev/read c 4096))) +(for i 0 maxconn (ev/spawn (do-connect i))) +(ev/sleep 0.1) +(assert (= maxconn connect-count)) +(:close s) + (end-suite) From 3441bcbd69e304aefa34ff5ec272f0a6b8d3e783 Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Sun, 16 Feb 2025 10:31:34 -0600 Subject: [PATCH 47/47] Add more cases for checking coerce error. Channel operations inside a janet_call could wreak more havoc later, when scheduled fibers are shooting off when they shouldn't be. The easiest way to prevent this is simply check that we are not inside janet_call before doing channel operations. We also get nicer error messages this way. --- src/core/capi.c | 5 +++++ src/core/ev.c | 10 ++++++++++ src/core/vm.c | 5 +++++ test/suite-ev.janet | 8 ++++++++ 4 files changed, 28 insertions(+) diff --git a/src/core/capi.c b/src/core/capi.c index d4169dc0..af6ea582 100644 --- a/src/core/capi.c +++ b/src/core/capi.c @@ -64,6 +64,11 @@ void janet_signalv(JanetSignal sig, Janet message) { if (janet_vm.return_reg != NULL) { /* Should match logic in janet_call for coercing everything not ok to an error (no awaits, yields, etc.) */ if (janet_vm.coerce_error && sig != JANET_SIGNAL_OK) { +#ifdef JANET_EV + if (NULL != janet_vm.root_fiber && sig == JANET_SIGNAL_EVENT) { + janet_vm.root_fiber->sched_id++; + } +#endif if (sig != JANET_SIGNAL_ERROR) { message = janet_wrap_string(janet_formatc("%v coerced from %s to error", message, janet_signal_names[sig])); } diff --git a/src/core/ev.c b/src/core/ev.c index 4cde9ff8..e51fee80 100644 --- a/src/core/ev.c +++ b/src/core/ev.c @@ -1037,6 +1037,9 @@ JANET_CORE_FN(cfun_channel_push, "Returns the channel if the write succeeded, nil otherwise.") { janet_fixarity(argc, 2); JanetChannel *channel = janet_getchannel(argv, 0); + if (janet_vm.coerce_error) { + janet_panic("cannot give to channel inside janet_call"); + } if (janet_channel_push(channel, argv[1], 0)) { janet_await(); } @@ -1049,6 +1052,9 @@ JANET_CORE_FN(cfun_channel_pop, janet_fixarity(argc, 1); JanetChannel *channel = janet_getchannel(argv, 0); Janet item; + if (janet_vm.coerce_error) { + janet_panic("cannot take from channel inside janet_call"); + } if (janet_channel_pop(channel, &item, 0)) { janet_schedule(janet_vm.root_fiber, item); } @@ -1085,6 +1091,10 @@ JANET_CORE_FN(cfun_channel_choice, int32_t len; const Janet *data; + if (janet_vm.coerce_error) { + janet_panic("cannot select from channel inside janet_call"); + } + /* Check channels for immediate reads and writes */ for (int32_t i = 0; i < argc; i++) { if (janet_indexed_view(argv[i], &data, &len) && len == 2) { diff --git a/src/core/vm.c b/src/core/vm.c index 0e599aa3..4fe179d3 100644 --- a/src/core/vm.c +++ b/src/core/vm.c @@ -1388,6 +1388,11 @@ Janet janet_call(JanetFunction *fun, int32_t argc, const Janet *argv) { if (signal != JANET_SIGNAL_OK) { /* Should match logic in janet_signalv */ +#ifdef JANET_EV + if (janet_vm.root_fiber != NULL && signal == JANET_SIGNAL_EVENT) { + janet_vm.root_fiber->sched_id++; + } +#endif if (signal != JANET_SIGNAL_ERROR) { *janet_vm.return_reg = janet_wrap_string(janet_formatc("%v coerced from %s to error", *janet_vm.return_reg, janet_signal_names[signal])); } diff --git a/test/suite-ev.janet b/test/suite-ev.janet index 3e3c2502..c773e960 100644 --- a/test/suite-ev.janet +++ b/test/suite-ev.janet @@ -482,6 +482,14 @@ (:close chat-server) # Issue #1531 +(defn sleep-print [x] (ev/sleep 0) (print x)) +(protect (with-dyns [*out* sleep-print] (prin :foo))) +(defn level-trigger-handling [conn &] (:close conn)) +(def s (assert (net/server test-host test-port level-trigger-handling))) +(def c (assert (net/connect test-host test-port))) +(:close s) + +# Issue #1531 no. 2 (def c (ev/chan 0)) (ev/spawn (while (def x (ev/take c)))) (defn print-to-chan [x] (ev/give c x))