1.3.0 Release

Update CHANGELOG.md
Add put-in.
2025-11-09 03:53:06 +00:00 · 2019-09-05 19:33:08 -05:00 · 2019-09-05 13:21:17 -05:00 · 2019-09-05 13:19:25 -05:00 · 2019-09-05 13:11:53 -05:00 · 2019-09-05 12:28:11 -05:00
25 changed files with 1437 additions and 1032 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,23 @@
 # Changelog
 All notable changes to this project will be documented in this file.

+## 1.3.0 - 2019-09-05
+- Add `get-in`, `put-in`, `update-in`, and `freeze` to core.
+- Add `jpm run rule` and `jpm rules` to jpm to improve utility and discoverability of jpm.
+- Remove `cook` module and move `path` module to https://github.com/janet-lang/path.git.
+  The functionality in `cook` is now bundled directly in the `jpm` script.
+- Add `buffer/format` and `string/format` format flags `Q` and `q` to print colored and
+  non-colored single-line values, similar to `P` and `p`.
+- Change default repl to print long sequences on one line and color stacktraces if color is enabled.
+- Add `backmatch` pattern for PEGs.
+- jpm detects if not in a Developer Command prompt on windows for a better error message.
+- jpm install git submodules in dependencies
+- Change default fiber stack limit to the maximum value of a 32 bit signed integer.
+- Some bug fixes with `jpm`
+- Fix bugs with pegs.
+- Add `os/arch` to get ISA that janet was compiled for
+- Add color to stacktraces via `(dyn :err-color)`
+
 ## 1.2.0 - 2019-08-08
 - Add `take` and `drop` functions that are easier to use compared to the
  existing slice functions.
--- a/62
+++ b/62
@@ -135,7 +135,7 @@ build/janet_boot: $(JANET_BOOT_OBJECTS)

 # Now the reason we bootstrap in the first place
 build/core_image.c: build/janet_boot
-	build/janet_boot $@ JANET_PATH $(JANET_PATH) JANET_HEADERPATH $(INCLUDEDIR)/janet
+	build/janet_boot $@ JANET_PATH '$(JANET_PATH)' JANET_HEADERPATH '$(INCLUDEDIR)/janet'

 ##########################################################
 ##### The main interpreter program and shared object #####
@@ -253,7 +253,7 @@ dist: build/janet-dist.tar.gz

 build/janet-%.tar.gz: $(JANET_TARGET) \
 	src/include/janet.h src/conf/janetconf.h \
-	janet.1 LICENSE CONTRIBUTING.md $(JANET_LIBRARY) $(JANET_STATIC_LIBRARY) \
+	jpm.1 janet.1 LICENSE CONTRIBUTING.md $(JANET_LIBRARY) $(JANET_STATIC_LIBRARY) \
 	build/doc.html README.md build/janet.c
 	tar -czvf $@ $^

@@ -272,9 +272,8 @@ build/doc.html: $(JANET_TARGET) tools/gendoc.janet

 SONAME=libjanet.so.1

-.PHONY: $(PKG_CONFIG_PATH)/janet.pc
-$(PKG_CONFIG_PATH)/janet.pc: $(JANET_TARGET)
-	mkdir -p $(PKG_CONFIG_PATH)
+.PHONY: build/janet.pc
+build/janet.pc: $(JANET_TARGET)
 	echo 'prefix=$(PREFIX)' > $@
 	echo 'exec_prefix=$${prefix}' >> $@
 	echo 'includedir=$(INCLUDEDIR)/janet' >> $@
@@ -288,31 +287,34 @@ $(PKG_CONFIG_PATH)/janet.pc: $(JANET_TARGET)
 	echo 'Libs: -L$${libdir} -ljanet $(LDFLAGS)' >> $@
 	echo 'Libs.private: $(CLIBS)' >> $@

-install: $(JANET_TARGET) $(PKG_CONFIG_PATH)/janet.pc
-	mkdir -p $(BINDIR)
-	cp $(JANET_TARGET) $(BINDIR)/janet
-	mkdir -p $(INCLUDEDIR)/janet
-	cp -rf $(JANET_HEADERS) $(INCLUDEDIR)/janet
-	mkdir -p $(JANET_PATH)
-	mkdir -p $(LIBDIR)
-	cp $(JANET_LIBRARY) $(LIBDIR)/libjanet.so.$(shell $(JANET_TARGET) -e '(print janet/version)')
-	cp $(JANET_STATIC_LIBRARY) $(LIBDIR)/libjanet.a
-	ln -sf $(SONAME) $(LIBDIR)/libjanet.so
+install: $(JANET_TARGET) build/janet.pc
+	mkdir -p '$(BINDIR)'
+	cp $(JANET_TARGET) '$(BINDIR)/janet'
+	mkdir -p '$(INCLUDEDIR)/janet'
+	cp -rf $(JANET_HEADERS) '$(INCLUDEDIR)/janet'
+	mkdir -p '$(JANET_PATH)'
+	mkdir -p '$(LIBDIR)'
+	cp $(JANET_LIBRARY) '$(LIBDIR)/libjanet.so.$(shell $(JANET_TARGET) -e '(print janet/version)')'
+	cp $(JANET_STATIC_LIBRARY) '$(LIBDIR)/libjanet.a'
+	ln -sf $(SONAME) '$(LIBDIR)/libjanet.so'
 	ln -sf libjanet.so.$(shell $(JANET_TARGET) -e '(print janet/version)') $(LIBDIR)/$(SONAME)
-	cp -rf auxlib/* $(JANET_PATH)
-	cp -rf auxbin/* $(BINDIR)
-	mkdir -p $(MANPATH)
-	cp janet.1 $(MANPATH)
+	cp -rf auxbin/* '$(BINDIR)'
+	mkdir -p '$(MANPATH)'
+	cp janet.1 '$(MANPATH)'
+	cp jpm.1 '$(MANPATH)'
+	mkdir -p '$(PKG_CONFIG_PATH)'
+	cp build/janet.pc '$(PKG_CONFIG_PATH)/janet.pc'
 	-ldconfig $(LIBDIR)

 uninstall:
-	-rm $(BINDIR)/janet
-	-rm $(BINDIR)/jpm
-	-rm -rf $(INCLUDEDIR)/janet
-	-rm -rf $(LIBDIR)/libjanet.*
-	-rm $(PKG_CONFIG_PATH)/janet.pc
-	-rm $(MANPATH)/janet.1
-	# -rm -rf $(JANET_PATH)/* - err on the side of correctness here
+	-rm '$(BINDIR)/janet'
+	-rm '$(BINDIR)/jpm'
+	-rm -rf '$(INCLUDEDIR)/janet'
+	-rm -rf '$(LIBDIR)'/libjanet.*
+	-rm '$(PKG_CONFIG_PATH)/janet.pc'
+	-rm '$(MANPATH)/janet.1'
+	-rm '$(MANPATH)/jpm.1'
+	# -rm -rf '$(JANET_PATH)'/* - err on the side of correctness here

 #################
 ##### Other #####
@@ -329,8 +331,12 @@ clean:
 	-rm -rf build vgcore.* callgrind.*

 test-install:
-	cd test/install && rm -rf build .cache .manifests && jpm --verbose build && jpm --verbose test \
-		&& build/testexec
+	cd test/install \
+		&& rm -rf build .cache .manifests \
+		&& jpm --verbose build \
+		&& jpm --verbose test \
+		&& build/testexec \
+		&& jpm --verbose --modpath=. install https://github.com/janet-lang/json.git

 build/embed_janet.o: build/janet.c $(JANET_HEADERS)
 	$(CC) $(CFLAGS) -c $< -o $@
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -15,9 +15,10 @@ matrix:

 # skip unsupported combinations
 init:
- call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars32.bat"
+    - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars32.bat"

 install:
+    - set JANET_BUILD=%appveyor_repo_commit:~0,7%
    - choco install nsis -y -pre
    # Replace makensis.exe and files with special long string build. This should
    # prevent issues when setting PATH during installation.
@@ -27,6 +28,8 @@ install:
    # We need to reload vcvars after refreshing
    - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars32.bat"
    - build_win test-install
+    - set janet_outname=%appveyor_repo_tag_name%
+    - if "%janet_outname%"=="" set janet_outname=v1.3.0
 build: off

 only_commits:
@@ -35,8 +38,20 @@ only_commits:
    - src/

 artifacts:
-    - path: janet-v1.2.0-windows-installer.exe
-      name: janet-v1.2.0-windows-installer.exe
+    - name: janet.c
+      path: dist\janet.c
+      type: File
+    - name: janet.h
+      path: dist\janet.h
+      type: File
+    - name: janetconf.h
+      path: dist\janetconf.h
+      type: File
+    - name: "janet-$(janet_outname)-windows"
+      path: dist
+      type: Zip
+    - path: "janet-$(janet_outname)-windows-installer.exe"
+      name: "janet-$(janet_outname)-windows-installer.exe"
      type: File

 deploy:
@@ -44,7 +59,7 @@ deploy:
  provider: GitHub
  auth_token:
    secure: lwEXy09qhj2jSH9s1C/KvCkAUqJSma8phFR+0kbsfUc3rVxpNK5uD3z9Md0SjYRx
-  artifact: janet-windows
+  artifact: /janet.*/
  draft: true
  on:
      APPVEYOR_REPO_TAG: true
--- a/auxbin/jpm
+++ b/auxbin/jpm
@@ -1,8 +1,794 @@
 #!/usr/bin/env janet

-# CLI tool for building janet projects. Wraps cook.
+# CLI tool for building janet projects.

-(import cook)
+#
+# Basic Path Settings
+#
+
+# Windows is the OS outlier
+(def- is-win (= (os/which) :windows))
+(def- is-mac (= (os/which) :macos))
+(def- sep (if is-win "\\" "/"))
+(def- objext (if is-win ".obj" ".o"))
+(def- modext (if is-win ".dll" ".so"))
+(def- statext (if is-win ".static.lib" ".a"))
+(def- absprefix (if is-win "C:\\" "/"))
+
+#
+# Rule Engine
+#
+
+(defn- getrules []
+  (if-let [rules (dyn :rules)] rules (setdyn :rules @{})))
+
+(defn- gettarget [target]
+  (def item ((getrules) target))
+  (unless item (error (string "No rule for target " target)))
+  item)
+
+(defn- rule-impl
+  [target deps thunk &opt phony]
+  (put (getrules) target @[(array/slice deps) thunk phony]))
+
+(defmacro rule
+  "Add a rule to the rule graph."
+  [target deps & body]
+  ~(,rule-impl ,target ,deps (fn [] nil ,;body)))
+
+(defmacro phony
+  "Add a phony rule to the rule graph. A phony rule will run every time
+  (it is always considered out of date). Phony rules are good for defining
+  user facing tasks."
+  [target deps & body]
+  ~(,rule-impl ,target ,deps (fn [] nil ,;body) true))
+
+(defn add-dep
+  "Add a dependency to an existing rule. Useful for extending phony
+  rules or extending the dependency graph of existing rules."
+  [target dep]
+  (def [deps] (gettarget target))
+  (array/push deps dep))
+
+(defn- add-thunk
+  [target more]
+  (def item (gettarget target))
+  (def [_ thunk] item)
+  (put item 1 (fn [] (more) (thunk))))
+
+(defmacro add-body
+  "Add recipe code to an existing rule. This makes existing rules do more but
+  does not modify the dependency graph."
+  [target & body]
+  ~(,add-thunk ,target (fn [] ,;body)))
+
+(defn- needs-build
+  [dest src]
+  (let [mod-dest (os/stat dest :modified)
+        mod-src (os/stat src :modified)]
+    (< mod-dest mod-src)))
+
+(defn- needs-build-some
+  [dest sources]
+  (def f (file/open dest))
+  (if (not f) (break true))
+  (file/close f)
+  (some (partial needs-build dest) sources))
+
+(defn do-rule
+  "Evaluate a given rule."
+  [target]
+  (def item ((getrules) target))
+  (unless item
+    (if (os/stat target :mode)
+      (break target)
+      (error (string "No rule for file " target " found."))))
+  (def [deps thunk phony] item)
+  (def realdeps (seq [dep :in deps :let [x (do-rule dep)] :when x] x))
+  (when (or phony (needs-build-some target realdeps))
+    (thunk))
+  (unless phony target))
+
+#
+# Configuration
+#
+
+(def JANET_MODPATH (or (os/getenv "JANET_MODPATH") (dyn :syspath)))
+(def JANET_HEADERPATH (or (os/getenv "JANET_HEADERPATH")
+                          (if-let [j (dyn :syspath)]
+                            (string j "/../../include/janet"))))
+(def JANET_BINPATH (or (os/getenv "JANET_BINPATH")
+                       (if-let [j (dyn :syspath)]
+                         (string j "/../../bin"))))
+(def JANET_LIBPATH (or (os/getenv "JANET_LIBPATH")
+                       (if-let [j (dyn :syspath)]
+                         (string j "/.."))))
+
+#
+# Compilation Defaults
+#
+
+(def default-compiler (if is-win "cl" "cc"))
+(def default-linker (if is-win "link" "cc"))
+(def default-archiver (if is-win "lib" "ar"))
+
+# Default flags for natives, but not required
+(def default-lflags (if is-win ["/nologo"] []))
+(def default-cflags
+  (if is-win
+    ["/nologo"]
+    ["-std=c99" "-Wall" "-Wextra"]))
+
+# Required flags for dynamic libraries. These
+# are used no matter what for dynamic libraries.
+(def- dynamic-cflags
+  (if is-win
+    []
+    ["-fpic"]))
+(def- dynamic-lflags
+    (if is-win
+      ["/DLL"]
+      (if is-mac
+        ["-shared" "-undefined" "dynamic_lookup"]
+        ["-shared"])))
+
+(defn- opt
+  "Get an option, allowing overrides via dynamic bindings AND some
+  default value dflt if no dynamic binding is set."
+  [opts key dflt]
+  (def ret (or (opts key) (dyn key dflt)))
+  (if (= nil ret)
+    (error (string "option :" key " not set")))
+  ret)
+
+(defn check-cc
+  "Ensure we have a c compiler"
+  []
+  (if is-win
+    (do
+      (if (os/getenv "INCLUDE") (break))
+      (error "Run jpm inside a Developer Command Prompt.
+ jpm needs a c compiler to compile natives. You can install the MSVC compiler at "))
+    (do)))
+
+#
+# Importing a file
+#
+
+(def- _env (fiber/getenv (fiber/current)))
+
+(defn- proto-flatten
+  [into x]
+  (when x
+    (proto-flatten into (table/getproto x))
+    (loop [k :keys x]
+      (put into k (x k))))
+  into)
+
+(defn import-rules
+  "Import another file that defines more rules. This ruleset
+  is merged into the current ruleset."
+  [path]
+  (def env (make-env))
+  (unless (os/stat path :mode)
+    (error (string "cannot open " path)))
+  (loop [k :keys _env :when (symbol? k)]
+     (unless ((_env k) :private) (put env k (_env k))))
+  (def currenv (proto-flatten @{} (fiber/getenv (fiber/current))))
+  (loop [k :keys currenv :when (keyword? k)]
+    (put env k (currenv k)))
+  (dofile path :env env :exit true)
+  (when-let [rules (env :rules)] (merge-into (getrules) rules)))
+
+#
+# OS and shell helpers
+#
+
+(def- filepath-replacer
+  "Convert url with potential bad characters into a file path element."
+  (peg/compile ~(% (any (+ (/ '(set "<>:\"/\\|?*") "_") '1)))))
+
+(defn filepath-replace
+  "Remove special characters from a string or path
+  to make it into a path segment."
+  [repo]
+  (get (peg/match filepath-replacer repo) 0))
+
+(defn shell
+  "Do a shell command"
+  [& args]
+  (if (dyn :verbose)
+    (print ;(interpose " " args)))
+  (def res (os/execute args :p))
+  (unless (zero? res)
+    (error (string "command exited with status " res))))
+
+(defn- shell2
+  "Do a shell command, but don't assum 0 is the (only) passing exit code."
+  [pred & args]
+  (if (dyn :verbose)
+    (print ;(interpose " " args)))
+  (def res (os/execute args :p))
+  (unless (pred res)
+    (error (string "command exited with status " res))))
+
+(defn rm
+  "Remove a directory and all sub directories."
+  [path]
+  (if (= (os/stat path :mode) :directory)
+    (do
+      (each subpath (os/dir path)
+        (rm (string path sep subpath)))
+      (os/rmdir path))
+    (os/rm path)))
+
+(defn copy
+  "Copy a file or directory recursively from one location to another."
+  [src dest]
+  (print "copying " src " to " dest "...")
+  (if is-win
+    (if (= (os/stat src :mode) :directory)
+      (let [dname (last (filter |(pos? (length $)) (string/split "/" src)))]
+        (shell2 |(< $ 9) "robocopy" "/MIR" "/COPY:DAT" "/NS" "/NS" "/NFL" "/NDL" "/NP" "/NJH" "/NJS"
+                src (string dest sep dname)))
+      (shell "xcopy" src dest "/y" "/e"))
+    (shell "cp" "-rf" src dest)))
+
+#
+# C Compilation
+#
+
+(defn- embed-name
+  "Rename a janet symbol for embedding."
+  [path]
+  (->> path
+       (string/replace-all sep "___")
+       (string/replace-all ".janet" "")))
+
+(defn- out-path
+  "Take a source file path and convert it to an output path."
+  [path from-ext to-ext]
+  (->> path
+       (string/replace-all sep "___")
+       (string/replace-all from-ext to-ext)
+       (string "build" sep)))
+
+(defn- make-define
+  "Generate strings for adding custom defines to the compiler."
+  [define value]
+  (if value
+    (string (if is-win "/D" "-D") define "=" value)
+    (string (if is-win "/D" "-D") define)))
+
+(defn- make-defines
+  "Generate many defines. Takes a dictionary of defines. If a value is
+  true, generates -DNAME (/DNAME on windows), otherwise -DNAME=value."
+  [defines]
+  (seq [[d v] :pairs defines] (make-define d (if (not= v true) v))))
+
+(defn- getcflags
+  "Generate the c flags from the input options."
+  [opts]
+  @[;(opt opts :cflags default-cflags)
+    (string (if is-win "/I" "-I") (dyn :headerpath JANET_HEADERPATH))
+    (string (if is-win "/O" "-O") (opt opts :optimize 2))])
+
+(defn- entry-name
+  "Name of symbol that enters static compilation of a module."
+  [name]
+  (string "janet_module_entry_" (filepath-replace name)))
+
+(defn- compile-c
+  "Compile a C file into an object file."
+  [opts src dest &opt static?]
+  (def cc (opt opts :compiler default-compiler))
+  (def cflags [;(getcflags opts) ;(if static? [] dynamic-cflags)])
+  (def entry-defines (if-let [n (opts :entry-name)]
+                       [(make-define "JANET_ENTRY_NAME" n)]
+                       []))
+  (def defines [;(make-defines (opt opts :defines {})) ;entry-defines])
+  (def headers (or (opts :headers) []))
+  (rule dest [src ;headers]
+        (check-cc)
+        (print "compiling " dest "...")
+        (if is-win
+          (shell cc ;defines "/c" ;cflags (string "/Fo" dest) src)
+          (shell cc "-c" src ;defines ;cflags "-o" dest))))
+
+(defn- libjanet
+  "Find libjanet.a (or libjanet.lib on windows) at compile time"
+  []
+  (def libpath (dyn :libpath JANET_LIBPATH))
+  (unless libpath
+    (error "cannot find libpath: provide --libpath or JANET_LIBPATH"))
+  (string (dyn :libpath JANET_LIBPATH)
+          sep
+          (if is-win "libjanet.lib" "libjanet.a")))
+
+(defn- win-import-library
+  "On windows, an import library is needed to link to a dll statically."
+  []
+  (def hpath (dyn :headerpath JANET_HEADERPATH))
+  (unless hpath
+    (error "cannot find headerpath: provide --headerpath or JANET_HEADERPATH"))
+  (string hpath `\\janet.lib`))
+
+(defn- link-c
+  "Link object files together to make a native module."
+  [opts target & objects]
+  (def ld (opt opts :linker default-linker))
+  (def cflags (getcflags opts))
+  (def lflags [;(opt opts :lflags default-lflags)
+               ;(if (opts :static) [] dynamic-lflags)])
+  (rule target objects
+        (check-cc)
+        (print "linking " target "...")
+        (if is-win
+          (shell ld ;lflags (string "/OUT:" target) ;objects (win-import-library))
+          (shell ld ;cflags `-o` target ;objects ;lflags))))
+
+(defn- archive-c
+  "Link object files together to make a static library."
+  [opts target & objects]
+  (def ar (opt opts :archiver default-archiver))
+  (rule target objects
+        (check-cc)
+        (print "creating static library " target "...")
+        (if is-win
+          (shell ar "/nologo" (string "/out:" target) ;objects)
+          (shell ar "rcs" target ;objects))))
+
+(defn- create-buffer-c-impl
+  [bytes dest name]
+  (def out (file/open dest :w))
+  (def chunks (seq [b :in bytes] (string b)))
+  (file/write out
+              "#include <janet.h>\n"
+              "static const unsigned char bytes[] = {"
+              (string/join (interpose ", " chunks))
+              "};\n\n"
+              "const unsigned char *" name "_embed = bytes;\n"
+              "size_t " name "_embed_size = sizeof(bytes);\n")
+  (file/close out))
+
+(defn- create-buffer-c
+  "Inline raw byte file as a c file."
+  [source dest name]
+  (rule dest [source]
+        (print "generating " dest "...")
+        (with [f (file/open source :r)]
+          (create-buffer-c-impl (:read f :all) dest name))))
+
+(def- root-env (table/getproto (fiber/getenv (fiber/current))))
+
+(defn- modpath-to-meta
+  "Get the meta file path (.meta.janet) corresponding to a native module path (.so)."
+  [path]
+  (string (string/slice path 0 (- (length modext))) "meta.janet"))
+
+(defn- modpath-to-static
+  "Get the static library (.a) path corresponding to a native module path (.so)."
+  [path]
+  (string (string/slice path 0 (- -1 (length modext))) statext))
+
+(defn- create-executable
+  "Links an image with libjanet.a (or .lib) to produce an
+  executable. Also will try to link native modules into the
+  final executable as well."
+  [opts source dest]
+
+  # Create executable's janet image
+  (def cimage_dest (string dest ".c"))
+  (rule dest [source]
+        (check-cc)
+        (print "generating executable c source...")
+        # Load entry environment and get main function.
+        (def entry-env (dofile source))
+        (def main ((entry-env 'main) :value))
+
+        # Create marshalling dictionary
+        (def mdict (invert (env-lookup root-env)))
+        # Load all native modules
+        (def prefixes @{})
+        (def static-libs @[])
+        (loop [[name m] :pairs module/cache
+               :let [n (m :native)]
+               :when n
+               :let [prefix (gensym)]]
+          (print "found native " n "...")
+          (put prefixes prefix n)
+          (array/push static-libs (modpath-to-static n))
+          (def oldproto (table/getproto m))
+          (table/setproto m nil)
+          (loop [[sym value] :pairs (env-lookup m)]
+            (put mdict value (symbol prefix sym)))
+          (table/setproto m oldproto))
+
+        # Find static modules
+        (def declarations @"")
+        (def lookup-into-invocations @"")
+        (loop [[prefix name] :pairs prefixes]
+          (def meta (eval-string (slurp (modpath-to-meta name))))
+          (buffer/push-string lookup-into-invocations
+                              "    temptab = janet_table(0);\n"
+                              "    temptab->proto = env;\n"
+                              "    " (meta :static-entry) "(temptab);\n"
+                              "    janet_env_lookup_into(lookup, temptab, \""
+                              prefix
+                              "\", 0);\n\n")
+          (buffer/push-string declarations
+                              "extern void "
+                              (meta :static-entry)
+                              "(JanetTable *);\n"))
+
+
+        # Build image
+        (def image (marshal main mdict))
+        # Make image byte buffer
+        (create-buffer-c-impl image cimage_dest "janet_payload_image")
+        # Append main function
+        (spit cimage_dest (string
+                            "\n"
+                            declarations
+```
+
+int main(int argc, const char **argv) {
+    janet_init();
+
+    /* Get core env */
+    JanetTable *env = janet_core_env(NULL);
+    JanetTable *lookup = janet_env_lookup(env);
+    JanetTable *temptab;
+    int handle = janet_gclock();
+
+    /* Load natives into unmarshalling dictionary */
+
+```
+                            lookup-into-invocations
+```
+    /* Unmarshal bytecode */
+    Janet marsh_out = janet_unmarshal(
+      janet_payload_image_embed,
+      janet_payload_image_embed_size,
+      0,
+      lookup,
+      NULL);
+
+    /* Verify the marshalled object is a function */
+    if (!janet_checktype(marsh_out, JANET_FUNCTION)) {
+      fprintf(stderr, "invalid bytecode image - expected function.");
+      return 1;
+    }
+
+    /* Collect command line arguments */
+    JanetArray *args = janet_array(argc);
+    for (int i = 0; i < argc; i++) {
+      janet_array_push(args, janet_cstringv(argv[i]));
+    }
+
+    /* Create enviornment */
+    JanetTable *runtimeEnv = janet_table(0);
+    runtimeEnv->proto = env;
+    janet_table_put(runtimeEnv, janet_ckeywordv("args"), janet_wrap_array(args));
+    janet_gcroot(janet_wrap_table(runtimeEnv));
+
+    /* Unlock GC */
+    janet_gcunlock(handle);
+
+    /* Run everything */
+    JanetFiber *fiber = janet_fiber(janet_unwrap_function(marsh_out), 64, argc, args->data);
+    fiber->env = runtimeEnv;
+    Janet out;
+    JanetSignal result = janet_continue(fiber, janet_wrap_nil(), &out);
+    if (result) {
+      janet_stacktrace(fiber, out);
+      janet_deinit();
+      return result;
+    }
+    janet_deinit();
+    return 0;
+}
+
+```) :ab)
+
+# Compile and link final exectable
+(do
+  (def extra-lflags (case (os/which)
+                      :macos ["-ldl" "-lm"]
+                      :windows []
+                      :linux ["-lm" "-ldl" "-lrt"]
+                      #default
+                      ["-lm"]))
+  (def cc (opt opts :compiler default-compiler))
+  (def lflags [;(opt opts :lflags default-lflags) ;extra-lflags])
+  (def cflags (getcflags opts))
+  (def defines (make-defines (opt opts :defines {})))
+  (print "compiling and linking " dest "...")
+  (if is-win
+    (shell cc ;cflags cimage_dest ;static-libs (libjanet) ;lflags `/link` (string "/OUT:" dest))
+    (shell cc ;cflags `-o` dest cimage_dest ;static-libs (libjanet) ;lflags)))))
+
+(defn- abspath
+  "Create an absolute path. Does not resolve . and .. (useful for
+  generating entries in install manifest file)."
+  [path]
+  (if (if is-win
+        (peg/match '(+ "\\" (* (range "AZ" "az") ":\\")) path)
+        (string/has-prefix? "/" path))
+    path
+    (string (os/cwd) sep path)))
+
+#
+# Public utilities
+#
+
+(defn find-manifest-dir
+  "Get the path to the directory containing manifests for installed
+  packages."
+  []
+  (string (dyn :modpath JANET_MODPATH) sep ".manifests"))
+
+(defn find-manifest
+  "Get the full path of a manifest file given a package name."
+  [name]
+  (string (find-manifest-dir) sep name ".txt"))
+
+(defn find-cache
+  "Return the path to the global cache."
+  []
+  (def path (dyn :modpath JANET_MODPATH))
+  (string path sep ".cache"))
+
+(defn uninstall
+  "Uninstall bundle named name"
+  [name]
+  (def manifest (find-manifest name))
+  (def f (file/open manifest :r))
+  (unless f (print manifest " does not exist") (break))
+  (loop [line :iterate (:read f :line)]
+    (def path ((string/split "\n" line) 0))
+    (def path ((string/split "\r" path) 0))
+    (print "removing " path)
+    (try (rm path) ([err]
+                    (unless (= err "No such file or directory")
+                      (error err)))))
+  (:close f)
+  (print "removing " manifest)
+  (rm manifest)
+  (print "Uninstalled."))
+
+(defn clear-cache
+  "Clear the global git cache."
+  []
+  (def cache (find-cache))
+  (print "clearing " cache "...")
+  (if is-win
+    # Git for windows decided that .git should be hidden and everything in it read-only.
+    # This means we can't delete things easily.
+    (os/shell (string `rmdir /S /Q "` cache `"`))
+    (rm cache)))
+
+(defn install-git
+  "Install a bundle from git. If the bundle is already installed, the bundle
+  is reinistalled (but not rebuilt if artifacts are cached)."
+  [repotab]
+  (def repo (if (string? repotab) repotab (repotab :repo)))
+  (def tag (unless (string? repotab) (repotab :tag)))
+  (def cache (find-cache))
+  (os/mkdir cache)
+  (def id (filepath-replace repo))
+  (def module-dir (string cache sep id))
+  (var fresh false)
+  (when (os/mkdir module-dir)
+    (set fresh true)
+    (os/execute ["git" "clone" repo module-dir] :p))
+  (def olddir (os/cwd))
+  (try
+    (with-dyns [:rules @{}
+                :modpath (abspath (dyn :modpath JANET_MODPATH))
+                :headerpath (abspath (dyn :headerpath JANET_HEADERPATH))
+                :libpath (abspath (dyn :libpath JANET_LIBPATH))
+                :binpath (abspath (dyn :binpath JANET_BINPATH))]
+      (os/cd module-dir)
+      (unless fresh
+        (os/execute ["git" "pull" "origin" "master"] :p))
+      (when tag
+        (os/execute ["git" "reset" "--hard" tag] :p))
+      (os/execute ["git" "submodule" "update" "--init" "--recursive"] :p)
+      (import-rules "./project.janet")
+      (do-rule "install-deps")
+      (do-rule "build")
+      (do-rule "install"))
+    ([err] (print "Error building git repository dependency: " err)))
+  (os/cd olddir))
+
+(defn install-rule
+  "Add install and uninstall rule for moving file from src into destdir."
+  [src destdir]
+  (def parts (string/split sep src))
+  (def name (last parts))
+  (def path (string destdir sep name))
+  (array/push (dyn :installed-files) path)
+  (add-body "install"
+            (try (os/mkdir destdir) ([err] nil))
+            (copy src destdir)))
+
+#
+# Declaring Artifacts - used in project.janet, targets specifically
+# tailored for janet.
+#
+
+(defn declare-native
+  "Declare a native module. This is a shared library that can be loaded
+  dynamically by a janet runtime. This also builds a static libary that
+  can be used to bundle janet code and native into a single executable."
+  [&keys opts]
+  (def sources (opts :source))
+  (def name (opts :name))
+  (def path (dyn :modpath JANET_MODPATH))
+
+  # Make dynamic module
+  (def lname (string "build" sep name modext))
+  (loop [src :in sources]
+    (compile-c opts src (out-path src ".c" objext)))
+  (def objects (map (fn [path] (out-path path ".c" objext)) sources))
+  (when-let [embedded (opts :embedded)]
+            (loop [src :in embedded]
+              (def c-src (out-path src ".janet" ".janet.c"))
+              (def o-src (out-path src ".janet" (if is-win ".janet.obj" ".janet.o")))
+              (array/push objects o-src)
+              (create-buffer-c src c-src (embed-name src))
+              (compile-c opts c-src o-src)))
+  (link-c opts lname ;objects)
+  (add-dep "build" lname)
+  (install-rule lname path)
+
+  # Add meta file
+  (def metaname (modpath-to-meta lname))
+  (def ename (entry-name name))
+  (rule metaname []
+        (print "generating meta file " metaname "...")
+        (spit metaname (string/format
+                         "# Metadata for static library %s\n\n%.20p"
+                         (string name statext)
+                         {:static-entry ename
+                          :lflags (opts :lflags)})))
+  (add-dep "build" metaname)
+  (install-rule metaname path)
+
+  # Make static module
+  (unless (dyn :nostatic)
+    (def sname (string "build" sep name statext))
+    (def opts (merge @{:entry-name ename} opts))
+    (def sobjext (string ".static" objext))
+    (def sjobjext (string ".janet" sobjext))
+    (loop [src :in sources]
+      (compile-c opts src (out-path src ".c" sobjext) true))
+    (def sobjects (map (fn [path] (out-path path ".c" sobjext)) sources))
+    (when-let [embedded (opts :embedded)]
+              (loop [src :in embedded]
+                (def c-src (out-path src ".janet" ".janet.c"))
+                (def o-src (out-path src ".janet" sjobjext))
+                (array/push sobjects o-src)
+                # Buffer c-src is already declared by dynamic module
+                (compile-c opts c-src o-src true)))
+    (archive-c opts sname ;sobjects)
+    (add-dep "build" sname)
+    (install-rule sname path)))
+
+(defn declare-source
+  "Create a Janet modules. This does not actually build the module(s),
+  but registers it for packaging and installation."
+  [&keys {:source sources}]
+  (def path (dyn :modpath JANET_MODPATH))
+  (if (bytes? sources)
+    (install-rule sources path)
+    (each s sources
+      (install-rule s path))))
+
+(defn declare-bin
+  "Declare a generic file to be installed as an executable."
+  [&keys {:main main}]
+  (install-rule main (dyn :binpath JANET_BINPATH)))
+
+(defn declare-executable
+  "Declare a janet file to be the entry of a standalone executable program. The entry
+  file is evaluated and a main function is looked for in the entry file. This function
+  is marshalled into bytecode which is then embedded in a final executable for distribution.\n\n
+  This executable can be installed as well to the --binpath given."
+  [&keys {:install install :name name :entry entry}]
+  (def name (if is-win (string name ".exe") name))
+  (def dest (string "build" sep name))
+  (create-executable @{} entry dest)
+  (add-dep "build" dest)
+  (when install
+    (install-rule dest (dyn :binpath JANET_BINPATH))))
+
+(defn declare-binscript
+  "Declare a janet file to be installed as an executable script. Creates
+  a shim on windows."
+  [&keys opts]
+  (def main (opts :main))
+  (def binpath (dyn :binpath JANET_BINPATH))
+  (install-rule main binpath)
+  # Create a dud batch file when on windows.
+  (when is-win
+    (def name (last (string/split sep main)))
+    (def fullname (string binpath sep name))
+    (def bat (string "@echo off\r\njanet \"" fullname "\" %*"))
+    (def newname (string binpath sep name ".bat"))
+    (array/push (dyn :installed-files) newname)
+    (add-body "install"
+              (spit newname bat))))
+
+(defn declare-archive
+  "Build a janet archive. This is a file that bundles together many janet
+  scripts into a janet image. This file can the be moved to any machine with
+  a janet vm and the required dependencies and run there."
+  [&keys opts]
+  (def entry (opts :entry))
+  (def name (opts :name))
+  (def iname (string "build" sep name ".jimage"))
+  (rule iname (or (opts :deps) [])
+        (spit iname (make-image (require entry))))
+  (def path (dyn :modpath JANET_MODPATH))
+  (add-dep "build" iname)
+  (install-rule iname path))
+
+(defn declare-project
+  "Define your project metadata. This should
+  be the first declaration in a project.janet file.
+  Also sets up basic phony targets like clean, build, test, etc."
+  [&keys meta]
+  (setdyn :project meta)
+
+  (def installed-files @[])
+  (def manifests (find-manifest-dir))
+  (def manifest (find-manifest (meta :name)))
+  (setdyn :manifest manifest)
+  (setdyn :manifest-dir manifests)
+  (setdyn :installed-files installed-files)
+
+  (rule "./build" [] (os/mkdir "build"))
+  (phony "build" ["./build"])
+
+  (phony "manifest" []
+         (print "generating " manifest "...")
+         (os/mkdir manifests)
+         (spit manifest (string (string/join installed-files "\n") "\n")))
+  (phony "install" ["uninstall" "build" "manifest"]
+         (print "Installed as '" (meta :name) "'."))
+
+  (phony "install-deps" []
+         (if-let [deps (meta :dependencies)]
+           (each dep deps
+             (install-git dep))
+           (print "no dependencies found")))
+
+  (phony "uninstall" []
+         (uninstall (meta :name)))
+
+  (phony "clean" []
+         (when (os/stat "./build" :mode)
+           (rm "build")
+           (print "Deleted build directory.")))
+
+  (phony "test" ["build"]
+         (defn dodir
+           [dir]
+           (each sub (os/dir dir)
+             (def ndir (string dir sep sub))
+             (case (os/stat ndir :mode)
+               :file (when (string/has-suffix? ".janet" ndir)
+                       (print "running " ndir " ...")
+                       (dofile ndir :exit true))
+               :directory (dodir ndir))))
+         (dodir "test")
+         (print "All tests passed.")))
+
+#
+# CLI
+#

 (def- argpeg
  (peg/compile
@@ -10,82 +796,100 @@

 (defn- local-rule
  [rule]
-  (cook/import-rules "./project.janet")
-  (cook/do-rule rule))
+  (import-rules "./project.janet")
+  (do-rule rule))

 (defn- help
  []
  (print `
 usage: jpm [--key=value, --flag] ... [subcommand] [args] ...

+Run from a directory containing a project.janet file to perform operations
+on a project, or from anywhere to do operations on the global module cache (modpath).
+
 Subcommands are:
  build : build all artifacts
+  help : show this help text
  install (repo) : install artifacts. If a repo is given, install the contents of that
                   git repository, assuming that the repository is a jpm project. If not, build
                   and install the current project.
  uninstall (module) : uninstall a module. If no module is given, uninstall the module
                       defined by the current directory.
  clean : remove any generated files or artifacts
-  test : run tests
-  deps : install dependencies.
+  test : run tests. Tests should be .janet files in the test/ directory relative to project.janet.
+  deps : install dependencies for the current project.
  clear-cache : clear the git cache. Useful for updating dependencies.
+  run rule : run a rule. Can also run custom rules added via (phony "task" [deps...] ...)
+             or (rule "ouput.file" [deps...] ...).
+  rules : list rules available with run.

 Keys are:
-  --modpath : The directory to install modules to. Defaults to $JANET_MODPATH or (dyn :syspath)
+  --modpath : The directory to install modules to. Defaults to $JANET_MODPATH, $JANET_PATH, or (dyn :syspath)
  --headerpath : The directory containing janet headers. Defaults to $JANET_HEADERPATH.
  --binpath : The directory to install binaries and scripts. Defaults to $JANET_BINPATH.
  --libpath : The directory containing janet C libraries (libjanet.*). Defaults to $JANET_LIBPATH.
-  --optimize : Optimization level for natives. Defaults to 2.
  --compiler : C compiler to use for natives. Defaults to cc (cl on windows).
+  --archiver : C compiler to use for static libraries. Defaults to ar (lib on windows).
  --linker : C linker to use for linking natives. Defaults to cc (link on windows).
-  --cflags : Extra compiler flags for native modules.
-  --lflags : Extra linker flags for native modules.

 Flags are:
  --verbose : Print shell commands as they are executed.
    `))

-(defn build
+(defn- show-help
+  []
+  (print help))
+
+(defn- build
  []
  (local-rule "build"))

-(defn clean
+(defn- clean
  []
  (local-rule "clean"))

-(defn install
+(defn- install
  [&opt repo]
  (if repo
-    (cook/install-git repo)
+    (install-git repo)
    (local-rule "install")))

-(defn test
+(defn- test
  []
  (local-rule "test"))

-(defn uninstall
+(defn- uninstall-cmd
  [&opt what]
  (if what
-    (cook/uninstall what)
+    (uninstall what)
    (local-rule "uninstall")))

-(defn deps
+(defn- deps
  []
  (local-rule "install-deps"))

-(def subcommands
+(defn- list-rules
+  []
+  (import-rules "./project.janet")
+  (def ks (sort (seq [k :keys (dyn :rules)] k)))
+  (each k ks (print k)))
+
+(def- subcommands
  {"build" build
   "clean" clean
+   "help" show-help
   "install" install
   "test" test
   "help" help
   "deps" deps
-   "clear-cache" cook/clear-cache
-   "uninstall" uninstall})
+   "clear-cache" clear-cache
+   "run" local-rule
+   "rules" list-rules
+   "uninstall" uninstall-cmd})

-(def args (tuple/slice (dyn :args) 1))
-(def len (length args))
-(var i 0)
+(def- args (tuple/slice (dyn :args) 1))
+(def- len (length args))
+(var i :private 0)

 # Get flags
 (while (< i len)
--- a/auxlib/cook.janet
+++ b/auxlib/cook.janet
@@ -1,747 +0,0 @@
-### cook.janet
-###
-### Library to help build janet natives and other
-### build artifacts.
-###
-### Copyright 2019 © Calvin Rose
-
-#
-# Basic Path Settings
-#
-
-# Windows is the OS outlier
-(def- is-win (= (os/which) :windows))
-(def- is-mac (= (os/which) :macos))
-(def- sep (if is-win "\\" "/"))
-(def- objext (if is-win ".obj" ".o"))
-(def- modext (if is-win ".dll" ".so"))
-(def- statext (if is-win ".static.lib" ".a"))
-(def- absprefix (if is-win "C:\\" "/"))
-
-#
-# Rule Engine
-#
-
-(defn- getrules []
-  (if-let [rules (dyn :rules)] rules (setdyn :rules @{})))
-
-(defn- gettarget [target]
-  (def item ((getrules) target))
-  (unless item (error (string "No rule for target " target)))
-  item)
-
-(defn- rule-impl
-  [target deps thunk &opt phony]
-  (put (getrules) target @[(array/slice deps) thunk phony]))
-
-(defmacro rule
-  "Add a rule to the rule graph."
-  [target deps & body]
-  ~(,rule-impl ,target ,deps (fn [] nil ,;body)))
-
-(defmacro phony
-  "Add a phony rule to the rule graph. A phony rule will run every time
-  (it is always considered out of date). Phony rules are good for defining
-  user facing tasks."
-  [target deps & body]
-  ~(,rule-impl ,target ,deps (fn [] nil ,;body) true))
-
-(defn add-dep
-  "Add a dependency to an existing rule. Useful for extending phony
-  rules or extending the dependency graph of existing rules."
-  [target dep]
-  (def [deps] (gettarget target))
-  (array/push deps dep))
-
-(defn- add-thunk
-  [target more]
-  (def item (gettarget target))
-  (def [_ thunk] item)
-  (put item 1 (fn [] (more) (thunk))))
-
-(defmacro add-body
-  "Add recipe code to an existing rule. This makes existing rules do more but
-  does not modify the dependency graph."
-  [target & body]
-  ~(,add-thunk ,target (fn [] ,;body)))
-
-(defn- needs-build
-  [dest src]
-  (let [mod-dest (os/stat dest :modified)
-        mod-src (os/stat src :modified)]
-    (< mod-dest mod-src)))
-
-(defn- needs-build-some
-  [dest sources]
-  (def f (file/open dest))
-  (if (not f) (break true))
-  (file/close f)
-  (some (partial needs-build dest) sources))
-
-(defn do-rule
-  "Evaluate a given rule."
-  [target]
-  (def item ((getrules) target))
-  (unless item
-    (if (os/stat target :mode)
-      (break target)
-      (error (string "No rule for file " target " found."))))
-  (def [deps thunk phony] item)
-  (def realdeps (seq [dep :in deps :let [x (do-rule dep)] :when x] x))
-  (when (or phony (needs-build-some target realdeps))
-    (thunk))
-  (unless phony target))
-
-#
-# Configuration
-#
-
-(def JANET_MODPATH (or (os/getenv "JANET_MODPATH") (dyn :syspath)))
-(def JANET_HEADERPATH (or (os/getenv "JANET_HEADERPATH")
-                          (if-let [j JANET_MODPATH]
-                            (string j "/../../include/janet"))))
-(def JANET_BINPATH (or (os/getenv "JANET_BINPATH")
-                       (if-let [j JANET_MODPATH]
-                         (string j "/../../bin"))))
-(def JANET_LIBPATH (or (os/getenv "JANET_LIBPATH")
-                       (if-let [j JANET_MODPATH]
-                         (string j "/.."))))
-
-#
-# Compilation Defaults
-#
-
-(def default-compiler (if is-win "cl" "cc"))
-(def default-linker (if is-win "link" "cc"))
-(def default-archiver (if is-win "lib" "ar"))
-
-# Default flags for natives, but not required
-(def default-lflags (if is-win ["/nologo"] []))
-(def default-cflags
-  (if is-win
-    ["/nologo"]
-    ["-std=c99" "-Wall" "-Wextra"]))
-
-# Required flags for dynamic libraries. These
-# are used no matter what for dynamic libraries.
-(def- dynamic-cflags
-  (if is-win
-    []
-    ["-fpic"]))
-(def- dynamic-lflags
-    (if is-win
-      ["/DLL"]
-      (if is-mac
-        ["-shared" "-undefined" "dynamic_lookup"]
-        ["-shared"])))
-
-(defn- opt
-  "Get an option, allowing overrides via dynamic bindings AND some
-  default value dflt if no dynamic binding is set."
-  [opts key dflt]
-  (def ret (or (opts key) (dyn key dflt)))
-  (if (= nil ret)
-    (error (string "option :" key " not set")))
-  ret)
-
-#
-# Importing a file
-#
-
-(def- _env (fiber/getenv (fiber/current)))
-
-(defn- proto-flatten
-  [into x]
-  (when x
-    (proto-flatten into (table/getproto x))
-    (loop [k :keys x]
-      (put into k (x k))))
-  into)
-
-(defn import-rules
-  "Import another file that defines more cook rules. This ruleset
-  is merged into the current ruleset."
-  [path]
-  (def env (make-env))
-  (unless (os/stat path :mode)
-    (error (string "cannot open " path)))
-  (loop [k :keys _env :when (symbol? k)]
-     (unless ((_env k) :private) (put env k (_env k))))
-  (def currenv (proto-flatten @{} (fiber/getenv (fiber/current))))
-  (loop [k :keys currenv :when (keyword? k)]
-    (put env k (currenv k)))
-  (dofile path :env env :exit true)
-  (when-let [rules (env :rules)] (merge-into (getrules) rules)))
-
-#
-# OS and shell helpers
-#
-
-(def- filepath-replacer
-  "Convert url with potential bad characters into a file path element."
-  (peg/compile ~(% (any (+ (/ '(set "<>:\"/\\|?*") "_") '1)))))
-
-(defn filepath-replace
-  "Remove special characters from a string or path
-  to make it into a path segment."
-  [repo]
-  (get (peg/match filepath-replacer repo) 0))
-
-(defn shell
-  "Do a shell command"
-  [& args]
-  (if (dyn :verbose)
-    (print ;(interpose " " args)))
-  (def res (os/execute args :p))
-  (unless (zero? res)
-    (error (string "command exited with status " res))))
-
-(defn rm
-  "Remove a directory and all sub directories."
-  [path]
-  (if (= (os/stat path :mode) :directory)
-    (do
-      (each subpath (os/dir path)
-        (rm (string path sep subpath)))
-      (os/rmdir path))
-    (os/rm path)))
-
-(defn copy
-  "Copy a file or directory recursively from one location to another."
-  [src dest]
-  (print "copying " src " to " dest "...")
-  (if is-win
-    (shell "xcopy" src dest "/y" "/e")
-    (shell "cp" "-rf" src dest)))
-
-#
-# C Compilation
-#
-
-(defn- embed-name
-  "Rename a janet symbol for embedding."
-  [path]
-  (->> path
-       (string/replace-all sep "___")
-       (string/replace-all ".janet" "")))
-
-(defn- out-path
-  "Take a source file path and convert it to an output path."
-  [path from-ext to-ext]
-  (->> path
-       (string/replace-all sep "___")
-       (string/replace-all from-ext to-ext)
-       (string "build" sep)))
-
-(defn- make-define
-  "Generate strings for adding custom defines to the compiler."
-  [define value]
-  (if value
-    (string (if is-win "/D" "-D") define "=" value)
-    (string (if is-win "/D" "-D") define)))
-
-(defn- make-defines
-  "Generate many defines. Takes a dictionary of defines. If a value is
-  true, generates -DNAME (/DNAME on windows), otherwise -DNAME=value."
-  [defines]
-  (seq [[d v] :pairs defines] (make-define d (if (not= v true) v))))
-
-(defn- getcflags
-  "Generate the c flags from the input options."
-  [opts]
-  @[;(opt opts :cflags default-cflags)
-    (string (if is-win "/I" "-I") (dyn :headerpath JANET_HEADERPATH))
-    (string (if is-win "/O" "-O") (opt opts :optimize 2))])
-
-(defn- entry-name
-  "Name of symbol that enters static compilation of a module."
-  [name]
-  (string "janet_module_entry_" (filepath-replace name)))
-
-(defn- compile-c
-  "Compile a C file into an object file."
-  [opts src dest &opt static?]
-  (def cc (opt opts :compiler default-compiler))
-  (def cflags [;(getcflags opts) ;(if static? [] dynamic-cflags)])
-  (def entry-defines (if-let [n (opts :entry-name)]
-                       [(make-define "JANET_ENTRY_NAME" n)]
-                       []))
-  (def defines [;(make-defines (opt opts :defines {})) ;entry-defines])
-  (def headers (or (opts :headers) []))
-  (rule dest [src ;headers]
-        (print "compiling " dest "...")
-        (if is-win
-          (shell cc ;defines "/c" ;cflags (string "/Fo" dest) src)
-          (shell cc "-c" src ;defines ;cflags "-o" dest))))
-
-(defn- libjanet
-  "Find libjanet.a (or libjanet.lib on windows) at compile time"
-  []
-  (def libpath (dyn :libpath JANET_LIBPATH))
-  (unless libpath
-    (error "cannot find libpath: provide --libpath or JANET_LIBPATH"))
-  (string (dyn :libpath JANET_LIBPATH)
-          sep
-          (if is-win "libjanet.lib" "libjanet.a")))
-
-(defn- win-import-library
-  "On windows, an import library is needed to link to a dll statically."
-  []
-  (def hpath (dyn :headerpath JANET_HEADERPATH))
-  (unless hpath
-    (error "cannot find headerpath: provide --headerpath or JANET_HEADERPATH"))
-  (string hpath `\\janet.lib`))
-
-(defn- link-c
-  "Link object files together to make a native module."
-  [opts target & objects]
-  (def ld (opt opts :linker default-linker))
-  (def cflags (getcflags opts))
-  (def lflags [;(opt opts :lflags default-lflags)
-               ;(if (opts :static) [] dynamic-lflags)])
-  (rule target objects
-        (print "linking " target "...")
-        (if is-win
-          (shell ld ;lflags (string "/OUT:" target) ;objects (win-import-library))
-          (shell ld ;cflags `-o` target ;objects ;lflags))))
-
-(defn- archive-c
-  "Link object files together to make a static library."
-  [opts target & objects]
-  (def ar (opt opts :archiver default-archiver))
-  (rule target objects
-        (print "creating static library " target "...")
-        (if is-win
-          (shell ar "/nologo" (string "/out:" target) ;objects)
-          (shell ar "rcs" target ;objects))))
-
-(defn- create-buffer-c-impl
-  [bytes dest name]
-  (def out (file/open dest :w))
-  (def chunks (seq [b :in bytes] (string b)))
-  (file/write out
-              "#include <janet.h>\n"
-              "static const unsigned char bytes[] = {"
-              ;(interpose ", " chunks)
-              "};\n\n"
-              "const unsigned char *" name "_embed = bytes;\n"
-              "size_t " name "_embed_size = sizeof(bytes);\n")
-  (file/close out))
-
-(defn- create-buffer-c
-  "Inline raw byte file as a c file."
-  [source dest name]
-  (rule dest [source]
-        (print "generating " dest "...")
-        (with [f (file/open source :r)]
-          (create-buffer-c-impl (:read f :all) dest name))))
-
-(def- root-env (table/getproto (fiber/getenv (fiber/current))))
-
-(defn- modpath-to-meta
-  "Get the meta file path (.meta.janet) corresponding to a native module path (.so)."
-  [path]
-  (string (string/slice path 0 (- (length modext))) "meta.janet"))
-
-(defn- modpath-to-static
-  "Get the static library (.a) path corresponding to a native module path (.so)."
-  [path]
-  (string (string/slice path 0 (- -1 (length modext))) statext))
-
-(defn- create-executable
-  "Links an image with libjanet.a (or .lib) to produce an
-  executable. Also will try to link native modules into the
-  final executable as well."
-  [opts source dest]
-
-  # Create executable's janet image
-  (def cimage_dest (string dest ".c"))
-  (rule dest [source]
-        (print "generating executable c source...")
-        # Load entry environment and get main function.
-        (def entry-env (dofile source))
-        (def main ((entry-env 'main) :value))
-
-        # Create marshalling dictionary
-        (def mdict (invert (env-lookup root-env)))
-        # Load all native modules
-        (def prefixes @{})
-        (def static-libs @[])
-        (loop [[name m] :pairs module/cache
-               :let [n (m :native)]
-               :when n
-               :let [prefix (gensym)]]
-          (print "found native " n "...")
-          (put prefixes prefix n)
-          (array/push static-libs (modpath-to-static n))
-          (def oldproto (table/getproto m))
-          (table/setproto m nil)
-          (loop [[sym value] :pairs (env-lookup m)]
-            (put mdict value (symbol prefix sym)))
-          (table/setproto m oldproto))
-
-        # Find static modules
-        (def declarations @"")
-        (def lookup-into-invocations @"")
-        (loop [[prefix name] :pairs prefixes]
-          (def meta (eval-string (slurp (modpath-to-meta name))))
-          (buffer/push-string lookup-into-invocations
-                              "    temptab = janet_table(0);\n"
-                              "    temptab->proto = env;\n"
-                              "    " (meta :static-entry) "(temptab);\n"
-                              "    janet_env_lookup_into(lookup, temptab, \""
-                              prefix
-                              "\", 0);\n\n")
-          (buffer/push-string declarations
-                              "extern void "
-                              (meta :static-entry)
-                              "(JanetTable *);\n"))
-
-
-        # Build image
-        (def image (marshal main mdict))
-        # Make image byte buffer
-        (create-buffer-c-impl image cimage_dest "janet_payload_image")
-        # Append main function
-        (spit cimage_dest (string
-                            "\n"
-                            declarations
-```
-
-int main(int argc, const char **argv) {
-    janet_init();
-
-    /* Get core env */
-    JanetTable *env = janet_core_env(NULL);
-    JanetTable *lookup = janet_env_lookup(env);
-    JanetTable *temptab;
-    int handle = janet_gclock();
-
-    /* Load natives into unmarshalling dictionary */
-
-```
-                            lookup-into-invocations
-```
-    /* Unmarshal bytecode */
-    Janet marsh_out = janet_unmarshal(
-      janet_payload_image_embed,
-      janet_payload_image_embed_size,
-      0,
-      lookup,
-      NULL);
-
-    /* Verify the marshalled object is a function */
-    if (!janet_checktype(marsh_out, JANET_FUNCTION)) {
-      fprintf(stderr, "invalid bytecode image - expected function.");
-      return 1;
-    }
-
-    /* Collect command line arguments */
-    JanetArray *args = janet_array(argc);
-    for (int i = 0; i < argc; i++) {
-      janet_array_push(args, janet_cstringv(argv[i]));
-    }
-
-    /* Create enviornment */
-    JanetTable *runtimeEnv = janet_table(0);
-    runtimeEnv->proto = env;
-    janet_table_put(runtimeEnv, janet_ckeywordv("args"), janet_wrap_array(args));
-    janet_gcroot(janet_wrap_table(runtimeEnv));
-
-    /* Unlock GC */
-    janet_gcunlock(handle);
-
-    /* Run everything */
-    JanetFiber *fiber = janet_fiber(janet_unwrap_function(marsh_out), 64, argc, args->data);
-    fiber->env = runtimeEnv;
-    Janet out;
-    JanetSignal result = janet_continue(fiber, janet_wrap_nil(), &out);
-    if (result) {
-      janet_stacktrace(fiber, out);
-      janet_deinit();
-      return result;
-    }
-    janet_deinit();
-    return 0;
-}
-
-```) :ab)
-
-# Compile and link final exectable
-(do
-  (def extra-lflags (case (os/which)
-                      :macos ["-ldl" "-lm"]
-                      :windows []
-                      :linux ["-lm" "-ldl" "-lrt"]
-                      #default
-                      ["-lm"]))
-  (def cc (opt opts :compiler default-compiler))
-  (def lflags [;(opt opts :lflags default-lflags) ;extra-lflags])
-  (def cflags (getcflags opts))
-  (def defines (make-defines (opt opts :defines {})))
-  (print "compiling and linking " dest "...")
-  (if is-win
-    (shell cc ;cflags (string "/OUT:" dest) cimage_dest ;static-libs (libjanet) ;lflags)
-    (shell cc ;cflags `-o` dest cimage_dest ;static-libs (libjanet) ;lflags)))))
-
-(defn- abspath
-  "Create an absolute path. Does not resolve . and .. (useful for
-  generating entries in install manifest file)."
-  [path]
-  (if (string/has-prefix? absprefix)
-    path
-    (string (os/cwd) sep path)))
-
-#
-# Public utilities
-#
-
-(defn find-manifest-dir
-  "Get the path to the directory containing manifests for installed
-  packages."
-  []
-  (string (dyn :modpath JANET_MODPATH) sep ".manifests"))
-
-(defn find-manifest
-  "Get the full path of a manifest file given a package name."
-  [name]
-  (string (find-manifest-dir) sep name ".txt"))
-
-(defn find-cache
-  "Return the path to the global cache."
-  []
-  (def path (dyn :modpath JANET_MODPATH))
-  (string path sep ".cache"))
-
-(defn uninstall
-  "Uninstall bundle named name"
-  [name]
-  (def manifest (find-manifest name))
-  (def f (file/open manifest :r))
-  (unless f (print manifest " does not exist") (break))
-  (loop [line :iterate (:read f :line)]
-    (def path ((string/split "\n" line) 0))
-    (print "removing " path)
-    (try (rm path) ([err]
-                    (unless (= err "No such file or directory")
-                      (error err)))))
-  (:close f)
-  (print "removing " manifest)
-  (rm manifest)
-  (print "Uninstalled."))
-
-(defn clear-cache
-  "Clear the global git cache."
-  []
-  (def cache (find-cache))
-  (print "clearing " cache "...")
-  (if is-win
-    # Git for windows decided that .git should be hidden and everything in it read-only.
-    # This means we can't delete things easily.
-    (os/shell (string `rmdir /S /Q "` cache `"`))
-    (rm cache)))
-
-(defn install-git
-  "Install a bundle from git. If the bundle is already installed, the bundle
-  is reinistalled (but not rebuilt if artifacts are cached)."
-  [repo]
-  (def cache (find-cache))
-  (os/mkdir cache)
-  (def id (filepath-replace repo))
-  (def module-dir (string cache sep id))
-  (when (os/mkdir module-dir)
-    (os/execute ["git" "clone" repo module-dir] :p))
-  (def olddir (os/cwd))
-  (os/cd module-dir)
-  (try
-    (with-dyns [:rules @{}]
-      (import-rules "./project.janet")
-      (do-rule "install-deps")
-      (do-rule "build")
-      (do-rule "install"))
-    ([err] nil))
-  (os/cd olddir))
-
-(defn install-rule
-  "Add install and uninstall rule for moving file from src into destdir."
-  [src destdir]
-  (def parts (string/split sep src))
-  (def name (last parts))
-  (def path (string destdir sep name))
-  (array/push (dyn :installed-files) path)
-  (add-body "install"
-            (try (os/mkdir destdir) ([err] nil))
-            (copy src destdir)))
-
-#
-# Declaring Artifacts - used in project.janet, targets specifically
-# tailored for janet.
-#
-
-(defn declare-native
-  "Declare a native module. This is a shared library that can be loaded
-  dynamically by a janet runtime. This also builds a static libary that
-  can be used to bundle janet code and native into a single executable."
-  [&keys opts]
-  (def sources (opts :source))
-  (def name (opts :name))
-  (def path (dyn :modpath JANET_MODPATH))
-
-  # Make dynamic module
-  (def lname (string "build" sep name modext))
-  (loop [src :in sources]
-    (compile-c opts src (out-path src ".c" objext)))
-  (def objects (map (fn [path] (out-path path ".c" objext)) sources))
-  (when-let [embedded (opts :embedded)]
-            (loop [src :in embedded]
-              (def c-src (out-path src ".janet" ".janet.c"))
-              (def o-src (out-path src ".janet" (if is-win ".janet.obj" ".janet.o")))
-              (array/push objects o-src)
-              (create-buffer-c src c-src (embed-name src))
-              (compile-c opts c-src o-src)))
-  (link-c opts lname ;objects)
-  (add-dep "build" lname)
-  (install-rule lname path)
-
-  # Add meta file
-  (def metaname (modpath-to-meta lname))
-  (def ename (entry-name name))
-  (rule metaname []
-        (print "generating meta file " metaname "...")
-        (spit metaname (string/format
-                         "# Metadata for static library %s\n\n%.20p"
-                         (string name statext)
-                         {:static-entry ename
-                          :lflags (opts :lflags)})))
-  (add-dep "build" metaname)
-  (install-rule metaname path)
-
-  # Make static module
-  (unless (dyn :nostatic)
-    (def sname (string "build" sep name statext))
-    (def opts (merge @{:entry-name ename} opts))
-    (def sobjext (string ".static" objext))
-    (def sjobjext (string ".janet" sobjext))
-    (loop [src :in sources]
-      (compile-c opts src (out-path src ".c" sobjext) true))
-    (def sobjects (map (fn [path] (out-path path ".c" sobjext)) sources))
-    (when-let [embedded (opts :embedded)]
-              (loop [src :in embedded]
-                (def c-src (out-path src ".janet" ".janet.c"))
-                (def o-src (out-path src ".janet" sjobjext))
-                (array/push sobjects o-src)
-                # Buffer c-src is already declared by dynamic module
-                (compile-c opts c-src o-src true)))
-    (archive-c opts sname ;sobjects)
-    (add-dep "build" sname)
-    (install-rule sname path)))
-
-(defn declare-source
-  "Create a Janet modules. This does not actually build the module(s),
-  but registers it for packaging and installation."
-  [&keys {:source sources}]
-  (def path (dyn :modpath JANET_MODPATH))
-  (if (bytes? sources)
-    (install-rule sources path)
-    (each s sources
-      (install-rule s path))))
-
-(defn declare-bin
-  "Declare a generic file to be installed as an executable."
-  [&keys {:main main}]
-  (install-rule main (dyn :binpath JANET_BINPATH)))
-
-(defn declare-executable
-  "Declare a janet file to be the entry of a standalone executable program. The entry
-  file is evaluated and a main function is looked for in the entry file. This function
-  is marshalled into bytecode which is then embedded in a final executable for distribution.\n\n
-  This executable can be installed as well to the --binpath given."
-  [&keys {:install install :name name :entry entry}]
-  (def name (if is-win (string name ".exe") name))
-  (def dest (string "build" sep name))
-  (create-executable @{} entry dest)
-  (add-dep "build" dest)
-  (when install
-    (install-rule dest (dyn :binpath JANET_BINPATH))))
-
-(defn declare-binscript
-  "Declare a janet file to be installed as an executable script. Creates
-  a shim on windows."
-  [&keys opts]
-  (def main (opts :main))
-  (def binpath (dyn :binpath JANET_BINPATH))
-  (install-rule main binpath)
-  # Create a dud batch file when on windows.
-  (when is-win
-    (def name (last (string/split sep main)))
-    (def bat (string "@echo off\r\njanet %~dp0\\" name "%*"))
-    (def newname (string binpath sep name ".bat"))
-    (add-body "install"
-              (spit newname bat))
-    (add-body "uninstall"
-              (os/rm newname))))
-
-(defn declare-archive
-  "Build a janet archive. This is a file that bundles together many janet
-  scripts into a janet image. This file can the be moved to any machine with
-  a janet vm and the required dependencies and run there."
-  [&keys opts]
-  (def entry (opts :entry))
-  (def name (opts :name))
-  (def iname (string "build" sep name ".jimage"))
-  (rule iname (or (opts :deps) [])
-        (spit iname (make-image (require entry))))
-  (def path (dyn :modpath JANET_MODPATH))
-  (add-dep "build" iname)
-  (install-rule iname path))
-
-(defn declare-project
-  "Define your project metadata. This should
-  be the first declaration in a project.janet file.
-  Also sets up basic phony targets like clean, build, test, etc."
-  [&keys meta]
-  (setdyn :project meta)
-
-  (def installed-files @[])
-  (def manifests (find-manifest-dir))
-  (def manifest (find-manifest (meta :name)))
-  (setdyn :manifest manifest)
-  (setdyn :manifest-dir manifests)
-  (setdyn :installed-files installed-files)
-
-  (rule "./build" [] (os/mkdir "build"))
-  (phony "build" ["./build"])
-
-  (phony "manifest" []
-         (print "generating " manifest "...")
-         (os/mkdir manifests)
-         (spit manifest (string (string/join installed-files "\n") "\n")))
-  (phony "install" ["uninstall" "build" "manifest"]
-         (print "Installed as '" (meta :name) "'."))
-
-  (phony "install-deps" []
-         (if-let [deps (meta :dependencies)]
-           (each dep deps
-             (install-git dep))
-           (print "no dependencies found")))
-
-  (phony "uninstall" []
-         (uninstall (meta :name)))
-
-  (phony "clean" []
-         (when (os/stat "./build" :mode)
-           (rm "build")
-           (print "Deleted build directory.")))
-
-  (phony "test" ["build"]
-         (defn dodir
-           [dir]
-           (each sub (os/dir dir)
-             (def ndir (string dir sep sub))
-             (case (os/stat ndir :mode)
-               :file (when (string/has-suffix? ".janet" ndir)
-                       (print "running " ndir " ...")
-                       (dofile ndir :exit true))
-               :directory (dodir ndir))))
-         (dodir "test")
-         (print "All tests passed.")))
--- a/auxlib/path.janet
+++ b/auxlib/path.janet
@@ -1,149 +0,0 @@
-### path.janet
-###
-### A library for path manipulation.
-###
-### Copyright 2019 © Calvin Rose
-
-#
-# Common
-#
-
-(def- ext-peg
-  (peg/compile ~{:back (> -1 (+ (* ($) (set "\\/.")) :back))
-                 :main :back}))
-
-(defn ext
-  "Get the file extension for a path."
-  [path]
-  (if-let [m (peg/match ext-peg path (length path))]
-    (let [i (m 0)]
-      (if (= (path i) 46)
-        (string/slice path (m 0) -1)))))
-
-(defn- redef
-  "Redef a value, keeping all metadata."
-  [from to]
-  (setdyn (symbol to) (dyn (symbol from))))
-
-#
-# Generating Macros
-#
-
-(defmacro- decl-sep [pre sep] ~(def ,(symbol pre "/sep") ,sep))
-(defmacro- decl-delim [pre d] ~(def ,(symbol pre "/delim") ,d))
-
-(defmacro- decl-last-sep
-  [pre sep]
-  ~(def- ,(symbol pre "/last-sep-peg")
-    (peg/compile ~{:back (> -1 (+ (* ,sep ($)) :back))
-                   :main :back})))
-
-(defmacro- decl-basename
-  [pre]
-  ~(defn ,(symbol pre "/basename")
-    "Gets the base file name of a path."
-    [path]
-    (if-let [m (peg/match
-                 ,(symbol pre "/last-sep-peg")
-                 path
-                 (length path))]
-      (let [[p] m]
-        (string/slice path p -1))
-      path)))
-
-(defmacro- decl-parts
-  [pre sep]
-  ~(defn ,(symbol pre "/parts")
-     "Split a path into its parts."
-     [path]
-     (string/split ,sep path)))
-
-(defmacro- decl-normalize
-  [pre sep lead]
-  ~(defn ,(symbol pre "/normalize")
-     "Normalize a path. This removes . and .. in the
-     path, as well as empty path elements."
-     [path]
-     (def els (string/split ,sep path))
-     (def newparts @[])
-     (if (,(symbol pre "/abspath?") path) (array/push newparts ,lead))
-     (each part els
-       (case part
-         "" nil
-         "." nil
-         ".." (array/pop newparts)
-         (array/push newparts part)))
-     (string/join newparts ,sep)))
-
-(defmacro- decl-join
-  [pre sep]
-  ~(defn ,(symbol pre "/join")
-     "Join path elements together."
-     [& els]
-     (,(symbol pre "/normalize") (string/join els ,sep))))
-
-(defmacro- decl-abspath
-  [pre]
-  ~(defn ,(symbol pre "/abspath")
-     "Coerce a path to be absolute."
-     [path]
-     (if (,(symbol pre "/abspath?") path)
-       path
-       (,(symbol pre "/join") (os/cwd) path))))
-
-#
-# Posix
-#
-
-(defn posix/abspath?
-  "Check if a path is absolute."
-  [path]
-  (string/has-prefix? "/" path))
-
-(redef "ext" "posix/ext")
-(decl-sep "posix" "/")
-(decl-delim "posix" ":")
-(decl-last-sep "posix" "/")
-(decl-basename "posix")
-(decl-parts "posix" "/")
-(decl-normalize "posix" "/" "")
-(decl-join "posix" "/")
-(decl-abspath "posix")
-
-#
-# Windows
-#
-
-(def- abs-peg (peg/compile '(* (range "AZ") ":\\")))
-(defn win32/abspath?
-  "Check if a path is absolute."
-  [path]
-  (peg/match abs-peg path))
-
-(redef "ext" "win32/ext")
-(decl-sep "win32" "\\")
-(decl-delim "win32" ";")
-(decl-last-sep "win32" "\\")
-(decl-basename "win32")
-(decl-parts "win32" "\\")
-(decl-normalize "win32" "\\" "C:")
-(decl-join "win32" "\\")
-(decl-abspath "win32")
-
-#
-# Specialize for current OS
-#
-
-(def- syms
-  ["ext"
-   "sep"
-   "delim"
-   "basename"
-   "abspath?"
-   "abspath"
-   "parts"
-   "normalize"
-   "join"])
-(let [pre (if (= :windows (os/which)) "win32" "posix")]
-  (each sym syms
-    (redef (string pre "/" sym) sym)))
--- a/build_win.bat
+++ b/build_win.bat
@@ -23,36 +23,41 @@
@set JANET_LINK=link /nologo
@set JANET_LINK_STATIC=lib /nologo

+@rem Add janet build tag
+if not "%JANET_BUILD%" == "" (
+    @set JANET_COMPILE=%JANET_COMPILE% /DJANET_BUILD="\"%JANET_BUILD%\""
+)
+
 mkdir build
 mkdir build\core
 mkdir build\mainclient
 mkdir build\boot

@rem Build the xxd tool for generating sources
-@cl /nologo /c tools/xxd.c /Fobuild\xxd.obj
+cl /nologo /c tools/xxd.c /Fobuild\xxd.obj
@if errorlevel 1 goto :BUILDFAIL
-@link /nologo /out:build\xxd.exe build\xxd.obj
+link /nologo /out:build\xxd.exe build\xxd.obj
@if errorlevel 1 goto :BUILDFAIL

@rem Generate the embedded sources
-@build\xxd.exe src\mainclient\init.janet build\init.gen.c janet_gen_init
+build\xxd.exe src\mainclient\init.janet build\init.gen.c janet_gen_init
@if errorlevel 1 goto :BUILDFAIL
-@build\xxd.exe src\boot\boot.janet build\boot.gen.c janet_gen_boot
+build\xxd.exe src\boot\boot.janet build\boot.gen.c janet_gen_boot
@if errorlevel 1 goto :BUILDFAIL

@rem Build the generated sources
-@%JANET_COMPILE% /Fobuild\mainclient\init.gen.obj build\init.gen.c
+%JANET_COMPILE% /Fobuild\mainclient\init.gen.obj build\init.gen.c
@if errorlevel 1 goto :BUILDFAIL
-@%JANET_COMPILE% /Fobuild\boot\boot.gen.obj build\boot.gen.c
+%JANET_COMPILE% /Fobuild\boot\boot.gen.obj build\boot.gen.c
@if errorlevel 1 goto :BUILDFAIL

@rem Build the bootstrap interpreter
 for %%f in (src\core\*.c) do (
-    @%JANET_COMPILE% /DJANET_BOOTSTRAP /Fobuild\boot\%%~nf.obj %%f
+    %JANET_COMPILE% /DJANET_BOOTSTRAP /Fobuild\boot\%%~nf.obj %%f
    @if errorlevel 1 goto :BUILDFAIL
 )
 for %%f in (src\boot\*.c) do (
-    @%JANET_COMPILE% /DJANET_BOOTSTRAP /Fobuild\boot\%%~nf.obj %%f
+    %JANET_COMPILE% /DJANET_BOOTSTRAP /Fobuild\boot\%%~nf.obj %%f
    @if errorlevel 1 goto :BUILDFAIL
 )
 %JANET_LINK% /out:build\janet_boot.exe build\boot\*.obj
@@ -60,12 +65,12 @@ for %%f in (src\boot\*.c) do (
 build\janet_boot build\core_image.c

@rem Build the core image
-@%JANET_COMPILE% /Fobuild\core_image.obj build\core_image.c
+%JANET_COMPILE% /Fobuild\core_image.obj build\core_image.c
@if errorlevel 1 goto :BUILDFAIL

@rem Build the sources
 for %%f in (src\core\*.c) do (
-    @%JANET_COMPILE% /Fobuild\core\%%~nf.obj %%f
+    %JANET_COMPILE% /Fobuild\core\%%~nf.obj %%f
    @if errorlevel 1 goto :BUILDFAIL
 )

@@ -74,7 +79,7 @@ rc /nologo /fobuild\janet_win.res janet_win.rc

@rem Build the main client
 for %%f in (src\mainclient\*.c) do (
-    @%JANET_COMPILE% /Fobuild\mainclient\%%~nf.obj %%f
+    %JANET_COMPILE% /Fobuild\mainclient\%%~nf.obj %%f
    @if errorlevel 1 goto :BUILDFAIL
 )

@@ -145,9 +150,6 @@ copy src\include\janet.h dist\janet.h
 copy src\conf\janetconf.h dist\janetconf.h
 copy build\libjanet.lib dist\libjanet.lib

-copy auxlib\cook.janet dist\cook.janet
-copy auxlib\path.janet dist\path.janet
-
 copy auxbin\jpm dist\jpm
 copy tools\jpm.bat dist\jpm.bat

@@ -170,6 +172,10 @@ call jpm clean
@if errorlevel 1 goto :TESTFAIL
 call jpm test
@if errorlevel 1 goto :TESTFAIL
+call jpm --verbose --modpath=. install https://github.com/janet-lang/json.git
+@if errorlevel 1 goto :TESTFAIL
+call build\testexec
+@if errorlevel 1 goto :TESTFAIL
 popd
 exit /b 0

--- a/janet-installer.nsi
+++ b/janet-installer.nsi
@@ -1,5 +1,5 @@
 # Version
-!define VERSION "1.2.0"
+!define VERSION "1.3.0"
 !define PRODUCT_VERSION "${VERSION}.0"
 VIProductVersion "${PRODUCT_VERSION}"
 VIFileVersion "${PRODUCT_VERSION}"
@@ -25,7 +25,19 @@ VIFileVersion "${PRODUCT_VERSION}"

 # Basics
 Name "Janet"
-OutFile "janet-v${VERSION}-windows-installer.exe"
+
+# Do some NSIS-fu to figure out at compile time if we are in appveyor
+!define OUTNAME $%APPVEYOR_REPO_TAG_NAME%
+!define "CHECK_${OUTNAME}"
+!define DOLLAR "$"
+!ifdef CHECK_${DOLLAR}%APPVEYOR_REPO_TAG_NAME%
+    # We are not in the appveyor environment, use version name
+    !define OUTNAME_PART v${VERSION}
+!else
+    # We are in appveyor, use git tag name for installer
+    !define OUTNAME_PART ${OUTNAME}
+!endif
+OutFile "janet-${OUTNAME_PART}-windows-installer.exe"

 # Some Configuration
 !define APPNAME "Janet"
@@ -91,10 +103,6 @@ section "Janet" BfWSection
    file /oname=bin\jpm.janet auxbin\jpm
    file /oname=bin\jpm.bat tools\jpm.bat

-    # Modules
-    file /oname=Library\cook.janet auxlib\cook.janet
-    file /oname=Library\path.janet auxlib\path.janet
-
    # C headers and library files
    file /oname=C\janet.h dist\janet.h
    file /oname=C\janetconf.h dist\janetconf.h
--- a/jpm.1
+++ b/jpm.1
@@ -0,0 +1,176 @@
+.TH JPM 1
+.SH NAME
+jpm \- the Janet Project Manager, a build tool for Janet 
+.SH SYNOPSIS
+.B jpm
+[\fB\-\-flag ...\fR]
+[\fB\-\-option=value ...\fR]
+.IR command
+.IR args ...
+.SH DESCRIPTION
+jpm is the build tool that ships with a standard Janet install. It is
+used for building Janet projects, installing dependencies, installing
+projects, building native modules, and exporting your Janet project to a
+standalone executable. Although not required for working with Janet, it
+removes much of the boilerplate with installing dependencies and
+building native modules. jpm requires only Janet to run, and uses git
+to install dependencies (jpm will work without git installed).
+.SH DOCUMENTATION
+
+jpm has several subcommands, each used for managing either a single Janet project or
+all Janet modules installed on the system. Global commands, those that manage modules
+at the system level, do things like install and uninstall packages, as well as clear the cache.
+More interesting are the local commands. For more information on jpm usage, see https://janet-lang.org/docs/index.html
+
+.SH FLAGS
+
+.TP
+.BR \-\-verbose
+Print detailed messages of what jpm is doing, including compilation commands and other shell commands.
+
+.SH OPTIONS
+
+.TP
+.BR \-\-modpath=/some/path
+Set the path to install modules to. Defaults to $JANET_MODPATH, $JANET_PATH, or (dyn :syspath) in that order.
+
+.TP
+.BR \-\-headerpath=/some/path
+Set the path the jpm will include when building C source code. This lets
+you specify the location of janet.h and janetconf.h on your system. On a
+normal install, this option is not needed.
+
+.TP
+.BR \-\-binpath=/some/path
+Set the path that jpm will install scripts and standalone executables to. Executables
+defined via declare-execuatble or scripts declared via declare-binscript will be installed
+here when jpm install is run. Defaults to $JANET_BINPATH, or a reasonable default for the system.
+See JANET_BINPATH for more.
+
+.TP
+.BR \-\-libpath=/some/path
+Sets the path jpm will use to look for libjanet.a for building standalone executables. libjanet.so
+is \fBnot\fR used for building native modules or standalone executables, only
+for linking into applications that want to embed janet as a dynamic module.
+Linking statically might be a better idea, even in that case. Defaults to
+$JANET_LIBPATH, or a reasonable default. See JANET_LIBPATH for more.
+
+.TP
+.BR \-\-compiler=cc
+Sets the compiler used for compiling native modules and standalone executables. Defaults
+to cc.
+
+.TP
+.BR \-\-linker=ld
+Sets the linker used to create native modules and executables.
+
+.TP
+.BR \-\-archiver=ar
+Sets the command used for creating static libraries, use for linking into the standalone executable.
+Native modules are compiled twice, once a normal native module (shared object), and once as an
+archive.
+
+.SH COMMANDS
+.TP
+.BR help
+Shows the usage text and exits immediately.
+
+.TP
+.BR build
+Builds all artifacts specified in the project.janet file in the current directory. Artifacts will
+be created in the ./build/ directory.
+
+.TP
+.BR install\ [\fBrepo\fR]
+
+When run with no arguments, installs all installable artifacts in the current project to
+the current JANET_MODPATH for modules and JANET_BINPATH for executables and scripts. Can also
+take an optional git repository URL and will install all artifacts in that repository instead.
+When run with an argument, install does not need to be run from a jpm project directory.
+
+.TP
+.BR uninstall\ [\fBname\fR]
+Uninstall a project installed with install. uninstall expects the name of the project, not the
+repository url, path to installed file or executable name. The name of the project must be specified
+at the top of the project.janet file in the declare-project form. If no name is given, uninstalls
+the current project if installed.
+
+.TP
+.BR clean
+Remove all artifacts created by jpm. This just deletes the build folder.
+
+.TP
+.BR test
+Runs jpm tests. jpm will run all janet source files in the test directory as tests. A test
+is considered failing if it exits with a non-zero exit code.
+
+.TP
+.BR deps
+Install all dependencies that this project requires recursively. jpm does not
+resolve dependency issues, like conflicting versions of the same module are required, or
+different modules with the same name. Dependencies are installed with git, so deps requires
+git to be on the PATH.
+
+.TP
+.BR clear-cache
+jpm caches git repositories that are needed to install modules from a remote
+source in a global cache ($JANET_PATH/.cache). If these dependencies are out of
+date or too large, clear-cache will remove the cache and jpm will rebuild it
+when needed. clear-cache is a global command, so a project.janet is not
+required.
+
+.TP
+.BR run\ [\fBrule\fR]
+Run a given rule defined in project.janet. Project definitions files (project.janet) usually
+contain a few artifact declarations, which set up rules that jpm can then resolve, or execute.
+A project.janet can also create custom rules to create arbitrary files or run arbitrary code, much
+like make. run will run a single rule or build a single file.
+
+.TP
+.BR rules
+List all rules that can be run via run. This is useful for exploring rules in the project.
+
+.SH ENVIRONMENT
+
+.B JANET_PATH
+.RS
+The location to look for Janet libraries. This is the only environment variable Janet needs to
+find native and source code modules. If no JANET_PATH is set, Janet will look in
+the default location set at compile time, which can be determined with (dyn :syspath)
+.RE
+
+.B JANET_MODPATH
+.RS
+The location that jpm will use to install libraries to. Defaults to JANET_PATH, but you could
+set this to a different directory if you want to. Doing so would let you import Janet modules
+on the normal system path (JANET_PATH or (dyn :syspath)), but install to a different directory. It is also a more reliable way to install
+This variable is overwritten by the --modpath=/some/path if it is provided.
+.RE
+
+.B JANET_HEADERPATH
+.RS
+The location that jpm will look for janet header files (janet.h and janetconf.h) that are used
+to build native modules and standalone executables. If janet.h and janetconf.h are available as
+default includes on your system, this value is not required. If not provided, will default to
+(dyn :syspath)/../../include/janet. The --headerpath=/some/path will override this variable.
+.RE
+
+.B JANET_LIBPATH
+.RS
+Similar to JANET_HEADERPATH, this path is where jpm will look for
+libjanet.a for creating standalong executables. This does not need to be
+set on a normal install. 
+If not provided, this will default to (dyn :syspath)/../../lib.
+The --libpath=/some/path will override this variable.
+.RE
+
+.B JANET_BINPATH
+.RS
+The directory where jpm will install binary scripts and executables to.
+Defaults to
+(dyn :syspath)/../../lib.
+The --binpath=/some/path will override this variable.
+.RE
+
+.SH AUTHOR
+Written by Calvin Rose <calsrose@gmail.com>
--- a/meson.build
+++ b/meson.build
@@ -20,7 +20,7 @@

 project('janet', 'c',
  default_options : ['c_std=c99', 'b_lundef=false', 'default_library=both'],
-  version : '1.2.0')
+  version : '1.3.0')

 # Global settings
 janet_path = join_paths(get_option('prefix'), get_option('libdir'), 'janet')
@@ -65,6 +65,12 @@ conf.set('JANET_RECURSION_GUARD', get_option('recursion_guard'))
 conf.set('JANET_MAX_PROTO_DEPTH', get_option('max_proto_depth'))
 conf.set('JANET_MAX_MACRO_EXPAND', get_option('max_macro_expand'))
 conf.set('JANET_STACK_MAX', get_option('stack_max'))
+if get_option('os_name') != ''
+  conf.set('JANET_OS_NAME', get_option('os_name'))
+endif
+if get_option('arch_name') != ''
+  conf.set('JANET_ARCH_NAME', get_option('arch_name'))
+endif
 jconf = configure_file(output : 'janetconf.h',
  configuration : conf)

@@ -222,13 +228,9 @@ janet_dep = declare_dependency(include_directories : incdir,

 # Installation
 install_man('janet.1')
+install_man('jpm.1')
 install_headers(['src/include/janet.h', jconf], subdir: 'janet')
-janet_libs = [
-  'auxlib/cook.janet',
-  'auxlib/path.janet'
-]
 janet_binscripts = [
  'auxbin/jpm'
 ]
-install_data(sources : janet_libs, install_dir : janet_path)
 install_data(sources : janet_binscripts, install_dir : 'bin')
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -14,4 +14,7 @@ option('int_types', type : 'boolean', value : true)
 option('recursion_guard', type : 'integer', min : 10, max : 8000, value : 1024)
 option('max_proto_depth', type : 'integer', min : 10, max : 8000, value : 200)
 option('max_macro_expand', type : 'integer', min : 1, max : 8000, value : 200)
-option('stack_max', type : 'integer', min : 8096, max : 1000000000, value : 16384)
+option('stack_max', type : 'integer', min : 8096, max : 0x7fffffff, value : 0x7fffffff)
+
+option('arch_name', type : 'string', value: '')
+option('os_name', type : 'string', value: '')
--- a/src/boot/boot.janet
+++ b/src/boot/boot.janet
@@ -952,6 +952,58 @@
    (put res (get keys i) (get vals i)))
  res)

+(defn get-in
+  "Access a value in a nested data structure. Looks into the data structure via
+  a sequence of keys."
+  [ds ks &opt dflt]
+  (var d ds)
+  (loop [k :in ks :while d] (set d (get d k)))
+  (or d dflt))
+
+(defn update-in
+  "Update a value in a nested data structure by applying f to the current value.
+  Looks into the data structure via
+  a sequence of keys. Missing data structures will be replaced with tables. Returns
+  the modified, original data structure."
+  [ds ks f & args]
+  (var d ds)
+  (def len-1 (- (length ks) 1))
+  (if (< len-1 0) (error "expected at least 1 key in ks"))
+  (for i 0 len-1
+    (def k (get ks i))
+    (def v (get d k))
+    (if (= nil v)
+      (let [newv (table)]
+        (put d k newv)
+        (set d newv))
+      (set d v)))
+  (def last-key (get ks len-1))
+  (def last-val (get d last-key))
+  (put d last-key (f last-val ;args))
+  ds)
+
+(defn put-in
+  "Put a value into a nested data structure.
+  Looks into the data structure via
+  a sequence of keys. Missing data structures will be replaced with tables. Returns
+  the modified, original data structure."
+  [ds ks v]
+  (var d ds)
+  (def len-1 (- (length ks) 1))
+  (if (< len-1 0) (error "expected at least 1 key in ks"))
+  (for i 0 len-1
+    (def k (get ks i))
+    (def v (get d k))
+    (if (= nil v)
+      (let [newv (table)]
+        (put d k newv)
+        (set d newv))
+      (set d v)))
+  (def last-key (get ks len-1))
+  (def last-val (get d last-key))
+  (put d last-key v)
+  ds)
+
 (defn update
  "Accepts a key argument and passes its associated value to a function.
  The key is the re-associated to the function's return value. Returns the updated
@@ -1129,7 +1181,7 @@
 (defn pp
  "Pretty print to stdout."
  [x]
-  (print (buffer/format @"" (dyn :pretty-format "%p") x)))
+  (print (buffer/format @"" (dyn :pretty-format "%q") x)))


 ###
@@ -1444,6 +1496,21 @@
  [x y]
  (not (deep-not= x y)))

+(defn freeze
+  "Freeze an object (make it immutable) and do a deep copy, making
+  child values also immutable. Closures, fibers, and abstract types
+  will not be recursively frozen, but all other types will."
+  [x]
+  (case (type x)
+    :array (tuple/slice (map freeze x))
+    :tuple (tuple/slice (map freeze x))
+    :table (if-let [p (table/getproto x)]
+              (freeze (merge (table/clone p) x))
+              (struct ;(map freeze (kvs x))))
+    :struct (struct ;(map freeze (kvs x)))
+    :buffer (string x)
+    x))
+
 (defn macex
  "Expand macros completely.
  on-binding is an optional callback whenever a normal symbolic binding
--- a/src/conf/janetconf.h
+++ b/src/conf/janetconf.h
@@ -27,10 +27,10 @@
 #define JANETCONF_H

 #define JANET_VERSION_MAJOR 1
-#define JANET_VERSION_MINOR 2
+#define JANET_VERSION_MINOR 3
 #define JANET_VERSION_PATCH 0
 #define JANET_VERSION_EXTRA ""
-#define JANET_VERSION "1.2.0"
+#define JANET_VERSION "1.3.0"

 /* #define JANET_BUILD "local" */

@@ -56,5 +56,7 @@
 /* #define JANET_MAX_PROTO_DEPTH 200 */
 /* #define JANET_MAX_MACRO_EXPAND 200 */
 /* #define JANET_STACK_MAX 16384 */
+/* #define JANET_OS_NAME my-custom-os */
+/* #define JANET_ARCH_NAME pdp-8 */

 #endif /* end of include guard: JANETCONF_H */
--- a/src/core/array.c
+++ b/src/core/array.c
@@ -64,7 +64,9 @@ void janet_array_ensure(JanetArray *array, int32_t capacity, int32_t growth) {
    Janet *newData;
    Janet *old = array->data;
    if (capacity <= array->capacity) return;
-    capacity *= growth;
+    int64_t new_capacity = capacity * growth;
+    if (new_capacity > INT32_MAX) new_capacity = INT32_MAX;
+    capacity = (int32_t) new_capacity;
    newData = realloc(old, capacity * sizeof(Janet));
    if (NULL == newData) {
        JANET_OUT_OF_MEMORY;
--- a/src/core/debug.c
+++ b/src/core/debug.c
@@ -100,6 +100,9 @@ void janet_stacktrace(JanetFiber *fiber, Janet err) {
    JanetFiber **fibers = NULL;
    int wrote_error = 0;

+    int print_color = janet_truthy(janet_dyn("err-color"));
+    if (print_color) fprintf(out, "\x1b[31m");
+
    while (fiber) {
        janet_v_push(fibers, fiber);
        fiber = fiber->child;
@@ -157,6 +160,8 @@ void janet_stacktrace(JanetFiber *fiber, Janet err) {
        }
    }

+    if (print_color) fprintf(out, "\x1b[0m");
+
    janet_v_free(fibers);
 }

--- a/src/core/fiber.c
+++ b/src/core/fiber.c
@@ -87,19 +87,27 @@ void janet_fiber_setcapacity(JanetFiber *fiber, int32_t n) {
    fiber->capacity = n;
 }

+/* Grow fiber if needed */
+static void janet_fiber_grow(JanetFiber *fiber, int32_t needed) {
+    int32_t cap = needed > (INT32_MAX / 2) ? INT32_MAX : 2 * needed;
+    janet_fiber_setcapacity(fiber, cap);
+}
+
 /* Push a value on the next stack frame */
 void janet_fiber_push(JanetFiber *fiber, Janet x) {
+    if (fiber->stacktop == INT32_MAX) janet_panic("stack overflow");
    if (fiber->stacktop >= fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * fiber->stacktop);
+        janet_fiber_grow(fiber, fiber->stacktop);
    }
    fiber->data[fiber->stacktop++] = x;
 }

 /* Push 2 values on the next stack frame */
 void janet_fiber_push2(JanetFiber *fiber, Janet x, Janet y) {
+    if (fiber->stacktop >= INT32_MAX - 1) janet_panic("stack overflow");
    int32_t newtop = fiber->stacktop + 2;
    if (newtop > fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * newtop);
+        janet_fiber_grow(fiber, newtop);
    }
    fiber->data[fiber->stacktop] = x;
    fiber->data[fiber->stacktop + 1] = y;
@@ -108,9 +116,10 @@ void janet_fiber_push2(JanetFiber *fiber, Janet x, Janet y) {

 /* Push 3 values on the next stack frame */
 void janet_fiber_push3(JanetFiber *fiber, Janet x, Janet y, Janet z) {
+    if (fiber->stacktop >= INT32_MAX - 2) janet_panic("stack overflow");
    int32_t newtop = fiber->stacktop + 3;
    if (newtop > fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * newtop);
+        janet_fiber_grow(fiber, newtop);
    }
    fiber->data[fiber->stacktop] = x;
    fiber->data[fiber->stacktop + 1] = y;
@@ -120,9 +129,10 @@ void janet_fiber_push3(JanetFiber *fiber, Janet x, Janet y, Janet z) {

 /* Push an array on the next stack frame */
 void janet_fiber_pushn(JanetFiber *fiber, const Janet *arr, int32_t n) {
+    if (fiber->stacktop > INT32_MAX - n) janet_panic("stack overflow");
    int32_t newtop = fiber->stacktop + n;
    if (newtop > fiber->capacity) {
-        janet_fiber_setcapacity(fiber, 2 * newtop);
+        janet_fiber_grow(fiber, newtop);
    }
    memcpy(fiber->data + fiber->stacktop, arr, n * sizeof(Janet));
    fiber->stacktop = newtop;
--- a/src/core/os.c
+++ b/src/core/os.c
@@ -64,12 +64,17 @@ extern char **environ;

 /* Full OS functions */

+#define janet_stringify1(x) #x
+#define janet_stringify(x) janet_stringify1(x)
+
 static Janet os_which(int32_t argc, Janet *argv) {
    janet_fixarity(argc, 0);
    (void) argv;
-#ifdef JANET_WINDOWS
+#if defined(JANET_OS_NAME)
+    return janet_ckeywordv(janet_stringify(JANET_OS_NAME));
+#elif defined(JANET_WINDOWS)
    return janet_ckeywordv("windows");
-#elif __APPLE__
+#elif defined(__APPLE__)
    return janet_ckeywordv("macos");
 #elif defined(__EMSCRIPTEN__)
    return janet_ckeywordv("web");
@@ -86,6 +91,33 @@ static Janet os_which(int32_t argc, Janet *argv) {
 #endif
 }

+/* Detect the ISA we are compiled for */
+static Janet os_arch(int32_t argc, Janet *argv) {
+    janet_fixarity(argc, 0);
+    (void) argv;
+    /* Check 64-bit vs 32-bit */
+#if defined(JANET_ARCH_NAME)
+    return janet_ckeywordv(janet_stringify(JANET_ARCH_NAME));
+#elif defined(__EMSCRIPTEN__)
+    return janet_ckeywordv("wasm");
+#elif (defined(__x86_64__) || defined(_M_X64))
+    return janet_ckeywordv("x86-64");
+#elif defined(__i386) || defined(_M_IX86)
+    return janet_ckeywordv("x86");
+#elif defined(_M_ARM64) || defined(__aarch64__)
+    return janet_ckeywordv("aarch64");
+#elif defined(_M_ARM) || defined(__arm__)
+    return janet_ckeywordv("arm");
+#elif (defined(__sparc__))
+    return janet_ckeywordv("sparc");
+#else
+    return janet_ckeywordv("unknown");
+#endif
+}
+
+#undef janet_stringify1
+#undef janet_stringify
+
 static Janet os_exit(int32_t argc, Janet *argv) {
    janet_arity(argc, 0, 1);
    if (argc == 0) {
@@ -783,6 +815,18 @@ static const JanetReg os_cfuns[] = {
        JDOC("(os/getenv variable)\n\n"
             "Get the string value of an environment variable.")
    },
+    {
+        "os/arch", os_arch,
+        JDOC("(os/arch)\n\n"
+             "Check the ISA that janet was compiled for. Returns one of:\n\n"
+             "\t:x86\n"
+             "\t:x86-64\n"
+             "\t:arm\n"
+             "\t:aarch64\n"
+             "\t:sparc\n"
+             "\t:wasm\n"
+             "\t:unknown\n")
+    },
 #ifndef JANET_REDUCED_OS
    {
        "os/dir", os_dir,
--- a/src/core/peg.c
+++ b/src/core/peg.c
@@ -59,6 +59,7 @@ typedef enum {
    RULE_MATCHTIME,    /* [rule, constant, tag] */
    RULE_ERROR,        /* [rule] */
    RULE_DROP,         /* [rule] */
+    RULE_BACKMATCH,    /* [tag] */
 } Opcode;

 /* Hold captured patterns and match state */
@@ -417,6 +418,24 @@ tail:
            }
            return NULL;
        }
+
+        case RULE_BACKMATCH: {
+            uint32_t search = rule[1];
+            for (int32_t i = s->tags->count - 1; i >= 0; i--) {
+                if (s->tags->data[i] == search) {
+                    Janet capture = s->captures->data[i];
+                    if (!janet_checktype(capture, JANET_STRING))
+                        return NULL;
+                    const uint8_t *bytes = janet_unwrap_string(capture);
+                    int32_t len = janet_string_length(bytes);
+                    if (text + len > s->text_end)
+                        return NULL;
+                    return memcmp(text, bytes, len) ? NULL : text + len;
+                }
+            }
+            return NULL;
+        }
+
    }
 }

@@ -426,7 +445,6 @@ tail:

 typedef struct {
    JanetTable *grammar;
-    JanetTable *memoized;
    JanetTable *tags;
    Janet *constants;
    uint32_t *bytecode;
@@ -754,12 +772,20 @@ static void spec_reference(Builder *b, int32_t argc, const Janet *argv) {
    emit_2(r, RULE_GETTAG, search, tag);
 }

-static void spec_position(Builder *b, int32_t argc, const Janet *argv) {
+static void spec_tag1(Builder *b, int32_t argc, const Janet *argv, uint32_t op) {
    peg_arity(b, argc, 0, 1);
    Reserve r = reserve(b, 2);
    uint32_t tag = (argc) ? emit_tag(b, argv[0]) : 0;
    (void) argv;
-    emit_1(r, RULE_POSITION, tag);
+    emit_1(r, op, tag);
+}
+
+static void spec_position(Builder *b, int32_t argc, const Janet *argv) {
+    spec_tag1(b, argc, argv, RULE_POSITION);
+}
+
+static void spec_backmatch(Builder *b, int32_t argc, const Janet *argv) {
+    spec_tag1(b, argc, argv, RULE_BACKMATCH);
 }

 static void spec_argument(Builder *b, int32_t argc, const Janet *argv) {
@@ -824,6 +850,7 @@ static const SpecialPair peg_specials[] = {
    {"argument", spec_argument},
    {"at-least", spec_atleast},
    {"at-most", spec_atmost},
+    {"backmatch", spec_backmatch},
    {"backref", spec_reference},
    {"between", spec_between},
    {"capture", spec_capture},
@@ -850,27 +877,54 @@ static const SpecialPair peg_specials[] = {
 /* Compile a janet value into a rule and return the rule index. */
 static uint32_t peg_compile1(Builder *b, Janet peg) {

-    /* Check for already compiled rules */
-    Janet check = janet_table_get(b->memoized, peg);
-    if (!janet_checktype(check, JANET_NIL)) {
-        uint32_t rule = (uint32_t) janet_unwrap_number(check);
-        return rule;
-    }
-
    /* Keep track of the form being compiled for error purposes */
    Janet old_form = b->form;
+    JanetTable *old_grammar = b->grammar;
    b->form = peg;

-    /* Check depth */
-    if (b->depth-- == 0) {
-        peg_panic(b, "peg grammar recursed too deeply");
+    /* Resolve keyword references */
+    int i = JANET_RECURSION_GUARD;
+    JanetTable *grammar = old_grammar;
+    for (; i > 0 && janet_checktype(peg, JANET_KEYWORD); --i) {
+        peg = janet_table_get_ex(grammar, peg, &grammar);
+        if (!grammar)
+            peg_panic(b, "unknown rule");
+        b->form = peg;
+        b->grammar = grammar;
    }
+    if (i == 0)
+        peg_panic(b, "reference chain too deep");
+
+    /* Check cache - for tuples we check only the local cache, as
+     * in a different grammar, the same tuple can compile to a different
+     * rule - for example, (+ :a :b) depends on whatever :a and :b are bound to. */
+    Janet check = janet_checktype(peg, JANET_TUPLE)
+                  ? janet_table_rawget(grammar, peg)
+                  : janet_table_get(grammar, peg);
+    if (!janet_checktype(check, JANET_NIL)) {
+        b->form = old_form;
+        b->grammar = old_grammar;
+        return (uint32_t) janet_unwrap_number(check);
+    }
+
+    /* Check depth */
+    if (b->depth-- == 0)
+        peg_panic(b, "peg grammar recursed too deeply");

    /* The final rule to return */
    uint32_t rule = janet_v_count(b->bytecode);
-    if (!janet_checktype(peg, JANET_KEYWORD) &&
-            !janet_checktype(peg, JANET_STRUCT)) {
-        janet_table_put(b->memoized, peg, janet_wrap_number(rule));
+
+    /* Add to cache. Do not cache structs, as we don't yet know
+     * what rule they will return! We can just as effectively cache
+     * the structs main rule. */
+    if (!janet_checktype(peg, JANET_STRUCT)) {
+        JanetTable *which_grammar = grammar;
+        /* If we are a primitive pattern, add to the global cache (root grammar table) */
+        if (!janet_checktype(peg, JANET_TUPLE)) {
+            while (which_grammar->proto)
+                which_grammar = which_grammar->proto;
+        }
+        janet_table_put(which_grammar, peg, janet_wrap_number(rule));
    }

    switch (janet_type(peg)) {
@@ -893,22 +947,22 @@ static uint32_t peg_compile1(Builder *b, Janet peg) {
            emit_bytes(b, RULE_LITERAL, len, str);
            break;
        }
-        case JANET_KEYWORD: {
-            Janet check = janet_table_get(b->grammar, peg);
-            if (janet_checktype(check, JANET_NIL))
-                peg_panic(b, "unknown rule");
-            rule = peg_compile1(b, check);
-            break;
-        }
        case JANET_STRUCT: {
-            JanetTable *grammar = janet_struct_to_table(janet_unwrap_struct(peg));
-            grammar->proto = b->grammar;
-            b->grammar = grammar;
-            Janet main_rule = janet_table_get(grammar, janet_ckeywordv("main"));
+            /* Build grammar table */
+            const JanetKV *st = janet_unwrap_struct(peg);
+            JanetTable *new_grammar = janet_table(2 * janet_struct_capacity(st));
+            for (int32_t i = 0; i < janet_struct_capacity(st); i++) {
+                if (janet_checktype(st[i].key, JANET_KEYWORD)) {
+                    janet_table_put(new_grammar, st[i].key, st[i].value);
+                }
+            }
+            new_grammar->proto = grammar;
+            b->grammar = grammar = new_grammar;
+            /* Run the main rule */
+            Janet main_rule = janet_table_rawget(grammar, janet_ckeywordv("main"));
            if (janet_checktype(main_rule, JANET_NIL))
                peg_panic(b, "grammar requires :main rule");
            rule = peg_compile1(b, main_rule);
-            b->grammar = grammar->proto;
            break;
        }
        case JANET_TUPLE: {
@@ -935,6 +989,7 @@ static uint32_t peg_compile1(Builder *b, Janet peg) {
    /* Increase depth again */
    b->depth++;
    b->form = old_form;
+    b->grammar = old_grammar;
    return rule;
 }

@@ -1029,6 +1084,7 @@ static void peg_unmarshal(void *p, JanetMarshalContext *ctx) {
            case RULE_NOTNCHAR:
            case RULE_RANGE:
            case RULE_POSITION:
+            case RULE_BACKMATCH:
                /* [1 word] */
                i += 2;
                break;
@@ -1160,7 +1216,6 @@ static Peg *make_peg(Builder *b) {
 static Peg *compile_peg(Janet x) {
    Builder builder;
    builder.grammar = janet_table(0);
-    builder.memoized = janet_table(0);
    builder.tags = janet_table(0);
    builder.constants = NULL;
    builder.bytecode = NULL;
--- a/src/core/pp.c
+++ b/src/core/pp.c
@@ -310,7 +310,7 @@ struct pretty {

 static void print_newline(struct pretty *S, int just_a_space) {
    int i;
-    if (just_a_space) {
+    if (just_a_space || (S->flags & JANET_PRETTY_ONELINE)) {
        janet_buffer_push_u8(S->buffer, ' ');
        return;
    }
@@ -725,12 +725,20 @@ void janet_buffer_format(
                    janet_description_b(b, argv[arg]);
                    break;
                }
+                case 'Q':
+                case 'q':
                case 'P':
                case 'p': { /* janet pretty , precision = depth */
                    int depth = atoi(precision);
                    if (depth < 1)
                        depth = 4;
-                    janet_pretty_(b, depth, (strfrmt[-1] == 'P') ? JANET_PRETTY_COLOR : 0, argv[arg], startlen);
+                    char c = strfrmt[-1];
+                    int has_color = (c == 'P') || (c == 'Q');
+                    int has_oneline = (c == 'Q') || (c == 'q');
+                    int flags = 0;
+                    flags |= has_color ? JANET_PRETTY_COLOR : 0;
+                    flags |= has_oneline ? JANET_PRETTY_ONELINE : 0;
+                    janet_pretty_(b, depth, flags, argv[arg], startlen);
                    break;
                }
                default: {
--- a/src/core/table.c
+++ b/src/core/table.c
@@ -137,6 +137,27 @@ Janet janet_table_get(JanetTable *t, Janet key) {
    return janet_wrap_nil();
 }

+/* Get a value out of the table, and record which prototype it was from. */
+Janet janet_table_get_ex(JanetTable *t, Janet key, JanetTable **which) {
+    JanetKV *bucket = janet_table_find(t, key);
+    if (NULL != bucket && !janet_checktype(bucket->key, JANET_NIL)) {
+        *which = t;
+        return bucket->value;
+    }
+    /* Check prototypes */
+    {
+        int i;
+        for (i = JANET_MAX_PROTO_DEPTH, t = t->proto; t && i; t = t->proto, --i) {
+            bucket = janet_table_find(t, key);
+            if (NULL != bucket && !janet_checktype(bucket->key, JANET_NIL)) {
+                *which = t;
+                return bucket->value;
+            }
+        }
+    }
+    return janet_wrap_nil();
+}
+
 /* Get a value out of the table. Don't check prototype tables. */
 Janet janet_table_rawget(JanetTable *t, Janet key) {
    JanetKV *bucket = janet_table_find(t, key);
--- a/src/core/vm.c
+++ b/src/core/vm.c
@@ -179,7 +179,7 @@ static void vm_do_trace(JanetFunction *func) {
 static Janet call_nonfn(JanetFiber *fiber, Janet callee) {
    int32_t argn = fiber->stacktop - fiber->stackstart;
    Janet ds, key;
-    if (argn != 1) janet_panicf("%v called with arity %d, expected 1", callee, argn);
+    if (argn != 1) janet_panicf("%v called with %d arguments, possibly expected 1", callee, argn);
    if (janet_checktypes(callee, JANET_TFLAG_INDEXED | JANET_TFLAG_DICTIONARY |
                         JANET_TFLAG_STRING | JANET_TFLAG_BUFFER | JANET_TFLAG_ABSTRACT)) {
        ds = callee;
@@ -195,7 +195,7 @@ static Janet call_nonfn(JanetFiber *fiber, Janet callee) {
 /* Get a callable from a keyword method name and check ensure that it is valid. */
 static Janet resolve_method(Janet name, JanetFiber *fiber) {
    int32_t argc = fiber->stacktop - fiber->stackstart;
-    if (argc < 1) janet_panicf("method call takes at least 1 argument, got %d", argc);
+    if (argc < 1) janet_panicf("method call (%v) takes at least 1 argument, got 0", name);
    Janet callee = janet_get(fiber->data[fiber->stackstart], name);
    if (janet_checktype(callee, JANET_NIL))
        janet_panicf("unknown method %v invoked on %v", name, fiber->data[fiber->stackstart]);
@@ -628,6 +628,9 @@ static JanetSignal run_vm(JanetFiber *fiber, Janet in, JanetFiberStatus status)

    VM_OP(JOP_TAILCALL) {
        Janet callee = stack[D];
+        if (fiber->stacktop > fiber->maxstack) {
+            vm_throw("stack overflow");
+        }
        if (janet_checktype(callee, JANET_KEYWORD)) {
            vm_commit();
            callee = resolve_method(callee, fiber);
--- a/src/include/janet.h
+++ b/src/include/janet.h
@@ -171,11 +171,10 @@ extern "C" {
 /* Maximum depth to follow table prototypes before giving up and returning nil. */
 #define JANET_MAX_MACRO_EXPAND 200

-/* Define max stack size for stacks before raising a stack overflow error.
- * If this is not defined, fiber stacks can grow without limit (until memory
- * runs out) */
+/* Define default max stack size for stacks before raising a stack overflow error.
+ * This can also be set on a per fiber basis. */
 #ifndef JANET_STACK_MAX
-#define JANET_STACK_MAX 16384
+#define JANET_STACK_MAX 0x7fffffff
 #endif

 /* Use nanboxed values - uses 8 bytes per value instead of 12 or 16.
@@ -1195,6 +1194,7 @@ JANET_API JanetTable *janet_table(int32_t capacity);
 JANET_API JanetTable *janet_table_init(JanetTable *table, int32_t capacity);
 JANET_API void janet_table_deinit(JanetTable *table);
 JANET_API Janet janet_table_get(JanetTable *t, Janet key);
+JANET_API Janet janet_table_get_ex(JanetTable *t, Janet key, JanetTable **which);
 JANET_API Janet janet_table_rawget(JanetTable *t, Janet key);
 JANET_API Janet janet_table_remove(JanetTable *t, Janet key);
 JANET_API void janet_table_put(JanetTable *t, Janet key, Janet value);
@@ -1263,6 +1263,7 @@ JANET_API int janet_verify(JanetFuncDef *def);

 /* Pretty printing */
 #define JANET_PRETTY_COLOR 1
+#define JANET_PRETTY_ONELINE 2
 JANET_API JanetBuffer *janet_pretty(JanetBuffer *buffer, int depth, int flags, Janet x);

 /* Misc */
--- a/src/mainclient/init.janet
+++ b/src/mainclient/init.janet
@@ -91,5 +91,6 @@
    (defn getchunk [buf p]
      (getter (prompter p) buf))
    (def onsig (if *quiet* (fn [x &] x) nil))
-    (setdyn :pretty-format (if *colorize* "%.20P" "%.20p"))
+    (setdyn :pretty-format (if *colorize* "%.20Q" "%.20q"))
+    (setdyn :err-color (if *colorize* true))
    (repl getchunk onsig)))
--- a/test/install/test/test1.janet
+++ b/test/install/test/test1.janet
@@ -1,8 +1,3 @@
 (import build/testmod :as testmod)

 (if (not= 5 (testmod/get5)) (error "testmod/get5 failed"))
-
-(import cook)
-
-(with-dyns [:modpath (os/cwd)]
-  (cook/install-git "https://github.com/janet-lang/json.git"))
--- a/test/suite3.janet
+++ b/test/suite3.janet
@@ -356,6 +356,38 @@
 (check-match janet-longstring "``` `` ```" true)
 (check-match janet-longstring "``  ```" false)

+# Backmatch
+
+(def backmatcher-1 '(* (capture (any "x") :1) "y" (backmatch :1) -1))
+
+(check-match backmatcher-1 "y" true)
+(check-match backmatcher-1 "xyx" true)
+(check-match backmatcher-1 "xxxxxxxyxxxxxxx" true)
+(check-match backmatcher-1 "xyxx" false)
+(check-match backmatcher-1 "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy" false)
+(check-match backmatcher-1 (string (string/repeat "x" 10000) "y") false)
+(check-match backmatcher-1 (string (string/repeat "x" 10000) "y" (string/repeat "x" 10000)) true)
+
+(def backmatcher-2 '(* '(any "x") "y" (backmatch) -1))
+
+(check-match backmatcher-2 "y" true)
+(check-match backmatcher-2 "xyx" true)
+(check-match backmatcher-2 "xxxxxxxyxxxxxxx" true)
+(check-match backmatcher-2 "xyxx" false)
+(check-match backmatcher-2 "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy" false)
+(check-match backmatcher-2 (string (string/repeat "x" 10000) "y") false)
+(check-match backmatcher-2 (string (string/repeat "x" 10000) "y" (string/repeat "x" 10000)) true)
+
+(def longstring-2 '(* '(some "`") (some (if-not (backmatch) 1)) (backmatch) -1))
+
+(check-match longstring-2 "`john" false)
+(check-match longstring-2 "abc" false)
+(check-match longstring-2 "` `" true)
+(check-match longstring-2 "`  `" true)
+(check-match longstring-2 "``  ``" true)
+(check-match longstring-2 "``` `` ```" true)
+(check-match longstring-2 "``  ```" false)
+
 # Optional

 (check-match '(* (opt "hi") -1) "" true)
@@ -389,4 +421,22 @@
 (assert (= (tuple/type (-> '(1 2 3) marshal unmarshal)) :parens) "normal tuple marshalled/unmarshalled")
 (assert (= (tuple/type (-> '[1 2 3] marshal unmarshal)) :brackets) "normal tuple marshalled/unmarshalled")

+# Check for bad memoization (+ :a) should mean different things in different contexts.
+(def redef-a
+  ~{:a "abc"
+    :c (+ :a)
+    :main (* :c {:a "def" :main (+ :a)} -1)})
+
+(check-match redef-a "abcdef" true)
+(check-match redef-a "abcabc" false)
+(check-match redef-a "defdef" false)
+
+(def redef-b
+  ~{:pork {:pork "beef" :main (+ -1 (* 1 :pork))}
+    :main :pork})
+
+(check-match redef-b "abeef" true)
+(check-match redef-b "aabeef" false)
+(check-match redef-b "aaaaaa" false)
+
 (end-suite)
Author	SHA1	Message	Date
Calvin Rose	37a943d9b5	1.3.0 Release	2019-09-05 19:33:08 -05:00
Calvin Rose	2f2b875c2a	Update CHANGELOG.md	2019-09-05 13:21:17 -05:00
Calvin Rose	99f147219a	Add put-in.	2019-09-05 13:19:25 -05:00
Calvin Rose	7a13d24e6f	Add get-in, update-in, and freeze to core.	2019-09-05 13:11:53 -05:00
Calvin Rose	8dc91755f7	Work on makefile and build for jpm.1	2019-09-05 12:28:11 -05:00
Calvin Rose	96a3104fe2	Update to 1.3.0, add jpm.1	2019-09-04 23:44:23 -05:00
Calvin Rose	97f525d069	Update CHANGELOG.md	2019-09-01 11:37:43 -05:00
Calvin Rose	4ad1bdec15	Add `jpm run` and `jpm rules`	2019-09-01 11:26:48 -05:00
Calvin Rose	530d94a4b9	Allow relative paths for jpm commands (deps) Also default headerpath, libpath, and binpath of of (dyn :syspath) instead of $JANET_MODPATH. This allows setting $JANET_MODPATH without needing to mess with the other settings.	2019-09-01 11:08:39 -05:00
Calvin Rose	141d3e9588	Add option for using tags in jpm deps.	2019-08-30 18:23:13 -05:00
Calvin Rose	98eaadf2d1	Simplify peg caching further. Remove the multiple caching tables we were using and use the grammar table for caching. This works well because we can use raw_get for checking the local cache, and normal get fro checking the global cache.	2019-08-30 08:57:45 -05:00
Calvin Rose	54a04b5894	Fix some more recursion issues with pegs. A keyword reference only counts as visited if we have it as cached in the memoized->table, and we know it was originally referenced from the same grammar table. If these two conditions are true, then compilation must work correctly. Also add janet_table_get_ex.	2019-08-29 19:56:04 -05:00
Calvin Rose	8bc8709d0e	Try to address memoization problem in pegs.	2019-08-29 19:09:43 -05:00
Calvin Rose	d4b49cd622	Windows fixes for jpm.	2019-08-29 02:02:05 -04:00
Calvin Rose	7e0586cb55	Fix test-install on windows.	2019-08-28 23:50:15 -04:00
Calvin Rose	05695a35c7	Fix test-install after removing cook.	2019-08-28 21:05:34 -05:00
Calvin Rose	58ffb9d7a5	Remove cook and path from default install Instead, combine cook into jpm so we can manipulate JANET_PATH without messing with jpm. path was moved to and external repository, https://github.com/janet-lang/path.git	2019-08-28 20:54:31 -05:00
Calvin Rose	7eb487d998	Merge branch 'master' of github.com:janet-lang/janet	2019-08-27 18:11:22 -05:00
Calvin Rose	f903ee8acc	Add quotes and remove input path as make target. Make doesn't handle that or auto escape that very well, so we only put known paths as Make targets.	2019-08-27 18:10:03 -05:00
Calvin Rose	91cbe2e22c	Add quotes to shim if install-dir has spaces.	2019-08-25 17:18:01 -04:00
Calvin Rose	c45bad9437	Better shim for scripts on windows. Arguments should be passed in properly.	2019-08-25 17:16:44 -04:00
Calvin Rose	4aa6afbf47	Fix binscripts on windows.	2019-08-25 16:54:54 -04:00
Calvin Rose	29054e8072	Update changelog.	2019-08-24 23:43:51 -04:00
Calvin Rose	060d11e4c2	Add Q and q formatters to buffer/format. These are similar to P and p, but print values on a single line for a much more compact version.	2019-08-24 22:53:45 -04:00
Calvin Rose	77870508de	Update CHANGELOG.md	2019-08-24 19:06:02 -04:00
Calvin Rose	133ad0d355	Add test for longstring matcher using backmatches.	2019-08-24 19:02:55 -04:00
Calvin Rose	711fe64a51	Add backmatch operator to pegs. (backmatch [tag?]) is similar to a back reference in regular expressions (NOT to backwards capture in a peg). It only matches a pattern if it exactly matches the text of the last capture. It does not consume or push any captures to the capture stack.	2019-08-24 18:57:01 -04:00
Calvin Rose	78b5c94cb0	jpm updates. Add better error message if no c compiler detected on windows.	2019-08-24 17:36:50 -04:00
Calvin Rose	95266bdcf8	fix git submodule update command with :p flag	2019-08-23 08:57:41 -05:00
Calvin Rose	b78879dc18	missing closing paren	2019-08-23 08:40:11 -05:00
Calvin Rose	5d29079393	Merge branch 'master' of github.com:janet-lang/janet	2019-08-23 08:35:37 -05:00
Calvin Rose	b052a57fc8	Add error message when dep fails to build.	2019-08-23 08:35:18 -05:00
Calvin Rose	292be33b9d	Fix some stack overflow bugs.	2019-08-19 01:19:51 -04:00
Calvin Rose	0360942942	Add build commit hash to windows build from appveyor.	2019-08-18 21:01:47 -04:00
Calvin Rose	c35d6d2396	More batch syntax issues.	2019-08-18 20:27:26 -04:00
Calvin Rose	1c73d8ce2b	Remove some parens	2019-08-18 20:19:50 -04:00
Calvin Rose	6a539df480	Make sure all appveyor artifacts get deployed	2019-08-18 20:07:12 -04:00
Calvin Rose	1de09ec149	release test 5	2019-08-18 20:02:06 -04:00
Calvin Rose	a1f785038d	release-test4	2019-08-18 19:53:18 -04:00
Calvin Rose	5d475848a6	Fix appveyor.yml	2019-08-18 19:35:17 -04:00
Calvin Rose	2695f2da46	Update installer with appveyor commands.	2019-08-18 19:16:15 -04:00
Calvin Rose	3cdbf5753d	Add some more artifacts to automate release.	2019-08-18 18:02:28 -04:00
Calvin Rose	daf92be5bc	Better deploy test.	2019-08-18 17:54:52 -04:00
Calvin Rose	79bbb0ee1c	Appveyor test2.	2019-08-18 17:05:53 -04:00
Calvin Rose	826bb1abbe	Update appveyor deployment.	2019-08-18 16:54:43 -04:00
Calvin Rose	81789a6930	Add wasm to architectures returned by os/arch.	2019-08-18 10:08:52 -05:00
Calvin Rose	28fb2403d9	Add os/arch to core. Also allow setting custom keywords for compiled os name and architecture name.	2019-08-18 10:00:04 -05:00
Calvin Rose	1872bd344f	Address #158 Use string/join to prevent stack overflow.	2019-08-18 08:41:22 -05:00
Calvin Rose	54170d92db	Add some color to stacktraces in repl.	2019-08-12 19:20:01 -05:00