Make grammar tool completely generate grammar from scratch.

Remove grammar from source tree.

Makefile

@@ -176,6 +176,10 @@ build/doc.html: $(JANET_TARGET) tools/gendoc.janet
 ##### Other #####
 #################
 
+grammar: build/janet.tmLanguage
+build/janet.tmLanguage: tools/tm_lang_gen.janet $(JANET_TARGET)
+	$(JANET_TARGET) $< > $@
+
 clean:
 	-rm -rf build vgcore.* callgrind.*
 
@@ -199,5 +203,5 @@ uninstall:
 	$(LDCONFIG)
 
 .PHONY: clean install repl debug valgrind test \
-	valtest emscripten dist uninstall docs \
+	valtest emscripten dist uninstall docs grammar \
 	$(TEST_PROGRAM_PHONIES) $(TEST_PROGRAM_VALPHONIES)

src/core/core.janet

@@ -1671,7 +1671,7 @@ value, one key will be ignored."
                         (status-pp sig x f source))))
   (run-context newenv chunks onsignal "repl"))
 
-(defn all-symbols
+(defn all-bindings
   "Get all symbols available in the current environment."
   [env &]
   (default env *env*)

test/suite3.janet

@@ -128,5 +128,9 @@
 (assert-error "- check types" (- 1 ()))
 (assert-error "* check types" (* 1 ()))
 (assert-error "/ check types" (/ 1 ()))
+(assert-error "band check types" (band 1 ()))
+(assert-error "bor check types" (bor 1 ()))
+(assert-error "bxor check types" (bxor 1 ()))
+(assert-error "bnot check types" (bnot ()))
 
 (end-suite)

tools/tm_lang_gen.janet

@@ -1,3 +1,9 @@
+# Helper to generate core library mappings for janet
+# Used to help build the tmLanguage grammar. Emits
+# the entire .tmLanguage file for janet.
+
+(def grammar-template
+`````
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
@@ -216,7 +222,7 @@
 		<key>corelib</key>
 		<dict>
 			<key>match</key>
-            <string>(?&lt;![\.:\w_\-=!@\$%^&amp;?|\\/&lt;&gt;*])(%|%=|\*|\*=|\*doc\-width\*|\*env\*|\+|\+\+|\+=|\-|\-\-|\-=|\-&gt;|\-&gt;&gt;|\-\?&gt;|\-\?&gt;&gt;|&#47;|&#47;=|&lt;|&lt;=|=|==|&gt;|&gt;=|_env|abstract\?|all|all\-symbols|allsyms|and|apply|array|array&#47;concat|array&#47;ensure|array&#47;insert|array&#47;new|array&#47;peek|array&#47;pop|array&#47;push|array&#47;slice|array\?|as\-&gt;|as\?\-&gt;|asm|band|blshift|bnot|boolean\?|bor|brshift|brushift|buffer|buffer&#47;clear|buffer&#47;new|buffer&#47;popn|buffer&#47;push\-byte|buffer&#47;push\-string|buffer&#47;push\-word|buffer&#47;slice|buffer\?|bxor|bytes\?|callable\?|case|cfunction\?|comment|comp|compile|complement|cond|coro|count|debug|debug&#47;arg\-stack|debug&#47;break|debug&#47;fbreak|debug&#47;lineage|debug&#47;stack|debug&#47;unbreak|debug&#47;unfbreak|dec|deep\-not=|deep=|def\-|default|defglobal|defmacro|defmacro\-|defn|defn\-|describe|dictionary\?|disasm|distinct|doc|doc\*|doc\-format|drop\-until|drop\-while|each|empty\?|env\-lookup|error|eval|eval\-string|even\?|every\?|extreme|false\?|fiber&#47;current|fiber&#47;maxstack|fiber&#47;new|fiber&#47;setmaxstack|fiber&#47;status|fiber\?|file&#47;close|file&#47;flush|file&#47;open|file&#47;popen|file&#47;read|file&#47;seek|file&#47;write|filter|find|find\-index|first|flatten|flatten\-into|for|frequencies|function\?|gccollect|gcinterval|gcsetinterval|generate|gensym|get|getline|hash|idempotent\?|identity|if\-let|if\-not|import|import\*|inc|indexed\?|interleave|interpose|invert|janet&#47;build|janet&#47;version|juxt|juxt\*|keep|keys|keyword|keyword\?|kvs|last|length|let|loop|macex|macex1|make\-env|map|mapcat|marshal|match|match\-1|math&#47;acos|math&#47;asin|math&#47;atan|math&#47;ceil|math&#47;cos|math&#47;e|math&#47;exp|math&#47;floor|math&#47;inf|math&#47;log|math&#47;log10|math&#47;pi|math&#47;pow|math&#47;random|math&#47;seedrandom|math&#47;sin|math&#47;sqrt|math&#47;tan|max|max\-order|merge|merge\-into|min|min\-order|module&#47;find|module&#47;native\-paths|module&#47;paths|native|neg\?|next|nil\?|not|not=|not==|number\?|odd\?|one\?|or|order&lt;|order&lt;=|order&gt;|order&gt;=|os&#47;clock|os&#47;cwd|os&#47;execute|os&#47;exit|os&#47;getenv|os&#47;setenv|os&#47;shell|os&#47;sleep|os&#47;time|os&#47;which|pairs|parser&#47;byte|parser&#47;consume|parser&#47;error|parser&#47;flush|parser&#47;has\-more|parser&#47;new|parser&#47;produce|parser&#47;state|parser&#47;status|parser&#47;where|partial|pos\?|postwalk|prewalk|print|process&#47;args|product|put|range|reduce|repl|require|resume|reverse|run\-context|scan\-number|sentinel|seq|some|sort|sorted|status\-pp|stderr|stdin|stdout|string|string&#47;ascii\-lower|string&#47;ascii\-upper|string&#47;bytes|string&#47;check\-set|string&#47;find|string&#47;find\-all|string&#47;from\-bytes|string&#47;join|string&#47;number|string&#47;pretty|string&#47;repeat|string&#47;replace|string&#47;replace\-all|string&#47;reverse|string&#47;slice|string&#47;split|string\?|struct|struct\?|sum|symbol|symbol\?|table|table&#47;getproto|table&#47;new|table&#47;rawget|table&#47;setproto|table&#47;to\-struct|table\?|take\-until|take\-while|true\?|try|tuple|tuple&#47;append|tuple&#47;prepend|tuple&#47;slice|tuple\?|type|unless|unmarshal|update|values|varglobal|walk|when|when\-let|with\-idemp|yield|zero\?|zipcoll)(?![\.:\w_\-=!@\$%^&amp;?|\\/&lt;&gt;*])</string>
+            <string>(?&lt;![\.:\w_\-=!@\$%^&amp;?|\\/&lt;&gt;*])(%ALLSYMBOLS%)(?![\.:\w_\-=!@\$%^&amp;?|\\/&lt;&gt;*])</string>
 			<key>name</key>
 			<string>keyword.control.janet</string>
 		</dict>
@@ -333,3 +339,49 @@
     <string>3743190f-20c4-44d0-8640-6611a983296b</string>
 </dict>
 </plist>
+`````)
+
+# Now we generate the bindings in the language.
+
+(def- specials
+  @["def"
+    "do"
+    "var"
+    "set"
+    "fn"
+    "while"
+    "if"
+    "quote"
+    "quasiquote"
+    "unquote"])
+
+(def allsyms (array/concat @[] specials (all-bindings)))
+
+(def- escapes
+  {(get "|" 0) `\|`
+   (get "-" 0) `\-`
+   (get "+" 0) `\+`
+   (get "*" 0) `\*`
+   (get "^" 0) `\^`
+   (get "$" 0) `\$`
+   (get "?" 0) `\?`
+   38 "&amp;"
+   60 "&lt;"
+   62 "&gt;"
+   34 "&quot;"
+   39 "&#39;"
+   47 "&#47;"})
+
+(defn- escape
+  "Escape special characters for HTML and regex encoding."
+  [str]
+  (def buf @"")
+  (loop [byte :in str]
+    (if-let [rep (get escapes byte)]
+      (buffer/push-string buf rep)
+      (buffer/push-byte buf byte)))
+  buf)
+
+(def pattern (string/join (map escape allsyms) "|"))
+
+(print (string/replace "%ALLSYMBOLS%" pattern grammar-template))

tools/tmcorelib.janet

@@ -1,31 +0,0 @@
-# Helper to generate core library mappings for janet
-# Used to help build the tmLanguage grammar.
-
-(def allsyms (all-symbols))
-
-(def- escapes
-  {(get "|" 0) `\|`
-   (get "-" 0) `\-`
-   (get "+" 0) `\+`
-   (get "*" 0) `\*`
-   (get "^" 0) `\^`
-   (get "$" 0) `\$`
-   (get "?" 0) `\?`
-   38 "&"
-   60 "<"
-   62 ">"
-   34 """
-   39 "'"
-   47 "/"})
-
-(defn- escape
-  "Escape special characters for HTML and regex encoding."
-  [str]
-  (def buf @"")
-  (loop [byte :in str]
-    (if-let [rep (get escapes byte)]
-      (buffer/push-string buf rep)
-      (buffer/push-byte buf byte)))
-  buf)
-
-(print (string/join (map escape allsyms) "|"))