From 5b1a3b820834a152090d85bbd9104f54396d18ae Mon Sep 17 00:00:00 2001 From: Calvin Rose Date: Tue, 8 Jan 2019 19:59:54 -0500 Subject: [PATCH] Make grammar tool completely generate grammar from scratch. Remove grammar from source tree. --- Makefile | 6 ++- src/core/core.janet | 2 +- test/suite3.janet | 4 ++ .../tm_lang_gen.janet | 54 ++++++++++++++++++- tools/tmcorelib.janet | 31 ----------- 5 files changed, 63 insertions(+), 34 deletions(-) rename grammar/janet.tmLanguage => tools/tm_lang_gen.janet (70%) delete mode 100644 tools/tmcorelib.janet diff --git a/Makefile b/Makefile index 5af2a39e..7a6463b9 100644 --- a/Makefile +++ b/Makefile @@ -176,6 +176,10 @@ build/doc.html: $(JANET_TARGET) tools/gendoc.janet ##### Other ##### ################# +grammar: build/janet.tmLanguage +build/janet.tmLanguage: tools/tm_lang_gen.janet $(JANET_TARGET) + $(JANET_TARGET) $< > $@ + clean: -rm -rf build vgcore.* callgrind.* @@ -199,5 +203,5 @@ uninstall: $(LDCONFIG) .PHONY: clean install repl debug valgrind test \ - valtest emscripten dist uninstall docs \ + valtest emscripten dist uninstall docs grammar \ $(TEST_PROGRAM_PHONIES) $(TEST_PROGRAM_VALPHONIES) diff --git a/src/core/core.janet b/src/core/core.janet index 169961a5..853b71c1 100644 --- a/src/core/core.janet +++ b/src/core/core.janet @@ -1671,7 +1671,7 @@ value, one key will be ignored." (status-pp sig x f source)))) (run-context newenv chunks onsignal "repl")) -(defn all-symbols +(defn all-bindings "Get all symbols available in the current environment." [env &] (default env *env*) diff --git a/test/suite3.janet b/test/suite3.janet index 8477800a..d584c154 100644 --- a/test/suite3.janet +++ b/test/suite3.janet @@ -128,5 +128,9 @@ (assert-error "- check types" (- 1 ())) (assert-error "* check types" (* 1 ())) (assert-error "/ check types" (/ 1 ())) +(assert-error "band check types" (band 1 ())) +(assert-error "bor check types" (bor 1 ())) +(assert-error "bxor check types" (bxor 1 ())) +(assert-error "bnot check types" (bnot ())) (end-suite) diff --git a/grammar/janet.tmLanguage b/tools/tm_lang_gen.janet similarity index 70% rename from grammar/janet.tmLanguage rename to tools/tm_lang_gen.janet index d010811e..a706be3e 100644 --- a/grammar/janet.tmLanguage +++ b/tools/tm_lang_gen.janet @@ -1,3 +1,9 @@ +# Helper to generate core library mappings for janet +# Used to help build the tmLanguage grammar. Emits +# the entire .tmLanguage file for janet. + +(def grammar-template +````` @@ -216,7 +222,7 @@ corelib match - (?<![\.:\w_\-=!@\$%^&?|\\/<>*])(%|%=|\*|\*=|\*doc\-width\*|\*env\*|\+|\+\+|\+=|\-|\-\-|\-=|\->|\->>|\-\?>|\-\?>>|/|/=|<|<=|=|==|>|>=|_env|abstract\?|all|all\-symbols|allsyms|and|apply|array|array/concat|array/ensure|array/insert|array/new|array/peek|array/pop|array/push|array/slice|array\?|as\->|as\?\->|asm|band|blshift|bnot|boolean\?|bor|brshift|brushift|buffer|buffer/clear|buffer/new|buffer/popn|buffer/push\-byte|buffer/push\-string|buffer/push\-word|buffer/slice|buffer\?|bxor|bytes\?|callable\?|case|cfunction\?|comment|comp|compile|complement|cond|coro|count|debug|debug/arg\-stack|debug/break|debug/fbreak|debug/lineage|debug/stack|debug/unbreak|debug/unfbreak|dec|deep\-not=|deep=|def\-|default|defglobal|defmacro|defmacro\-|defn|defn\-|describe|dictionary\?|disasm|distinct|doc|doc\*|doc\-format|drop\-until|drop\-while|each|empty\?|env\-lookup|error|eval|eval\-string|even\?|every\?|extreme|false\?|fiber/current|fiber/maxstack|fiber/new|fiber/setmaxstack|fiber/status|fiber\?|file/close|file/flush|file/open|file/popen|file/read|file/seek|file/write|filter|find|find\-index|first|flatten|flatten\-into|for|frequencies|function\?|gccollect|gcinterval|gcsetinterval|generate|gensym|get|getline|hash|idempotent\?|identity|if\-let|if\-not|import|import\*|inc|indexed\?|interleave|interpose|invert|janet/build|janet/version|juxt|juxt\*|keep|keys|keyword|keyword\?|kvs|last|length|let|loop|macex|macex1|make\-env|map|mapcat|marshal|match|match\-1|math/acos|math/asin|math/atan|math/ceil|math/cos|math/e|math/exp|math/floor|math/inf|math/log|math/log10|math/pi|math/pow|math/random|math/seedrandom|math/sin|math/sqrt|math/tan|max|max\-order|merge|merge\-into|min|min\-order|module/find|module/native\-paths|module/paths|native|neg\?|next|nil\?|not|not=|not==|number\?|odd\?|one\?|or|order<|order<=|order>|order>=|os/clock|os/cwd|os/execute|os/exit|os/getenv|os/setenv|os/shell|os/sleep|os/time|os/which|pairs|parser/byte|parser/consume|parser/error|parser/flush|parser/has\-more|parser/new|parser/produce|parser/state|parser/status|parser/where|partial|pos\?|postwalk|prewalk|print|process/args|product|put|range|reduce|repl|require|resume|reverse|run\-context|scan\-number|sentinel|seq|some|sort|sorted|status\-pp|stderr|stdin|stdout|string|string/ascii\-lower|string/ascii\-upper|string/bytes|string/check\-set|string/find|string/find\-all|string/from\-bytes|string/join|string/number|string/pretty|string/repeat|string/replace|string/replace\-all|string/reverse|string/slice|string/split|string\?|struct|struct\?|sum|symbol|symbol\?|table|table/getproto|table/new|table/rawget|table/setproto|table/to\-struct|table\?|take\-until|take\-while|true\?|try|tuple|tuple/append|tuple/prepend|tuple/slice|tuple\?|type|unless|unmarshal|update|values|varglobal|walk|when|when\-let|with\-idemp|yield|zero\?|zipcoll)(?![\.:\w_\-=!@\$%^&?|\\/<>*]) + (?<![\.:\w_\-=!@\$%^&?|\\/<>*])(%ALLSYMBOLS%)(?![\.:\w_\-=!@\$%^&?|\\/<>*]) name keyword.control.janet @@ -333,3 +339,49 @@ 3743190f-20c4-44d0-8640-6611a983296b +`````) + +# Now we generate the bindings in the language. + +(def- specials + @["def" + "do" + "var" + "set" + "fn" + "while" + "if" + "quote" + "quasiquote" + "unquote"]) + +(def allsyms (array/concat @[] specials (all-bindings))) + +(def- escapes + {(get "|" 0) `\|` + (get "-" 0) `\-` + (get "+" 0) `\+` + (get "*" 0) `\*` + (get "^" 0) `\^` + (get "$" 0) `\$` + (get "?" 0) `\?` + 38 "&" + 60 "<" + 62 ">" + 34 """ + 39 "'" + 47 "/"}) + +(defn- escape + "Escape special characters for HTML and regex encoding." + [str] + (def buf @"") + (loop [byte :in str] + (if-let [rep (get escapes byte)] + (buffer/push-string buf rep) + (buffer/push-byte buf byte))) + buf) + +(def pattern (string/join (map escape allsyms) "|")) + +(print (string/replace "%ALLSYMBOLS%" pattern grammar-template)) diff --git a/tools/tmcorelib.janet b/tools/tmcorelib.janet deleted file mode 100644 index a9df827d..00000000 --- a/tools/tmcorelib.janet +++ /dev/null @@ -1,31 +0,0 @@ -# Helper to generate core library mappings for janet -# Used to help build the tmLanguage grammar. - -(def allsyms (all-symbols)) - -(def- escapes - {(get "|" 0) `\|` - (get "-" 0) `\-` - (get "+" 0) `\+` - (get "*" 0) `\*` - (get "^" 0) `\^` - (get "$" 0) `\$` - (get "?" 0) `\?` - 38 "&" - 60 "<" - 62 ">" - 34 """ - 39 "'" - 47 "/"}) - -(defn- escape - "Escape special characters for HTML and regex encoding." - [str] - (def buf @"") - (loop [byte :in str] - (if-let [rep (get escapes byte)] - (buffer/push-string buf rep) - (buffer/push-byte buf byte))) - buf) - -(print (string/join (map escape allsyms) "|"))