From 09a704ae7c08b3460cd7bd28f2c5aea063cf7860 Mon Sep 17 00:00:00 2001 From: "Aidan K. Ewart" Date: Tue, 2 Jun 2020 12:41:24 +0100 Subject: [PATCH] added execution tracing & rewrote tokenizer --- Caddyfile | 6 +- new/builtin.mjs | 154 ++++++++++++------------------ new/index.html | 29 +++++- new/main.js | 18 ++-- new/parse.mjs | 51 +--------- new/shiny.mjs | 25 +++-- new/token.mjs | 35 +++---- npm-debug.log | 96 +++++++++++++++++++ tracer/builtin.mjs | 128 +++++++++++++++++++++++++ tracer/index.html | 36 +++++++ tracer/main.js | 105 +++++++++++++++++++++ tracer/parse.mjs | 231 +++++++++++++++++++++++++++++++++++++++++++++ tracer/shiny.mjs | 154 ++++++++++++++++++++++++++++++ tracer/token.mjs | 57 +++++++++++ 14 files changed, 931 insertions(+), 194 deletions(-) create mode 100644 npm-debug.log create mode 100644 tracer/builtin.mjs create mode 100644 tracer/index.html create mode 100644 tracer/main.js create mode 100644 tracer/parse.mjs create mode 100644 tracer/shiny.mjs create mode 100644 tracer/token.mjs diff --git a/Caddyfile b/Caddyfile index 970f98b..ab5a623 100644 --- a/Caddyfile +++ b/Caddyfile @@ -1,9 +1,9 @@ -aidan-network.duckdns.org { - root ./old/ +aidanpe.duckdns.org { + root ./tracer/ file_server } -localhost:8443 { +rpn.aidanpe.duckdns.org { root ./new/ file_server } \ No newline at end of file diff --git a/new/builtin.mjs b/new/builtin.mjs index cfbd0ed..4999dcf 100644 --- a/new/builtin.mjs +++ b/new/builtin.mjs @@ -1,9 +1,36 @@ -import {defnOp, makeOp, defn, execRPN} from './shiny.mjs'; +import {defnOp, makeOp, defn} from './shiny.mjs'; import {parseExprs} from './parse.mjs'; import {tokenize} from './token.mjs'; +const objEq = (a, b) => { + if (typeof a !== 'object' && typeof b !== 'object') { + return a === b; + } else if (typeof a !== 'object' || typeof b !== 'object') { + return false; + } else { + const aprop = Object.getOwnPropertyNames(a); + const bprop = Object.getOwnPropertyNames(b); + + if (bprop.length !== aprop.length) { + return false; + } + + for (let i = 0; i < aprop.length; i++) { + if (!objEq(a[aprop[i]], b[aprop[i]])) { + return false; + } + } + + return true; + } +} + export let scope = {}; +export const customHandler = (ins) => { + return [ins]; +} + const addRPNDefn = (name, def) => { let toks = tokenize(def); if (!toks) { @@ -16,23 +43,6 @@ const addRPNDefn = (name, def) => { scope = defn(name, ast.parsed, scope); } -const ast = (def) => { - let toks = tokenize(def); - if (!toks) { - throw 'could not load builtin' - } - let ast = parseExprs(toks); - if (!ast.parsed) { - throw 'could not load builtin' - } - return execRPN({}, ast.parsed).stacks[0]; -} - -const ASTs = { - "true":ast("(a b -> b)"), - "false":ast("(a b -> b)"), -} - const assertType = (type) => (elem) => { if (elem.type !== type) { throw 'typeerror' @@ -66,93 +76,49 @@ const addDefn = (name, args, fn) => { } } -const add = (args) => { - return [{type:"int", val:args[0] + args[1]}]; -} - -const sub = (args) => { - return [{type:"int", val:args[1] - args[0]}]; -} - -const div = (args) => { - return [{type:"int", val:args[1] / args[0]}]; -} - -const mult = (args) => { - return [{type:"int", val:args[0] * args[1]}]; -} - -const pow = (args) => { - return [{type:"int", val:Math.pow(args[1], args[0])}]; -} - -const root = (args) => { - return [{type:"int", val:Math.sqrt(args[0])}]; -} - -const type = (args) => { - return [{type:"type", val:args[0].type}]; -} - -const pair = (args) => { - return [{type:"pair", val:{fst:args[0], snd:args[1]}}]; -} - +const add = (args) => [{type:"num", val:args[0] + args[1]}]; +const sub = (args) => [{type:"num", val:args[0] - args[1]}]; +const div = (args) => [{type:"num", val:args[0] / args[1]}]; +const mult = (args) => [{type:"num", val:args[0] * args[1]}]; +const pow = (args) => [{type:"num", val:Math.pow(args[0], args[1])}]; +const root = (args) => [{type:"num", val:Math.sqrt(args[0])}]; +const type = (args) => [{type:"type", val:args[0].type}]; +const pair = (args) => [{type:"pair", val:{fst:args[0], snd:args[1]}}]; const fst = (args) => [args[0].fst]; - const snd = (args) => [args[0].snd]; +const tuple = (args) => makeOp([...Array(args[0]).keys()], (args) => {return [{type:"tuple", val:args}]}); +const index = (args) => args[0][args[1]]; +const len = (args) => [{type:"num", val:args[0].length}]; const eq = (args) => { - if (args[0].type === args[1].type && args[0].val === args[1].val) { - console.log(args[0], args[1]) - return ASTs["true"]; + console.log(args[2], args[3]) + if (args[2].type === args[3].type && objEq(args[2].val, args[3].val)) { + return [args[0]]; } else { - return ASTs["false"]; + return [args[1]]; } } -const tuple = (args) => { - return [makeFn(args[0], (_, args) => {return [{type:"tuple", val:args}]})]; -} - -const index = (args) => { - return [args[0][args[1]]]; -} - -const len = (args) => { - return [{type:"int", val:args[0].length}]; -} - -const coerce = (args) => { - if (args[0].type === "type") { - let o = {type:args[1].val, val:args[0].val}; - return [o]; - } else { - throw 'typeerror' - } -} - -addDefn("+", ["int", "int"], add); -addDefn("-", ["int", "int"], sub); -addDefn("/", ["int", "int"], div); -addDefn("*", ["int", "int"], mult); -addDefn("^", ["int", "int"], pow); -addDefn("sqrt", ["int"], root); -addDefn("==", 2, eq); -addDefn("typeof", 1, type); +addDefn("+", ["num", "num"], add); +addDefn("-", ["num", "num"], sub); +addDefn("/", ["num", "num"], div); +addDefn("*", ["num", "num"], mult); +addDefn("^", ["num", "num"], pow); +addDefn("sqrt", ["num"], root); +addDefn("==", 4, eq); addDefn("pair", 2, pair); addDefn("fst", ["pair"], fst); addDefn("snd", ["pair"], snd); -addDefn("tuple", ["int"], tuple); -addDefn("!!", ["tuple", "int"], index); +addDefn("tuple", ["num"], tuple); +addDefn("!!", ["tuple", "num"], index); addDefn("len", ["tuple"], len); -addDefn("unsafeCoerce", 2, coerce); -//addRPNDefn("unit", "(-> 0 arr)"); -//addRPNDefn("mono", "(-> 1 arr)"); -//addRPNDefn("unwrap", "(-> 0 !!)"); -addRPNDefn("true", "(a b -> a)"); -addRPNDefn("false", "(a b -> b)"); +addDefn("typeof", 1, type); addRPNDefn("stop", "\"stop"); -//addRPNDefn("id", "(a -> a)"); addRPNDefn("inv", "(x -> 1 x /)"); -addRPNDefn("fold", "(x acc fn -> acc '(-> x acc fn 'fn fold) 'x \"stop ==)"); \ No newline at end of file +addRPNDefn("fold", "(x acc fn -> acc '(-> x acc fn 'fn fold) 'x stop ==)"); +addRPNDefn("range", "(x y -> x '(->x x 1 + y range) 'x y ==)"); +addRPNDefn("listthen", "(fn -> (internal; x acc -> '(->acc fn) '(->x acc pair internal) x stop ==) 0 tuple internal)"); +addRPNDefn("list", "'(a -> a) listthen"); +addRPNDefn("lmap", "(list fn -> list '(->list fst fn list snd 'fn lmap pair) list 0 tuple ==)"); +addRPNDefn("unlist", "(l -> (internal; list -> '(->) '(->list fst list snd internal) list 0 tuple ==) stop l internal)"); +addRPNDefn("map", "fn -> '(l->l 'fn lmap unlist) listthen"); \ No newline at end of file diff --git a/new/index.html b/new/index.html index 25cf256..8f27113 100644 --- a/new/index.html +++ b/new/index.html @@ -1,6 +1,33 @@ - + + 

 
+
Documentation

+
use (name; value) to define something. the definition can be recursive. value is executed and name is set to the final state of the stack, i.e. (name; 1 3) is possible

+
use ' to push instead of apply to the stack, e.g. '(a -> a). This is useful for lazyness, i.e. '(->lazy evaluated thing)

+
 
\ No newline at end of file
diff --git a/new/main.js b/new/main.js
index c233e00..e75a75e 100644
--- a/new/main.js
+++ b/new/main.js
@@ -1,27 +1,21 @@
 import {execRPN} from './shiny.mjs';
 import {parseExprs} from './parse.mjs';
 import {tokenize} from './token.mjs';
-import {scope} from './builtin.mjs';
+import {scope, customHandler} from './builtin.mjs';
 
 const inbox = document.getElementById("inbox")
 const outbox = document.getElementById("outbox")
 const submit = document.getElementById("submit")
 
 const show = (elem) => {
-    if (elem.type === "int") {
-        return elem.val
-    } else if (elem.type === "pair") {
+    if (elem.type === "pair") {
         return "{" + show(elem.val.fst) + ", " + show(elem.val.snd) + "}"
     } else if (elem.type === "closure") {
         return "(needs: " + elem.val.args.join(", ") + ")"
-    } else if (elem.type === "type") {
-        return elem.val
     } else if (elem.type === "array") {
         return "[" + prettyprint(elem.val) + "]"
-    } else if (elem.val) {
-        return "(" + elem.val + ": " + elem.type + ")"
     } else {
-        return elem.type
+        return "(" + elem.val + ": " + elem.type + ")"
     }
 }
 
@@ -44,11 +38,11 @@ submit.onclick = (event) => {
         return;
     }
     let ast = parseExprs(toks);
-    if (!ast.parsed) {
-        outbox.innerHTML = "incorrect syntax somewhere";
+    if (ast.stream.length !== 0) {
+        outbox.innerHTML = "unexpected token: " + ((e)=>{if(e.type==="ident"){return e.name}else{return e.val}})(ast.stream[0]);
         return;
     }
-    let out = execRPN(scope, ast.parsed);
+    let out = execRPN(scope, ast.parsed, customHandler);
     if (!out) {
         outbox.innerHTML = "failed to execute";
         return;
diff --git a/new/parse.mjs b/new/parse.mjs
index 7e2474a..b576f1a 100644
--- a/new/parse.mjs
+++ b/new/parse.mjs
@@ -1,46 +1,3 @@
-/*
-parse.js
---------
-
-Parse tokenstream into AST
-
-convert:
-[
-{type:"int",val:1},
-{type:"int",val:2},
-{type:"ident",name:"+"},
-{type:"syntax",val:"'"},
-{type:"syntax",val:"("},
-{type:"ident",name:"name"},
-{type:"syntax",val:";"},
-{type:"ident",name:"args"},
-{type:"syntax",val:"->"},
-{type:"int",val:2},
-{type:"ident",name:"args"},
-{type:"ident",name:"+"},
-{type:"syntax",val:")"}
-]
-
-(for c+p)
-[{type:"int",val:1},{type:"int",val:2},{type:"ident",name:"+"},{type:"syntax",val:"'"},{type:"syntax",val:"("},{type:"ident",name:"name"},{type:"ident",name:"args"},{type:"syntax",val:"->"},{type:"int",val:2},{type:"ident",name:"args"},{type:"ident",name:"+"},{type:"syntax",val:")"}]
-
-to:
-[
-    {type:"int", val:1},
-    {type:"int", val:2},
-    {type:"builtin", op:"+"},
-    {type:"push", elem:
-            {type:"func", args:["args"], body:[
-            {type:"int", val:1},
-            {type:"ident", val:"args"},
-            {type:"builtin", op:"+"},
-    }}]}}
-]
-
-EXPORTED FUNCTIONS:
-parseExprs - takes in tokenstream, outputs AST
-*/
-
 /* make parser safe */
 const attempt = (parser) => (stream) => {
     let streamclone = [...stream];
@@ -101,12 +58,12 @@ const parseIdent = (stream) => {
 }
 
 /* takes in stream, outputs parsed item or null */
-const parseInteger = (stream) => {
+const parseNum = (stream) => {
     let e = stream[0];
     if (e === undefined) {
         return {parsed:null, stream:stream};
     }
-    if (e.type !== "int") {
+    if (e.type !== "num") {
         return {parsed:null, stream:stream};
     } else {
         stream.shift();
@@ -149,7 +106,7 @@ const parseType = (stream) => {
     if (syn.parsed === null) {
         return {parsed:null, stream:syn.stream};
     }
-    let id = or(parseIdent, parseInteger)(syn.stream);
+    let id = or(parseIdent, parseNum)(syn.stream);
     if (id.parsed === null) {
         return {parsed:null, stream:id.stream};
     }
@@ -204,7 +161,7 @@ const parseExpr = or(
     attempt(parseLambda), or(
     parseType, or(
     parseIdent, or(
-    parseInteger,
+    parseNum,
     parsePush
     ))))));
 
diff --git a/new/shiny.mjs b/new/shiny.mjs
index d059b5e..e017ba5 100644
--- a/new/shiny.mjs
+++ b/new/shiny.mjs
@@ -65,7 +65,7 @@ const runFn = (defn, state) => {
         state.calls.push(defn.ins);
         state.stacks.push([]);
     } else if (defn.type === "builtin") {
-        let scope = state.scopes.pop();
+        let scope = state.scopes[state.scopes.length-1];
         let out = defn.fn(scope);
         state.calls.push([]);
         state.stacks.push(out);
@@ -99,7 +99,7 @@ const applyMany = (elems, state) => {
     }
 }
 
-const makeStackElems = (ins, state) => {
+const makeStackElems = (ins, state, handler) => {
     if (ins.type === "push") {
         throw 'nested push error'
     } else if (ins.type === "ident") {
@@ -107,21 +107,21 @@ const makeStackElems = (ins, state) => {
     } else if (ins.type === "func") {
         return [{type:"closure", val:{scope:state.scopes[state.scopes.length-1], args:ins.args, defn:{type:"ins", ins:ins.body}}}];
     } else {
-        return [ins];
+        return handler(ins);
     }
 }
 
-const doIns = (ins, state) => {
+const doIns = (ins, state, handler) => {
     if (ins.type === "push") {
-        state.stacks[state.stacks.length-1] = state.stacks[state.stacks.length-1].concat(makeStackElems(ins.elem, state));
+        state.stacks[state.stacks.length-1] = state.stacks[state.stacks.length-1].concat(makeStackElems(ins.elem, state, handler));
     } else if (ins.type === "defn") {
         state.scopes[state.scopes.length-1] = defn(ins.ident, ins.defn, state.scopes[state.scopes.length-1]);
     } else {
-        applyMany(makeStackElems(ins, state), state);
+        applyMany(makeStackElems(ins, state, handler), state);
     }
 }
 
-const step = (state) => {
+const step = (state, handler) => {
     if (state.calls[state.calls.length-1].length === 0) {
         if (state.calls.length === 1) {
             throw 'finished execution'
@@ -130,23 +130,20 @@ const step = (state) => {
             throw 'nothing to return'
         }
         state.calls.pop();
+        state.scopes.pop();
         let out = state.stacks.pop();
         applyMany(out, state);
     } else {
         let ins = state.calls[state.calls.length-1][0];
         state.calls[state.calls.length-1] = state.calls[state.calls.length-1].slice(1);
-        try {
-            doIns(ins, state);
-        } catch (error) {
-            throw error + ' while executing "' + ins.val + '"'
-        }
+        doIns(ins, state, handler);
     }
 }
 
-export const execRPN = (scope, ins) => {
+export const execRPN = (scope, ins, handler=(x)=>[x]) => {
     let state = {scopes:[scope], stacks:[[]], calls:[ins]};
     while (state.calls[0].length > 0 || state.calls.length > 1) {
-        step(state);
+        step(state, handler);
         if (state.stacks.length > 4096) {
             throw 'max recursion depth exceeded'
         }
diff --git a/new/token.mjs b/new/token.mjs
index 3e0a341..6276cd9 100644
--- a/new/token.mjs
+++ b/new/token.mjs
@@ -1,49 +1,38 @@
 export function tokenize(input) {
     let i;
     let inputWithAddedSpaces = "";
-    const syntax = /['";()]/;                                            //  -> is a special syntax (because it uses 2 characters) so is coded separately
+    const syntax = /['";()]/;
     for(i = 0; i"){
             if(input.charAt(i-1) != " "){
-                inputWithAddedSpaces += " ";                             //  adds a space
+                inputWithAddedSpaces += " ";
             }
             if(input.charAt(i+2) != " "){
-                inputWithAddedSpaces += "->" + " ";                      //  adds a space after the two characters
+                inputWithAddedSpaces += "->" + " ";
             } else {
-                inputWithAddedSpaces += "->";                            //  adds the two characters
+                inputWithAddedSpaces += "->";
             }
-        } else if(input.charAt(i) != ">") {                              //  if it is syntax, it was already detected at "-"
-            inputWithAddedSpaces += input.charAt(i);
-        }
-    }                                                                    //  i feel like this for loop is inefficient and could just add spaces around syntax all the time, so the following code has more writes to memory but may be more efficient. replace lines 5-27 with 29-37 by commenting out lines 5-27 and removing the /* and */ around lines 29-37. Note: i have not tested the code but it probably works.
-    /*
-    for(i = 0; i"){
-            inputWithAddedSpaces += " -> ";
-        } else {
+        } else if(input.charAt(i) != ">") {
             inputWithAddedSpaces += input.charAt(i);
         }
     }
-    */
     let splitInput = inputWithAddedSpaces.split(" ").filter((s) => s !== "");
     let output = [];
     for(i = 0; i
-        } else if(syntax.test(splitInput[i]) || splitInput[i] === "->"){//  needs a || as -> wasn't included in the syntax regexp. it wasn't in because -> uses two characters so i wanted to have separate code for it. (also because regexps are confusing)
+        if(!isNaN(splitInput[i])){
+            output.push({type: "num", val:Number(splitInput[i])});
+        } else if(syntax.test(splitInput[i]) || splitInput[i] === "->"){
             output.push({type: "syntax", val:splitInput[i]});
-        } else if(splitInput[i] != '') {                                //  if syntax is next to the end of the string or other bits of syntax the two spaces are in inputWithAddedSpaces so .split returns '' as one element. this makes sure that it is not read as an identifier
+        } else if(splitInput[i] != '') {
             output.push({type: "ident", name:splitInput[i]});
         }
     }
diff --git a/npm-debug.log b/npm-debug.log
new file mode 100644
index 0000000..2f67d2c
--- /dev/null
+++ b/npm-debug.log
@@ -0,0 +1,96 @@
+0 info it worked if it ends with ok
+1 verbose cli [ '/usr/bin/node', '/usr/bin/npm', 'i', 'rationals' ]
+2 info using npm@3.5.2
+3 info using node@v8.10.0
+4 verbose config Skipping project config: /home/aidan/.npmrc. (matches userconfig)
+5 silly loadCurrentTree Starting
+6 silly install loadCurrentTree
+7 silly install readLocalPackageData
+8 silly fetchPackageMetaData rationals
+9 silly fetchNamedPackageData rationals
+10 silly mapToRegistry name rationals
+11 silly mapToRegistry using default registry
+12 silly mapToRegistry registry https://registry.npmjs.org/
+13 silly mapToRegistry uri https://registry.npmjs.org/rationals
+14 verbose request uri https://registry.npmjs.org/rationals
+15 verbose request no auth needed
+16 info attempt registry request try #1 at 19:23:48
+17 verbose request id 1f9e752341fc452f
+18 verbose etag W/"e0fb616d3d75eb1a38936789fa99e693"
+19 verbose lastModified Sat, 12 Oct 2019 22:31:44 GMT
+20 http request GET https://registry.npmjs.org/rationals
+21 http 304 https://registry.npmjs.org/rationals
+22 verbose headers { date: 'Mon, 01 Jun 2020 18:23:48 GMT',
+22 verbose headers   connection: 'keep-alive',
+22 verbose headers   'set-cookie':
+22 verbose headers    [ '__cfduid=da1ea1de35cfdbd41bc4da4edd15edf091591035828; expires=Wed, 01-Jul-20 18:23:48 GMT; path=/; domain=.npmjs.org; HttpOnly; SameSite=Lax' ],
+22 verbose headers   'cf-ray': '59caf7cabb5ad210-MAN',
+22 verbose headers   age: '10',
+22 verbose headers   'cache-control': 'public, max-age=300',
+22 verbose headers   etag: '"e0fb616d3d75eb1a38936789fa99e693"',
+22 verbose headers   'last-modified': 'Sat, 12 Oct 2019 22:31:44 GMT',
+22 verbose headers   vary: 'Accept-Encoding',
+22 verbose headers   'cf-cache-status': 'HIT',
+22 verbose headers   'cf-request-id': '0312b932ae0000d2108792b200000001',
+22 verbose headers   'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"',
+22 verbose headers   server: 'cloudflare' }
+23 silly get cb [ 304,
+23 silly get   { date: 'Mon, 01 Jun 2020 18:23:48 GMT',
+23 silly get     connection: 'keep-alive',
+23 silly get     'set-cookie':
+23 silly get      [ '__cfduid=da1ea1de35cfdbd41bc4da4edd15edf091591035828; expires=Wed, 01-Jul-20 18:23:48 GMT; path=/; domain=.npmjs.org; HttpOnly; SameSite=Lax' ],
+23 silly get     'cf-ray': '59caf7cabb5ad210-MAN',
+23 silly get     age: '10',
+23 silly get     'cache-control': 'public, max-age=300',
+23 silly get     etag: '"e0fb616d3d75eb1a38936789fa99e693"',
+23 silly get     'last-modified': 'Sat, 12 Oct 2019 22:31:44 GMT',
+23 silly get     vary: 'Accept-Encoding',
+23 silly get     'cf-cache-status': 'HIT',
+23 silly get     'cf-request-id': '0312b932ae0000d2108792b200000001',
+23 silly get     'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"',
+23 silly get     server: 'cloudflare' } ]
+24 verbose etag https://registry.npmjs.org/rationals from cache
+25 verbose get saving rationals to /home/aidan/.npm/registry.npmjs.org/rationals/.cache.json
+26 silly install normalizeTree
+27 silly loadCurrentTree Finishing
+28 silly loadIdealTree Starting
+29 silly install loadIdealTree
+30 silly cloneCurrentTree Starting
+31 silly install cloneCurrentTreeToIdealTree
+32 silly cloneCurrentTree Finishing
+33 silly loadShrinkwrap Starting
+34 silly install loadShrinkwrap
+35 silly loadShrinkwrap Finishing
+36 silly loadAllDepsIntoIdealTree Starting
+37 silly install loadAllDepsIntoIdealTree
+38 verbose stack Error: Missing required argument #1
+38 verbose stack     at andLogAndFinish (/usr/share/npm/lib/fetch-package-metadata.js:31:3)
+38 verbose stack     at fetchPackageMetadata (/usr/share/npm/lib/fetch-package-metadata.js:51:22)
+38 verbose stack     at resolveWithNewModule (/usr/share/npm/lib/install/deps.js:456:12)
+38 verbose stack     at /usr/share/npm/lib/install/deps.js:190:5
+38 verbose stack     at /usr/share/npm/node_modules/slide/lib/async-map.js:52:35
+38 verbose stack     at Array.forEach ()
+38 verbose stack     at /usr/share/npm/node_modules/slide/lib/async-map.js:52:11
+38 verbose stack     at Array.forEach ()
+38 verbose stack     at asyncMap (/usr/share/npm/node_modules/slide/lib/async-map.js:51:8)
+38 verbose stack     at exports.loadRequestedDeps (/usr/share/npm/lib/install/deps.js:188:3)
+39 verbose cwd /home/aidan/Desktop/js-projects/rpncalc-v4
+40 error Linux 5.3.0-53-generic
+41 error argv "/usr/bin/node" "/usr/bin/npm" "i" "rationals"
+42 error node v8.10.0
+43 error npm  v3.5.2
+44 error code EMISSINGARG
+45 error typeerror Error: Missing required argument #1
+45 error typeerror     at andLogAndFinish (/usr/share/npm/lib/fetch-package-metadata.js:31:3)
+45 error typeerror     at fetchPackageMetadata (/usr/share/npm/lib/fetch-package-metadata.js:51:22)
+45 error typeerror     at resolveWithNewModule (/usr/share/npm/lib/install/deps.js:456:12)
+45 error typeerror     at /usr/share/npm/lib/install/deps.js:190:5
+45 error typeerror     at /usr/share/npm/node_modules/slide/lib/async-map.js:52:35
+45 error typeerror     at Array.forEach ()
+45 error typeerror     at /usr/share/npm/node_modules/slide/lib/async-map.js:52:11
+45 error typeerror     at Array.forEach ()
+45 error typeerror     at asyncMap (/usr/share/npm/node_modules/slide/lib/async-map.js:51:8)
+45 error typeerror     at exports.loadRequestedDeps (/usr/share/npm/lib/install/deps.js:188:3)
+46 error typeerror This is an error with npm itself. Please report this error at:
+46 error typeerror     
+47 verbose exit [ 1, true ]
diff --git a/tracer/builtin.mjs b/tracer/builtin.mjs
new file mode 100644
index 0000000..0c5ca5a
--- /dev/null
+++ b/tracer/builtin.mjs
@@ -0,0 +1,128 @@
+import {defnOp, makeOp, defn} from './shiny.mjs';
+import {parseExprs} from './parse.mjs';
+import {tokenize} from './token.mjs';
+
+const objEq = (a, b) => {
+    if (typeof a !== 'object' && typeof b !== 'object') {
+        return a === b;
+    } else if (typeof a !== 'object' || typeof b !== 'object') {
+        return false;
+    } else {
+        const aprop = Object.getOwnPropertyNames(a);
+        const bprop = Object.getOwnPropertyNames(b);
+
+        if (bprop.length !== aprop.length) {
+            return false;
+        }
+
+        for (let i = 0; i < aprop.length; i++) {
+            if (!objEq(a[aprop[i]], b[aprop[i]])) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+}
+
+export let scope = {};
+
+export const customHandler = (ins) => {
+    return [ins];
+}
+
+const addRPNDefn = (name, def) => {
+    let toks = tokenize(def);
+    if (!toks) {
+        throw 'could not load builtin'
+    }
+    toks = toks.map(elem => {
+        elem.startPos = 0;
+        elem.endPos = 0;
+        return elem;
+    });
+    let ast = parseExprs(toks);
+    if (!ast.parsed) {
+        throw 'could not load builtin'
+    }
+    scope = defn(name, ast.parsed.arr, scope);
+}
+
+const assertType = (type) => (elem) => {
+    if (elem.type !== type) {
+        throw 'typeerror'
+    }
+}
+
+const addDefn = (name, args, fn) => {
+    if (Array.isArray(args)) {
+        const nargs = [...Array(args.length).keys()];
+        const liftFn = (scope) => {
+            let newscope = Object.create(scope);
+            for (let i = 0; i < args.length; i++) {
+                assertType(args[i])(scope[i][0]);
+                newscope[i] = scope[i][0].val;
+            }
+            return fn(newscope);
+        }
+        const op = makeOp(nargs, liftFn);
+        defnOp(name, op);
+    } else {
+        const nargs = [...Array(args).keys()];
+        const liftFn = (scope) => {
+            let newscope = Object.create(scope);
+            for (let i = 0; i < args; i++) {
+                newscope[i] = scope[i][0];
+            }
+            return fn(newscope);
+        }
+        const op = makeOp(nargs, liftFn);
+        defnOp(name, op);
+    }
+}
+
+const add = (args) => [{type:"num", val:args[0] + args[1]}];
+const sub = (args) => [{type:"num", val:args[0] - args[1]}];
+const div = (args) => [{type:"num", val:args[0] / args[1]}];
+const mult = (args) => [{type:"num", val:args[0] * args[1]}];
+const pow = (args) => [{type:"num", val:Math.pow(args[0], args[1])}];
+const root = (args) => [{type:"num", val:Math.sqrt(args[0])}];
+const type = (args) => [{type:"type", val:args[0].type}];
+const pair = (args) => [{type:"pair", val:{fst:args[0], snd:args[1]}}];
+const fst = (args) => [args[0].fst];
+const snd = (args) => [args[0].snd];
+const tuple = (args) => makeOp([...Array(args[0]).keys()], (args) => {return [{type:"tuple", val:args}]});
+const index = (args) => args[0][args[1]];
+const len = (args) => [{type:"num", val:args[0].length}];
+
+const eq = (args) => {
+    if (args[2].type === args[3].type && objEq(args[2].val, args[3].val)) {
+        return [args[0]];
+    } else {
+        return [args[1]];
+    }
+}
+
+addDefn("+", ["num", "num"], add);
+addDefn("-", ["num", "num"], sub);
+addDefn("/", ["num", "num"], div);
+addDefn("*", ["num", "num"], mult);
+addDefn("^", ["num", "num"], pow);
+addDefn("sqrt", ["num"], root);
+addDefn("==", 4, eq);
+addDefn("pair", 2, pair);
+addDefn("fst", ["pair"], fst);
+addDefn("snd", ["pair"], snd);
+addDefn("tuple", ["num"], tuple);
+addDefn("!!", ["tuple", "num"], index);
+addDefn("len", ["tuple"], len);
+addDefn("typeof", 1, type);
+addRPNDefn("stop", "\"stop");
+addRPNDefn("inv", "(x -> 1 x /)");
+addRPNDefn("fold", "(x acc fn -> acc '(-> x acc fn 'fn fold) 'x stop ==)");
+addRPNDefn("range", "(x y -> x '(->x x 1 + y range) 'x y ==)");
+addRPNDefn("listthen", "(fn -> (internal; x acc -> '(->acc fn) '(->x acc pair internal) x stop ==) 0 tuple internal)");
+addRPNDefn("list", "'(a -> a) listthen");
+addRPNDefn("lmap", "(list fn -> list '(->list fst fn list snd 'fn lmap pair) list 0 tuple ==)");
+addRPNDefn("unlist", "(l -> (internal; list -> '(->) '(->list fst list snd internal) list 0 tuple ==) stop l internal)");
+addRPNDefn("map", "fn -> '(l->l 'fn lmap unlist) listthen");
\ No newline at end of file
diff --git a/tracer/index.html b/tracer/index.html
new file mode 100644
index 0000000..d5690b3
--- /dev/null
+++ b/tracer/index.html
@@ -0,0 +1,36 @@
+
+
+
+
+
+
+

+

+
+
Documentation

+
use (name; value) to define something. the definition can be recursive. value is executed and name is set to the final state of the stack, i.e. (name; 1 3) is possible

+
use ' to push instead of apply to the stack, e.g. '(a -> a). This is useful for lazyness, i.e. '(->lazy evaluated thing)

+
    
+
  • +, -, *, /, ^, sqrt: mathematical operations
    • 
+
  • ==: equality (automatically derived for all types); returns a b -> a if true, a b -> b if false
    • 
+
  • typeof: returns the type of the object
    • 
+
  • pair, fst, snd: pairs two objects, gets first or second item of pair
    • 
+
  • tuple: used like ... 3 tuple; creates an n tuple of n items on the stack
    • 
+
  • !!: index into a tuple
    • 
+
  • len: length of a tuple
    • 
+
  • unsafeCoerce: unsafely transforms one type into another. useage: 1 "type unsafeCoerce
    • 
+
  • true, false: true/false used in ==
    • 
+
  • stop: equivalent to "stop
    • 
+
  • inv: equivalent to 1 x /
    • 
+
  • fold: equivalent to x acc fn -> acc '(-> x acc fn 'fn fold) 'x \"stop ==
    • 
+

+
\ No newline at end of file
diff --git a/tracer/main.js b/tracer/main.js
new file mode 100644
index 0000000..6fc1414
--- /dev/null
+++ b/tracer/main.js
@@ -0,0 +1,105 @@
+import {execRPN, step} from './shiny.mjs';
+import {parseExprs} from './parse.mjs';
+import {tokenize} from './token.mjs';
+import {scope, customHandler} from './builtin.mjs';
+
+//const scope = {};
+//const customHandler = a => [a];
+
+const inbox = document.getElementById("inbox")
+const insbox = document.getElementById("insbox")
+const outbox = document.getElementById("outbox")
+const stepb = document.getElementById("step")
+const play = document.getElementById("play")
+const load = document.getElementById("load")
+
+let state = null;
+let input = null;
+
+const highlight = (str, start, end, color) => {
+    return str.slice(0, start) + `` + str.slice(start, end) + "" + str.slice(end);
+}
+
+const loadState = () => {
+    input = inbox.value;
+    let toks = tokenize(input);
+    let ast = parseExprs(toks);
+    console.log(ast);
+    if (ast.parsed === null) {
+        insbox.innerHTML = "could not parse AST!"
+    } else if (ast.stream.length !== 0) {
+        insbox.innerHTML = highlight(input, ast.stream[0].startPos, ast.stream[0].endPos, "red");
+        insbox.innerHTML += "
unexpected token"
+        input = null;
+        state = null;
+        return;
+    }
+    insbox.innerHTML = input;
+    outbox.innerHTML = "";
+    state = {scopes:[scope], stacks:[[]], calls:[ast.parsed.arr]};
+}
+
+load.onclick = _ => {
+    loadState();
+}
+
+play.onclick = _ => {
+    if (state === null) {
+        loadState();
+    }
+    insbox.innerHTML = "";
+    while (state.calls[0].length > 0 || state.calls.length > 1) {
+        step(state, customHandler);
+        if (state.stacks.length > 4096) {
+            insbox.innerHTML = "max recursion depth exceeded"
+            return;
+        }
+    }
+    outbox.innerHTML = prettyprint(state.stacks[0]);
+    state = null;
+    input = null;
+}
+
+stepb.onclick = _ => {
+    if (state === null) {
+        return;
+    }
+    if (state.calls[0].length > 0 || state.calls.length > 1) {
+        let pos = step(state, customHandler);
+        if (!(pos.start === 0 && pos.end === 0)) {
+            insbox.innerHTML = highlight(input, pos.start, pos.end, "green");
+        }
+        if (state.stacks.length > 1) {
+            outbox.innerHTML = "... " + prettyprint(state.stacks[state.stacks.length-1]);
+        } else {
+            outbox.innerHTML = prettyprint(state.stacks[0]);
+        }
+        if (state.stacks.length > 4096) {
+            insbox.innerHTML = "max recursion depth exceeded"
+            return;
+        }
+    }
+}
+
+const show = (elem) => {
+    if (elem.type === "pair") {
+        return "{" + show(elem.val.fst) + ", " + show(elem.val.snd) + "}"
+    } else if (elem.type === "closure") {
+        return "(needs: " + elem.val.args.join(", ") + ")"
+    } else if (elem.type === "array") {
+        return "[" + prettyprint(elem.val) + "]"
+    } else {
+        return "(" + elem.val + ": " + elem.type + ")"
+    }
+}
+
+const prettyprint = (out) => {
+    let str = "";
+    for (let i = 0; i < out.length; i++) {
+        str += show(out[i]);
+        if (i < out.length - 1) {
+            str += " ";
+        }
+    }
+    return str;
+}
\ No newline at end of file
diff --git a/tracer/parse.mjs b/tracer/parse.mjs
new file mode 100644
index 0000000..2d2ddfe
--- /dev/null
+++ b/tracer/parse.mjs
@@ -0,0 +1,231 @@
+/* make parser safe */
+const debug = false;
+
+const attempt = (parser) => (stream) => {
+    if (debug) {
+        console.log("attempt");
+        console.log(stream);
+    }
+    let streamclone = [...stream];
+    try {
+        let out = parser(stream);
+        return out;
+    } catch(err) {
+        return {parsed:null,stream:streamclone};
+    }
+}
+
+/* chain */
+const or = (a, b) => (stream) => {
+    let streamclone = [...stream];
+    if (debug) {
+        console.log("or");
+        console.log(stream);
+    }
+    let aout = a(stream);
+    if (aout.parsed === null) {
+        return b(streamclone);
+    } else {
+        return aout;
+    }
+}
+
+/* (parser) */
+const parens = (parser) => (stream) => {
+    if (debug) {
+        console.log("parens");
+        console.log(stream);
+    }
+    let a = parseSyntax("(")(stream);
+    if (a.parsed === null) {
+        return {parsed:null, stream:a.stream};
+    }
+    let dat = parser(a.stream);
+    let b = parseSyntax(")")(dat.stream);
+    if (b.parsed === null) {
+        throw 'mismatched parens!';
+    }
+    dat.parsed.pos.start = a.parsed.pos.start;
+    dat.parsed.pos.end = b.parsed.pos.end;
+    return {parsed:dat.parsed, stream:b.stream};
+}
+
+/* [parser] */
+const many = (parser) => (stream) => {
+    if (debug) {
+        console.log("many");
+        console.log(stream);
+    }
+    let parsed = [];
+    while (true) {
+        let i = parser(stream);
+        stream = i.stream;
+        if (i.parsed === null) {
+            break;
+        } else {
+            parsed.push(i.parsed);
+        }
+    }
+    let startPos = 0;
+    let endPos = 0;
+    if (parsed.length > 1) {
+        startPos = parsed[0].pos.start;
+        endPos = parsed[parsed.length-1].pos.start;
+    }
+    return {parsed:{arr:parsed, pos:{start:startPos, end:endPos}}, stream:stream};
+}
+
+/* takes in stream, outputs parsed item or null */
+const parseIdent = (stream) => {
+    if (debug) {
+        console.log("ident");
+        console.log(stream);
+    }
+    let e = stream[0];
+    if (e === undefined) {
+        return {parsed:null, stream:stream};
+    }
+    if (e.type !== "ident") {
+        return {parsed:null, stream:stream};
+    } else {
+        stream.shift();
+        return {parsed:{type:e.type, val:e.name, pos:{start:e.startPos, end:e.endPos}}, stream:stream};
+    }
+}
+
+/* takes in stream, outputs parsed item or null */
+const parseNum = (stream) => {
+    if (debug) {
+        console.log("num");
+        console.log(stream);
+    }
+    let e = stream[0];
+    if (e === undefined) {
+        return {parsed:null, stream:stream};
+    }
+    if (e.type !== "num") {
+        return {parsed:null, stream:stream};
+    } else {
+        stream.shift();
+        return {parsed:{type:e.type, val:e.val, pos:{start:e.startPos, end:e.endPos}}, stream:stream};
+    }
+}
+
+/* takes in stream, outputs parsed item or null */
+const parseSyntax = (syntax) => (stream) => {
+    if (debug) {
+        console.log("syntax", syntax);
+        console.log(stream);
+    }
+    let e = stream[0];
+    if (e === undefined) {
+        return {parsed:null, stream:stream};
+    }
+    if (e.type !== "syntax") {
+        return {parsed:null, stream:stream};
+    }
+    if (e.val !== syntax) {
+        return {parsed:null, stream:stream};
+    } else {
+        stream.shift();
+        return {parsed:{type:"syntax", val:syntax, pos:{start:e.startPos, end:e.endPos}}, stream:stream};
+    }
+}
+
+/* takes in stream, outputs string or null - FAILABLE */
+const parseName = (stream) => {
+    if (debug) {
+        console.log("name");
+        console.log(stream);
+    }
+    let id = parseIdent(stream);
+    if (id.parsed === null) {
+        return {parsed:null, stream:id.stream};
+    }
+    let syn = parseSyntax(";")(id.stream);
+    if (syn.parsed === null) {
+        throw 'could not parse name!'
+    }
+    return {parsed:{name:id.parsed.val, pos:{start:id.parsed.pos.start, end:syn.parsed.pos.end}}, stream:syn.stream};
+}
+
+const parseType = (stream) => {
+    if (debug) {
+        console.log("type");
+        console.log(stream);
+    }
+    let syn = attempt(parseSyntax("\""))(stream);
+    if (syn.parsed === null) {
+        return {parsed:null, stream:syn.stream};
+    }
+    let id = or(parseIdent, parseNum)(syn.stream);
+    if (id.parsed === null) {
+        return {parsed:null, stream:id.stream};
+    }
+    return {parsed:{type:"type", val:id.parsed.val, pos:{start:syn.parsed.pos.start, end:id.parsed.pos.end}}, stream:id.stream};
+}
+
+/* takes in stream, outputs parsed item or null - FAILABLE */
+const parsePush = (stream) => {
+    if (debug) {
+        console.log("push");
+        console.log(stream);
+    }
+    let syn = attempt(parseSyntax("'"))(stream);
+    if (syn.parsed === null) {
+        return {parsed:null, stream:syn.stream};
+    }
+    let id = parseExpr(syn.stream);
+    if (id.parsed === null) {
+        return {parsed:null, stream:id.stream};
+    }
+    return {parsed:{type:"push", elem:id.parsed, pos:{start:syn.parsed.pos.start, end:id.parsed.pos.end}}, stream:id.stream};
+}
+
+const parseDefn = (stream) => {
+    if (debug) {
+        console.log("defn");
+        console.log(stream);
+    }
+    let name = parseName(stream);
+    if (name.parsed === null) {
+        throw 'no name found!'
+    }
+    let expr = parseExprs(stream);
+    if (expr.parsed === null) {
+        throw 'no body found!'
+    }
+    return {parsed:{type:"defn", ident:name.parsed.name, defn:expr.parsed.arr, pos:{start:name.parsed.pos.start, end:expr.parsed.pos.end}}, stream:expr.stream}
+}
+
+/* takes in stream, outputs parsed item or null - FAILABLE */
+const parseLambda = (stream) => {
+    if (debug) {
+        console.log("lambda");
+        console.log(stream);
+    }
+    let args = many(parseIdent)(stream);
+    let syn = parseSyntax("->")(args.stream);
+    if (syn.parsed === null) {
+        throw 'no lambda body found!';
+    }
+    let body = parseExprs(syn.stream); // .parsed should never be null, but anyway...
+    if (body.parsed === null) {
+        throw 'no lambda body found!';
+    }
+    return {parsed:{type:"func", args:args.parsed.arr.map(x => x.val), body:body.parsed.arr, pos:{start:args.parsed.pos.start, end:body.parsed.pos.end}}, stream:body.stream};
+}
+
+/* takes in stream, outputs parsed item or null */
+const parseExpr = or(
+    attempt(parens(parseDefn)), or(
+    attempt(parens(parseLambda)), or(
+    attempt(parseLambda), or(
+    parseType, or(
+    parseIdent, or(
+    parseNum,
+    parsePush
+    ))))));
+
+/* takes in stream, outputs parsed items */
+export const parseExprs = many(parseExpr);
\ No newline at end of file
diff --git a/tracer/shiny.mjs b/tracer/shiny.mjs
new file mode 100644
index 0000000..0bb01e4
--- /dev/null
+++ b/tracer/shiny.mjs
@@ -0,0 +1,154 @@
+/*
+EVAL.js
+-------
+
+reads exprs from AST and executes them
+
+types of elem:
+all have type
+all have val
+
+TYPE        VAL
+"closure"   {scope:{}, args:['x', 'y'], defn:}
+
+:
+{type:"ins", ins:[]}
+{type:"builtin", fn:(scope) => {stack}}
+
+exported functionality:
+defnOp(name, function) - define a built-in operator
+defn(name, ins, scope) - extend scope with AST
+makeOp(name, args, fn) - lift a function to an operator
+evalRPN(scope, ast)
+*/
+
+let builtins = {};
+
+export const defnOp = (name, data) => {
+    builtins[name] = data;
+}
+
+export const defn = (name, ins, scope) => {
+    let defscope = Object.create(scope);
+    let fnForm = {type:"closure", val:{scope:defscope, args:[], defn:{type:"ins", ins:ins}}};
+    defscope[name] = [fnForm];
+    let out = execRPN(defscope, ins);
+    defscope[name] = out.stacks[0];
+    return defscope;
+}
+
+export const makeOp = (args, fn) => {
+    return [{type:"closure", val:{scope:{}, args:args, defn:{type:"builtin", fn:fn}}}];
+}
+
+const lookup = (name, scope) => {
+    let n = scope[name];
+    if (n) {
+        return n;
+    }
+    n = builtins[name];
+    if (n) {
+        return n;
+    }
+    console.log(scope);
+    throw '"' + name + '" not in scope'
+}
+
+const extend = (scope, name, elems) => {
+    let o = Object.create(scope);
+    o[name] = elems;
+    return o;
+}
+
+const runFn = (defn, state) => {
+    if (defn.type === "ins") {
+        state.calls.push(defn.ins);
+        state.stacks.push([]);
+    } else if (defn.type === "builtin") {
+        let scope = state.scopes[state.scopes.length-1];
+        let out = defn.fn(scope);
+        state.calls.push([]);
+        state.stacks.push(out);
+    }
+}
+
+const giveArg = (arg, elem) => {
+    let argN = elem.val.args[elem.val.args.length-1];
+    let newscope = extend(elem.val.scope, argN, [arg]);
+    return {type:elem.type, val:{scope:newscope, args:elem.val.args.slice(0,-1), defn:elem.val.defn}};
+}
+
+const apply = (elem, state) => {
+    if (elem.type === "closure") {
+        if (elem.val.args.length === 0) {
+            state.scopes.push(elem.val.scope);
+            runFn(elem.val.defn, state);
+        } else if (state.stacks[state.stacks.length-1].length > 0) {
+            apply(giveArg(state.stacks[state.stacks.length-1].pop(), elem), state);
+        } else {
+            state.stacks[state.stacks.length-1].push(elem);
+        }
+    } else {
+        state.stacks[state.stacks.length-1].push(elem);
+    }
+}
+
+const applyMany = (elems, state) => {
+    for (let i = 0; i < elems.length; i++) {
+        apply(elems[i], state);
+    }
+}
+
+const makeStackElems = (ins, state, handler) => {
+    if (ins.type === "push") {
+        throw 'nested push error'
+    } else if (ins.type === "ident") {
+        return lookup(ins.val, state.scopes[state.scopes.length-1]);
+    } else if (ins.type === "func") {
+        return [{type:"closure", val:{scope:state.scopes[state.scopes.length-1], args:ins.args, defn:{type:"ins", ins:ins.body}}}];
+    } else {
+        return handler(ins);
+    }
+}
+
+const doIns = (ins, state, handler) => {
+    if (ins.type === "push") {
+        state.stacks[state.stacks.length-1] = state.stacks[state.stacks.length-1].concat(makeStackElems(ins.elem, state, handler));
+    } else if (ins.type === "defn") {
+        state.scopes[state.scopes.length-1] = defn(ins.ident, ins.defn, state.scopes[state.scopes.length-1]);
+    } else {
+        applyMany(makeStackElems(ins, state, handler), state);
+    }
+}
+
+export const step = (state, handler) => {
+    if (state.calls[state.calls.length-1].length === 0) {
+        if (state.calls.length === 1) {
+            throw 'finished execution'
+        }
+        if (state.stacks.length < 2) {
+            throw 'nothing to return'
+        }
+        state.calls.pop();
+        state.scopes.pop();
+        let out = state.stacks.pop();
+        applyMany(out, state);
+        return {start:0, end:0};
+    } else {
+        let ins = state.calls[state.calls.length-1][0];
+        state.calls[state.calls.length-1] = state.calls[state.calls.length-1].slice(1);
+        doIns(ins, state, handler);
+        return ins.pos;
+    }
+}
+
+export const execRPN = (scope, ins, handler=(x)=>[x]) => {
+    let state = {scopes:[scope], stacks:[[]], calls:[ins]};
+    while (state.calls[0].length > 0 || state.calls.length > 1) {
+        step(state, handler);
+        if (state.stacks.length > 4096) {
+            throw 'max recursion depth exceeded'
+        }
+    }
+    return state;
+}
\ No newline at end of file
diff --git a/tracer/token.mjs b/tracer/token.mjs
new file mode 100644
index 0000000..b1c0ad9
--- /dev/null
+++ b/tracer/token.mjs
@@ -0,0 +1,57 @@
+const tokens = (stream) => {
+    let toks = [];
+    let currTok = {startPos:0, val:"", type:"str", endPos:0};
+    for (let i = 0; i < stream.length; i++) {
+        if ("()';\"".includes(stream[i])) {
+            if (currTok.val !== "") {
+                currTok.endPos = i;
+                toks.push(currTok);
+            }
+            toks.push({startPos:i, endPos:i+1, val:stream[i], type:"syntax"});
+            currTok = {startPos:i+1, val:"", type:"str"};
+        } else if (stream[i] === "-") {
+            if (stream[i+1] === ">") {
+                if (currTok.val !== "") {
+                    currTok.endPos = i;
+                    toks.push(currTok);
+                }
+                toks.push({startPos:i, endPos:i+1, val:"->", type:"syntax"});
+                i++;
+                currTok = {startPos:i+1, val:"", type:"str"};
+            } else {
+                currTok.val += "-";
+            }
+        } else if (/\s/.test(stream[i])) {
+            if (currTok.val !== "") {
+                currTok.endPos = i;
+                toks.push(currTok);
+            }
+            currTok = {startPos:i+1, val:"", type:"str"};
+        } else {
+            currTok.val += stream[i];
+        }
+    }
+    if (currTok.val !== "") {
+        currTok.endPos = stream.length;
+        toks.push(currTok);
+    }
+    return toks;
+}
+
+const classify = (tokens) => {
+    return tokens.map(tok => {
+        if (tok.type === "str") {
+            if (!isNaN(tok.val)) {
+                return {startPos:tok.startPos, endPos:tok.endPos, val:Number(tok.val), type:"num"};
+            } else {
+                return {startPos:tok.startPos, endPos:tok.endPos, name:tok.val, type:"ident"};
+            }
+        } else {
+            return tok;
+        }
+    });
+}
+
+export const tokenize = (stream) => {
+    return classify(tokens(stream));
+}
\ No newline at end of file