From 95cba6f712cad5ea4e3a920bcc325be7ec768235 Mon Sep 17 00:00:00 2001
From: Baidicoot <aidanprattewart@protonmail.com>
Date: Mon, 25 May 2020 14:39:42 +0100
Subject: [PATCH] Add files via upload

---
 token.js | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 token.js

diff --git a/token.js b/token.js
new file mode 100644
index 0000000..80914bb
--- /dev/null
+++ b/token.js
@@ -0,0 +1,51 @@
+function tokenize(input) {
+    var i;
+    var inputWithAddedSpaces = "";
+    const syntax = /[`;()]/;                                             //  -> is a special syntax (because it uses 2 characters) so is coded separately
+    for(i = 0; i<input.length; i++){
+        if(syntax.test(input.charAt(i))){
+            if(input.charAt(i-1) != " "){
+                inputWithAddedSpaces += " ";                             //  adds a space
+            }
+            if(input.charAt(i+1) != " "){
+                inputWithAddedSpaces += input.charAt(i) + " ";           //  adds a space after the character
+            } else {
+                inputWithAddedSpaces += input.charAt(i);                 //  adds the character
+            }
+        } else if(input.charAt(i) === "-" && input.charAt(i+1) === ">"){
+            if(input.charAt(i-1) != " "){
+                inputWithAddedSpaces += " ";                             //  adds a space
+            }
+            if(input.charAt(i+2) != " "){
+                inputWithAddedSpaces += "->" + " ";                      //  adds a space after the two characters
+            } else {
+                inputWithAddedSpaces += "->";                            //  adds the two characters
+            }
+        } else if(input.charAt(i) != ">") {                             //  if it is syntax, it was already detected at "-"
+            inputWithAddedSpaces += input.charAt(i);
+        }
+    }                                                                   //  i feel like this for loop is inefficient and could just add spaces around syntax all the time, so the following code has more writes to memory but may be more efficient. replace lines 5-27 with 29-37 by commenting out lines 5-27 and removing the /* and */ around lines 29-37. Note: i have not tested the code but it probably works.
+    /*
+    for(i = 0; i<input.length; i++){
+        if(syntax.test(input.charAt(i))){
+            inputWithAddedSpaces += " " + input.charAt(i) + " ";
+        } else if(input.charAt(i) === "-" && input.charAt(i+1) === ">"){
+            inputWithAddedSpaces += " -> ";
+        } else {
+            inputWithAddedSpaces += input.charAt(i);
+        }
+    }
+    */
+    splitInput = inputWithAddedSpaces.split(" ");
+    var output = [];
+    for(i = 0; i<splitInput.length; i++){
+        if(parseInt(splitInput[i])){                                    //  didn't need /[0-9]/, but that would be helpful to stop numbers from being in identifiers, but the ability to call a function "function_add_1" is easier to type than "function_add_one"
+            output.push({type: "int", val:splitInput[i]});               //      also, /[a-zA-Z]/ wasn't necessary as my code uses anything that isn't `;() or ->
+        } else if(syntax.test(splitInput[i]) || splitInput[i] === "->"){//  needs a || as -> wasn't included in the syntax regexp. it wasn't in because -> uses two characters so i wanted to have separate code for it. (also because regexps are confusing)
+            output.push({type: "syntax", val:splitInput[i]});
+        } else if(splitInput[i] != '') {                                //  if syntax is next to the end of the string or other bits of syntax the two spaces are in inputWithAddedSpaces so .split returns '' as one element. this makes sure that it is not read as an identifier
+            output.push({type: "ident", name:splitInput[i]});
+        }
+    }
+    return(output)
+}
\ No newline at end of file