2020-05-30 19:05:05 +00:00
|
|
|
export function tokenize(input) {
|
|
|
|
let i;
|
|
|
|
let inputWithAddedSpaces = "";
|
2020-06-02 11:41:24 +00:00
|
|
|
const syntax = /['";()]/;
|
2020-05-25 13:39:42 +00:00
|
|
|
for(i = 0; i<input.length; i++){
|
|
|
|
if(syntax.test(input.charAt(i))){
|
|
|
|
if(input.charAt(i-1) != " "){
|
2020-06-02 11:41:24 +00:00
|
|
|
inputWithAddedSpaces += " ";
|
2020-05-25 13:39:42 +00:00
|
|
|
}
|
|
|
|
if(input.charAt(i+1) != " "){
|
2020-06-02 11:41:24 +00:00
|
|
|
inputWithAddedSpaces += input.charAt(i) + " ";
|
2020-05-25 13:39:42 +00:00
|
|
|
} else {
|
2020-06-02 11:41:24 +00:00
|
|
|
inputWithAddedSpaces += input.charAt(i);
|
2020-05-25 13:39:42 +00:00
|
|
|
}
|
|
|
|
} else if(input.charAt(i) === "-" && input.charAt(i+1) === ">"){
|
|
|
|
if(input.charAt(i-1) != " "){
|
2020-06-02 11:41:24 +00:00
|
|
|
inputWithAddedSpaces += " ";
|
2020-05-25 13:39:42 +00:00
|
|
|
}
|
|
|
|
if(input.charAt(i+2) != " "){
|
2020-06-02 11:41:24 +00:00
|
|
|
inputWithAddedSpaces += "->" + " ";
|
2020-05-25 13:39:42 +00:00
|
|
|
} else {
|
2020-06-02 11:41:24 +00:00
|
|
|
inputWithAddedSpaces += "->";
|
2020-05-25 13:39:42 +00:00
|
|
|
}
|
2020-06-02 11:41:24 +00:00
|
|
|
} else if(input.charAt(i) != ">") {
|
2020-05-25 13:39:42 +00:00
|
|
|
inputWithAddedSpaces += input.charAt(i);
|
|
|
|
}
|
|
|
|
}
|
2020-05-30 21:39:45 +00:00
|
|
|
let splitInput = inputWithAddedSpaces.split(" ").filter((s) => s !== "");
|
2020-05-30 19:05:05 +00:00
|
|
|
let output = [];
|
2020-05-25 13:39:42 +00:00
|
|
|
for(i = 0; i<splitInput.length; i++){
|
2020-06-02 11:41:24 +00:00
|
|
|
if(!isNaN(splitInput[i])){
|
|
|
|
output.push({type: "num", val:Number(splitInput[i])});
|
|
|
|
} else if(syntax.test(splitInput[i]) || splitInput[i] === "->"){
|
2020-05-25 13:39:42 +00:00
|
|
|
output.push({type: "syntax", val:splitInput[i]});
|
2020-06-02 11:41:24 +00:00
|
|
|
} else if(splitInput[i] != '') {
|
2020-05-25 13:39:42 +00:00
|
|
|
output.push({type: "ident", name:splitInput[i]});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return(output)
|
|
|
|
}
|