Fix EOF bug and add token printing to main.c test()

main.c

@@ -1,4 +1,4 @@
-#include <stdint.h>
+#include <inttypes.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include "token.h"
@@ -31,6 +31,29 @@ test(void)
 	ts.pos = 0;
 
 	tokenize(&ts);
+
+	/* TODO do not enter more than 1024 tokens into this */
+	for (int i = 0; i < ts.tokenind; i++) {
+		struct token *token = &ts.tokens[0][i];
+		printf("type: %i, ", (int)token->type);
+		switch (token->type) {
+		case KW:
+			printf("val: %s\n", token->val.kw);
+			break;
+		case STRING:
+			printf("val: \"%s\"\n", token->val.string);
+			break;
+		case VAR:
+			printf("val: %s\n", token->val.var);
+			break;
+		case NUM:
+			printf("val: %" PRId64 "\n", token->val.num);
+			break;
+		case OP:
+			printf("val: %c\n", token->val.op);
+			break;
+		}
+	}
 }
 
 int

token.c

@@ -35,6 +35,16 @@ getch(struct tokenstate *ts)
 	}
 }
 
+/**
+ * Skips whitespace in the input.
+ */
+static inline void
+skip_whitespace(struct tokenstate *ts)
+{
+	while (isspace(ts->look))
+		getch(ts);
+}
+
 /**
  * Returns a pointer to where the next token should be put.
  * In struct tokenstate, the field tokens is an array of
@@ -238,6 +248,10 @@ tokenize(struct tokenstate *ts)
 	getch(ts);
 
 	while (ts->look != 0) {
+		skip_whitespace(ts);
+		if (ts->look == 0)
+			break;
+
 		/**
 		 * Get a pointer to the next struct token and set
 		 * information used across all cases.