/* please also see token.c */
enum tokentype { KW, STRING, VAR, NUM, OP, NEWLINE };
enum kwtype { IF, THEN, ELSIF, ELSE, WHILE, ENDKWTYPE };

struct token {
	union {
		const char *string, *var;
		int64_t num;
		char op;
		enum kwtype kw;
	} val;
	const char *filename;
	int col, line, pos;
	enum tokentype type;
};

struct tokenstate {
	/*
	 * tokens
	 * |
	 * |_> | ptr to 1024 struct token | ... |
	 */
	struct token **tokens;
	const char *filename;
	FILE *fh;
	size_t tokensz, tokenblk, tokenind; /* see get_next_token */
	int col, line, pos, look;
};

extern int tokenize(struct tokenstate *);