--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/tokenizer.h	Tue May 13 14:03:10 2008 -0700
@@ -0,0 +1,53 @@
+/*
+
+Copyright (c) 2007 Carl Byington - 510 Software Group, released under
+the GPL version 3 or any later version at your choice available at
+http://www.gnu.org/licenses/gpl-3.0.txt
+
+*/
+
+
+using namespace std;
+
+struct ltstr {
+	bool operator()(char* s1, char* s2) const {
+		return strcmp(s1, s2) < 0;
+	}
+};
+
+typedef list<ifstream *>	stream_list;
+typedef list<char *>		string_list;
+typedef set<char *, ltstr>	string_set;
+typedef list<int>			line_list;
+
+class TOKEN {
+	stream_list streams;
+	string_list filenames;
+	string_set	filenamess;
+	line_list	linenumbers;
+	string_list pending_tokens;
+	string_set	*include_files;
+	bool		pushed;
+	u_char		pushed_char;
+
+	void pop();
+	bool next_char(u_char &c);
+	void push_char(u_char c);
+
+public:
+	TOKEN(char *fn, string_set *includes);
+	~TOKEN();
+	bool		include(char *fn);
+	char		*next();			// return next token
+	int 		nextint();
+	void		skipeol();			// skip to eol
+	void		push(char *token)			{pending_tokens.push_front(token);};
+	const char	*cur_fn()					{return filenames.empty()	? "" : filenames.front();};
+	int 		cur_line()					{return linenumbers.empty() ? 0  : linenumbers.front();};
+	void		token_error(const char *err);
+	void		token_error(const char *fmt, int d, const char *s);
+	void		token_error(const char *fmt, const char *t, const char *h);
+	void		token_error(const char *want, const char *have);
+	void		token_error();
+};
+