--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/tokenizer.h	Wed May 15 13:15:59 2013 -0700
@@ -0,0 +1,52 @@
+/*
+
+Copyright (c) 2007 Carl Byington - 510 Software Group, released under
+the GPL version 3 or any later version at your choice available at
+http://www.gnu.org/licenses/gpl-3.0.txt
+
+*/
+
+using namespace std;
+
+struct ltstr {
+    bool operator()(const char* s1, const char* s2) const {
+        return strcmp(s1, s2) < 0;
+    }
+};
+
+typedef list<ifstream *>            stream_list;
+typedef list<const char *>          string_list;
+typedef set<const char *, ltstr>    string_set;
+typedef list<int>                   line_list;
+
+class TOKEN {
+    stream_list streams;
+    string_list filenames;
+    string_set  filenamess;
+    line_list   linenumbers;
+    string_list pending_tokens;
+    string_set  *include_files;
+    bool        pushed;
+    u_char      pushed_char;
+
+    void pop();
+    bool next_char(u_char &c);
+    void push_char(u_char c);
+
+public:
+    TOKEN(const char *fn, string_set *includes);
+    ~TOKEN();
+    bool        include(const char *fn);
+    const char  *next();            // return next token
+    int         nextint();
+    void        skipeol();          // skip to eol
+    void        push(const char *token) {pending_tokens.push_front(token);};
+    const char  *cur_fn()               {return filenames.empty()   ? "" : filenames.front();};
+    int         cur_line()              {return linenumbers.empty() ? 0  : linenumbers.front();};
+    void        token_error(const char *err);
+    void        token_error(const char *fmt, int d, const char *s);
+    void        token_error(const char *fmt, const char *t, const char *h);
+    void        token_error(const char *want, const char *have);
+    void        token_error();
+};
+