Mercurial > wflogs-daemon

comparison src/tokenizer.cpp @ 0:0aa1171aebd2 stable-1-0-0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
initial version
author Carl Byington <carl@five-ten-sg.com>
date Wed, 15 May 2013 13:15:59 -0700
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:0aa1171aebd2
1 /*
2
3 Copyright (c) 2007 Carl Byington - 510 Software Group, released under
4 the GPL version 3 or any later version at your choice available at
5 http://www.gnu.org/licenses/gpl-3.0.txt
6
7 */
8
9 // This version of the tokenizer does not force the config to lower
10 // case, to avoid lowercasing the iptables commands, which need some
11 // uppercase arguments. It also considers / to be a separate token
12 // since that is needed for the cidr style ignore statement.
13
14 #include "includes.h"
15
16 const int maxlen = 1000; // used for snprintf buffers
17
18 enum state {s_init,
19 s_token,
20 s_string,
21 s_ignore, // whitespace
22 s_eol, // ignore to eol
23 end_state,
24
25 s_term, // token terminator
26 s_single,
27 s_string1, // first " of string
28 s_string2, // last " of string
29 s_slash // possible start of ignore to eol
30 };
31
32 typedef state PARSE[end_state];
33
34 static PARSE parse_table[256] = {
35 // s_init s_token s_string s_ignore s_eol
36 { s_single, s_term, s_string, s_single, s_eol, }, // 0x00
37 { s_single, s_term, s_string, s_single, s_eol, }, // 0x01
38 { s_single, s_term, s_string, s_single, s_eol, }, // 0x02
39 { s_single, s_term, s_string, s_single, s_eol, }, // 0x03
40 { s_single, s_term, s_string, s_single, s_eol, }, // 0x04
41 { s_single, s_term, s_string, s_single, s_eol, }, // 0x05
42 { s_single, s_term, s_string, s_single, s_eol, }, // 0x06
43 { s_single, s_term, s_string, s_single, s_eol, }, // 0x07
44 { s_single, s_term, s_string, s_single, s_eol, }, // 0x08
45 { s_ignore, s_term, s_string, s_ignore, s_eol, }, // 0x09 <tab>
46 { s_ignore, s_term, s_string2, s_ignore, s_ignore, }, // 0x0a <lf>
47 { s_single, s_term, s_string, s_single, s_eol, }, // 0x0b
48 { s_single, s_term, s_string, s_single, s_eol, }, // 0x0c
49 { s_ignore, s_term, s_string2, s_ignore, s_eol, }, // 0x0d <cr>
50 { s_single, s_term, s_string, s_single, s_eol, }, // 0x0e
51 { s_single, s_term, s_string, s_single, s_eol, }, // 0x0f
52 { s_single, s_term, s_string, s_single, s_eol, }, // 0x10
53 { s_single, s_term, s_string, s_single, s_eol, }, // 0x11 xon char
54 { s_single, s_term, s_string, s_single, s_eol, }, // 0x12
55 { s_single, s_term, s_string, s_single, s_eol, }, // 0x13 xoff char
56 { s_single, s_term, s_string, s_single, s_eol, }, // 0x14
57 { s_single, s_term, s_string, s_single, s_eol, }, // 0x15
58 { s_single, s_term, s_string, s_single, s_eol, }, // 0x16
59 { s_single, s_term, s_string, s_single, s_eol, }, // 0x17
60 { s_single, s_term, s_string, s_single, s_eol, }, // 0x18
61 { s_single, s_term, s_string, s_single, s_eol, }, // 0x19
62 { s_single, s_term, s_string, s_single, s_eol, }, // 0x1a
63 { s_single, s_term, s_string, s_single, s_eol, }, // 0x1b
64 { s_single, s_term, s_string, s_single, s_eol, }, // 0x1c
65 { s_single, s_term, s_string, s_single, s_eol, }, // 0x1d
66 { s_single, s_term, s_string, s_single, s_eol, }, // 0x1e
67 { s_single, s_term, s_string, s_single, s_eol, }, // 0x1f
68 { s_ignore, s_term, s_string, s_ignore, s_eol, }, // 0x20 space
69 { s_single, s_term, s_string, s_single, s_eol, }, // 0x21 !
70 { s_string1, s_term, s_string2, s_string1, s_eol, }, // 0x22 "
71 { s_eol, s_term, s_string, s_eol, s_eol, }, // 0x23 #
72 { s_single, s_term, s_string, s_single, s_eol, }, // 0x24 $
73 { s_single, s_term, s_string, s_single, s_eol, }, // 0x25 %
74 { s_single, s_term, s_string, s_single, s_eol, }, // 0x26 &
75 { s_single, s_term, s_string, s_single, s_eol, }, // 0x27 '
76 { s_single, s_term, s_string, s_single, s_eol, }, // 0x28 (
77 { s_single, s_term, s_string, s_single, s_eol, }, // 0x29 )
78 { s_single, s_term, s_string, s_single, s_eol, }, // 0x2A *
79 { s_single, s_token, s_string, s_single, s_eol, }, // 0x2B +
80 { s_single, s_term, s_string, s_single, s_eol, }, // 0x2C ,
81 { s_single, s_token, s_string, s_single, s_eol, }, // 0x2D -
82 { s_single, s_token, s_string, s_single, s_eol, }, // 0x2E .
83 { s_slash, s_term, s_string, s_slash, s_eol, }, // 0x2F /
84 { s_token, s_token, s_string, s_token, s_eol, }, // 0x30 0
85 { s_token, s_token, s_string, s_token, s_eol, }, // 0x31 1
86 { s_token, s_token, s_string, s_token, s_eol, }, // 0x32 2
87 { s_token, s_token, s_string, s_token, s_eol, }, // 0x33 3
88 { s_token, s_token, s_string, s_token, s_eol, }, // 0x34 4
89 { s_token, s_token, s_string, s_token, s_eol, }, // 0x35 5
90 { s_token, s_token, s_string, s_token, s_eol, }, // 0x36 6
91 { s_token, s_token, s_string, s_token, s_eol, }, // 0x37 7
92 { s_token, s_token, s_string, s_token, s_eol, }, // 0x38 8
93 { s_token, s_token, s_string, s_token, s_eol, }, // 0x39 9
94 { s_single, s_term, s_string, s_single, s_eol, }, // 0x3A :
95 { s_single, s_term, s_string, s_single, s_eol, }, // 0x3B ;
96 { s_single, s_term, s_string, s_single, s_eol, }, // 0x3C <
97 { s_single, s_token, s_string, s_single, s_eol, }, // 0x3D =
98 { s_single, s_term, s_string, s_single, s_eol, }, // 0x3E >
99 { s_single, s_term, s_string, s_single, s_eol, }, // 0x3F ?
100 { s_single, s_token, s_string, s_single, s_eol, }, // 0x40 @
101 { s_token, s_token, s_string, s_token, s_eol, }, // 0x41 A
102 { s_token, s_token, s_string, s_token, s_eol, }, // 0x42 B
103 { s_token, s_token, s_string, s_token, s_eol, }, // 0x43 C
104 { s_token, s_token, s_string, s_token, s_eol, }, // 0x44 D
105 { s_token, s_token, s_string, s_token, s_eol, }, // 0x45 E
106 { s_token, s_token, s_string, s_token, s_eol, }, // 0x46 F
107 { s_token, s_token, s_string, s_token, s_eol, }, // 0x47 G
108 { s_token, s_token, s_string, s_token, s_eol, }, // 0x48 H
109 { s_token, s_token, s_string, s_token, s_eol, }, // 0x49 I
110 { s_token, s_token, s_string, s_token, s_eol, }, // 0x4A J
111 { s_token, s_token, s_string, s_token, s_eol, }, // 0x4B K
112 { s_token, s_token, s_string, s_token, s_eol, }, // 0x4C L
113 { s_token, s_token, s_string, s_token, s_eol, }, // 0x4D M
114 { s_token, s_token, s_string, s_token, s_eol, }, // 0x4E N
115 { s_token, s_token, s_string, s_token, s_eol, }, // 0x4F O
116 { s_token, s_token, s_string, s_token, s_eol, }, // 0x50 P
117 { s_token, s_token, s_string, s_token, s_eol, }, // 0x51 Q
118 { s_token, s_token, s_string, s_token, s_eol, }, // 0x52 R
119 { s_token, s_token, s_string, s_token, s_eol, }, // 0x53 S
120 { s_token, s_token, s_string, s_token, s_eol, }, // 0x54 T
121 { s_token, s_token, s_string, s_token, s_eol, }, // 0x55 U
122 { s_token, s_token, s_string, s_token, s_eol, }, // 0x56 V
123 { s_token, s_token, s_string, s_token, s_eol, }, // 0x57 W
124 { s_token, s_token, s_string, s_token, s_eol, }, // 0x58 X
125 { s_token, s_token, s_string, s_token, s_eol, }, // 0x59 Y
126 { s_token, s_token, s_string, s_token, s_eol, }, // 0x5A Z
127 { s_single, s_term, s_string, s_single, s_eol, }, // 0x5B [
128 { s_single, s_term, s_string, s_single, s_eol, }, // 0x5C backslash
129 { s_single, s_term, s_string, s_single, s_eol, }, // 0x5D ]
130 { s_single, s_term, s_string, s_single, s_eol, }, // 0x5E ^
131 { s_single, s_token, s_string, s_single, s_eol, }, // 0x5F _
132 { s_single, s_term, s_string, s_single, s_eol, }, // 0x60 `
133 { s_token, s_token, s_string, s_token, s_eol, }, // 0x61 a
134 { s_token, s_token, s_string, s_token, s_eol, }, // 0x62 b
135 { s_token, s_token, s_string, s_token, s_eol, }, // 0x63 c
136 { s_token, s_token, s_string, s_token, s_eol, }, // 0x64 d
137 { s_token, s_token, s_string, s_token, s_eol, }, // 0x65 e
138 { s_token, s_token, s_string, s_token, s_eol, }, // 0x66 f
139 { s_token, s_token, s_string, s_token, s_eol, }, // 0x67 g
140 { s_token, s_token, s_string, s_token, s_eol, }, // 0x68 h
141 { s_token, s_token, s_string, s_token, s_eol, }, // 0x69 i
142 { s_token, s_token, s_string, s_token, s_eol, }, // 0x6A j
143 { s_token, s_token, s_string, s_token, s_eol, }, // 0x6B k
144 { s_token, s_token, s_string, s_token, s_eol, }, // 0x6C l
145 { s_token, s_token, s_string, s_token, s_eol, }, // 0x6D m
146 { s_token, s_token, s_string, s_token, s_eol, }, // 0x6E n
147 { s_token, s_token, s_string, s_token, s_eol, }, // 0x6F o
148 { s_token, s_token, s_string, s_token, s_eol, }, // 0x70 p
149 { s_token, s_token, s_string, s_token, s_eol, }, // 0x71 q
150 { s_token, s_token, s_string, s_token, s_eol, }, // 0x72 r
151 { s_token, s_token, s_string, s_token, s_eol, }, // 0x73 s
152 { s_token, s_token, s_string, s_token, s_eol, }, // 0x74 t
153 { s_token, s_token, s_string, s_token, s_eol, }, // 0x75 u
154 { s_token, s_token, s_string, s_token, s_eol, }, // 0x76 v
155 { s_token, s_token, s_string, s_token, s_eol, }, // 0x77 w
156 { s_token, s_token, s_string, s_token, s_eol, }, // 0x78 x
157 { s_token, s_token, s_string, s_token, s_eol, }, // 0x79 y
158 { s_token, s_token, s_string, s_token, s_eol, }, // 0x7A z
159 { s_single, s_term, s_string, s_single, s_eol, }, // 0x7B {
160 { s_single, s_term, s_string, s_single, s_eol, }, // 0x7C |
161 { s_single, s_term, s_string, s_single, s_eol, }, // 0x7D }
162 { s_single, s_term, s_string, s_single, s_eol, }, // 0x7E ~
163 { s_single, s_term, s_string, s_single, s_eol, }, // 0x7f
164 { s_single, s_term, s_string, s_single, s_eol, }, // 0x80
165 { s_single, s_term, s_string, s_single, s_eol, }, // 0x81
166 { s_single, s_term, s_string, s_single, s_eol, }, // 0x82
167 { s_single, s_term, s_string, s_single, s_eol, }, // 0x83
168 { s_single, s_term, s_string, s_single, s_eol, }, // 0x84
169 { s_single, s_term, s_string, s_single, s_eol, }, // 0x85
170 { s_single, s_term, s_string, s_single, s_eol, }, // 0x86
171 { s_single, s_term, s_string, s_single, s_eol, }, // 0x87
172 { s_single, s_term, s_string, s_single, s_eol, }, // 0x88
173 { s_single, s_term, s_string, s_single, s_eol, }, // 0x89
174 { s_single, s_term, s_string, s_single, s_eol, }, // 0x8a
175 { s_single, s_term, s_string, s_single, s_eol, }, // 0x8b
176 { s_single, s_term, s_string, s_single, s_eol, }, // 0x8c
177 { s_single, s_term, s_string, s_single, s_eol, }, // 0x8d
178 { s_single, s_term, s_string, s_single, s_eol, }, // 0x8e
179 { s_single, s_term, s_string, s_single, s_eol, }, // 0x8f
180 { s_single, s_term, s_string, s_single, s_eol, }, // 0x90
181 { s_single, s_term, s_string, s_single, s_eol, }, // 0x91
182 { s_single, s_term, s_string, s_single, s_eol, }, // 0x92
183 { s_single, s_term, s_string, s_single, s_eol, }, // 0x93
184 { s_single, s_term, s_string, s_single, s_eol, }, // 0x94
185 { s_single, s_term, s_string, s_single, s_eol, }, // 0x95
186 { s_single, s_term, s_string, s_single, s_eol, }, // 0x96
187 { s_single, s_term, s_string, s_single, s_eol, }, // 0x97
188 { s_single, s_term, s_string, s_single, s_eol, }, // 0x98
189 { s_single, s_term, s_string, s_single, s_eol, }, // 0x99
190 { s_single, s_term, s_string, s_single, s_eol, }, // 0x9a
191 { s_single, s_term, s_string, s_single, s_eol, }, // 0x9b
192 { s_single, s_term, s_string, s_single, s_eol, }, // 0x9c
193 { s_single, s_term, s_string, s_single, s_eol, }, // 0x9d
194 { s_single, s_term, s_string, s_single, s_eol, }, // 0x9e
195 { s_single, s_term, s_string, s_single, s_eol, }, // 0x9f
196 { s_single, s_term, s_string, s_single, s_eol, }, // 0xa0
197 { s_single, s_term, s_string, s_single, s_eol, }, // 0xa1
198 { s_single, s_term, s_string, s_single, s_eol, }, // 0xa2
199 { s_single, s_term, s_string, s_single, s_eol, }, // 0xa3
200 { s_single, s_term, s_string, s_single, s_eol, }, // 0xa4
201 { s_single, s_term, s_string, s_single, s_eol, }, // 0xa5
202 { s_single, s_term, s_string, s_single, s_eol, }, // 0xa6
203 { s_single, s_term, s_string, s_single, s_eol, }, // 0xa7
204 { s_single, s_term, s_string, s_single, s_eol, }, // 0xa8
205 { s_single, s_term, s_string, s_single, s_eol, }, // 0xa9
206 { s_single, s_term, s_string, s_single, s_eol, }, // 0xaa
207 { s_single, s_term, s_string, s_single, s_eol, }, // 0xab
208 { s_single, s_term, s_string, s_single, s_eol, }, // 0xac
209 { s_single, s_term, s_string, s_single, s_eol, }, // 0xad
210 { s_single, s_term, s_string, s_single, s_eol, }, // 0xae
211 { s_single, s_term, s_string, s_single, s_eol, }, // 0xaf
212 { s_single, s_term, s_string, s_single, s_eol, }, // 0xb0
213 { s_single, s_term, s_string, s_single, s_eol, }, // 0xb1
214 { s_single, s_term, s_string, s_single, s_eol, }, // 0xb2
215 { s_single, s_term, s_string, s_single, s_eol, }, // 0xb3
216 { s_single, s_term, s_string, s_single, s_eol, }, // 0xb4
217 { s_single, s_term, s_string, s_single, s_eol, }, // 0xb5
218 { s_single, s_term, s_string, s_single, s_eol, }, // 0xb6
219 { s_single, s_term, s_string, s_single, s_eol, }, // 0xb7
220 { s_single, s_term, s_string, s_single, s_eol, }, // 0xb8
221 { s_single, s_term, s_string, s_single, s_eol, }, // 0xb9
222 { s_single, s_term, s_string, s_single, s_eol, }, // 0xba
223 { s_single, s_term, s_string, s_single, s_eol, }, // 0xbb
224 { s_single, s_term, s_string, s_single, s_eol, }, // 0xbc
225 { s_single, s_term, s_string, s_single, s_eol, }, // 0xbd
226 { s_single, s_term, s_string, s_single, s_eol, }, // 0xbe
227 { s_single, s_term, s_string, s_single, s_eol, }, // 0xbf
228 { s_single, s_term, s_string, s_single, s_eol, }, // 0xc0
229 { s_single, s_term, s_string, s_single, s_eol, }, // 0xc1
230 { s_single, s_term, s_string, s_single, s_eol, }, // 0xc2
231 { s_single, s_term, s_string, s_single, s_eol, }, // 0xc3
232 { s_single, s_term, s_string, s_single, s_eol, }, // 0xc4
233 { s_single, s_term, s_string, s_single, s_eol, }, // 0xc5
234 { s_single, s_term, s_string, s_single, s_eol, }, // 0xc6
235 { s_single, s_term, s_string, s_single, s_eol, }, // 0xc7
236 { s_single, s_term, s_string, s_single, s_eol, }, // 0xc8
237 { s_single, s_term, s_string, s_single, s_eol, }, // 0xc9
238 { s_single, s_term, s_string, s_single, s_eol, }, // 0xca
239 { s_single, s_term, s_string, s_single, s_eol, }, // 0xcb
240 { s_single, s_term, s_string, s_single, s_eol, }, // 0xcc
241 { s_single, s_term, s_string, s_single, s_eol, }, // 0xcd
242 { s_single, s_term, s_string, s_single, s_eol, }, // 0xce
243 { s_single, s_term, s_string, s_single, s_eol, }, // 0xcf
244 { s_single, s_term, s_string, s_single, s_eol, }, // 0xd0
245 { s_single, s_term, s_string, s_single, s_eol, }, // 0xd1
246 { s_single, s_term, s_string, s_single, s_eol, }, // 0xd2
247 { s_single, s_term, s_string, s_single, s_eol, }, // 0xd3
248 { s_single, s_term, s_string, s_single, s_eol, }, // 0xd4
249 { s_single, s_term, s_string, s_single, s_eol, }, // 0xd5
250 { s_single, s_term, s_string, s_single, s_eol, }, // 0xd6
251 { s_single, s_term, s_string, s_single, s_eol, }, // 0xd7
252 { s_single, s_term, s_string, s_single, s_eol, }, // 0xd8
253 { s_single, s_term, s_string, s_single, s_eol, }, // 0xd9
254 { s_single, s_term, s_string, s_single, s_eol, }, // 0xda
255 { s_single, s_term, s_string, s_single, s_eol, }, // 0xdb
256 { s_single, s_term, s_string, s_single, s_eol, }, // 0xdc
257 { s_single, s_term, s_string, s_single, s_eol, }, // 0xdd
258 { s_single, s_term, s_string, s_single, s_eol, }, // 0xde
259 { s_single, s_term, s_string, s_single, s_eol, }, // 0xdf
260 { s_single, s_term, s_string, s_single, s_eol, }, // 0xe0
261 { s_single, s_term, s_string, s_single, s_eol, }, // 0xe1
262 { s_single, s_term, s_string, s_single, s_eol, }, // 0xe2
263 { s_single, s_term, s_string, s_single, s_eol, }, // 0xe3
264 { s_single, s_term, s_string, s_single, s_eol, }, // 0xe4
265 { s_single, s_term, s_string, s_single, s_eol, }, // 0xe5
266 { s_single, s_term, s_string, s_single, s_eol, }, // 0xe6
267 { s_single, s_term, s_string, s_single, s_eol, }, // 0xe7
268 { s_single, s_term, s_string, s_single, s_eol, }, // 0xe8
269 { s_single, s_term, s_string, s_single, s_eol, }, // 0xe9
270 { s_single, s_term, s_string, s_single, s_eol, }, // 0xea
271 { s_single, s_term, s_string, s_single, s_eol, }, // 0xeb
272 { s_single, s_term, s_string, s_single, s_eol, }, // 0xec
273 { s_single, s_term, s_string, s_single, s_eol, }, // 0xed
274 { s_single, s_term, s_string, s_single, s_eol, }, // 0xee
275 { s_single, s_term, s_string, s_single, s_eol, }, // 0xef
276 { s_single, s_term, s_string, s_single, s_eol, }, // 0xf0
277 { s_single, s_term, s_string, s_single, s_eol, }, // 0xf1
278 { s_single, s_term, s_string, s_single, s_eol, }, // 0xf2
279 { s_single, s_term, s_string, s_single, s_eol, }, // 0xf3
280 { s_single, s_term, s_string, s_single, s_eol, }, // 0xf4
281 { s_single, s_term, s_string, s_single, s_eol, }, // 0xf5
282 { s_single, s_term, s_string, s_single, s_eol, }, // 0xf6
283 { s_single, s_term, s_string, s_single, s_eol, }, // 0xf7
284 { s_single, s_term, s_string, s_single, s_eol, }, // 0xf8
285 { s_single, s_term, s_string, s_single, s_eol, }, // 0xf9
286 { s_single, s_term, s_string, s_single, s_eol, }, // 0xfa
287 { s_single, s_term, s_string, s_single, s_eol, }, // 0xfb
288 { s_single, s_term, s_string, s_single, s_eol, }, // 0xfc
289 { s_single, s_term, s_string, s_single, s_eol, }, // 0xfd
290 { s_single, s_term, s_string, s_single, s_eol, }, // 0xfe
291 { s_single, s_term, s_string, s_single, s_eol, }, // 0xff
292 };
293
294
295 TOKEN::TOKEN(const char *fn, string_set *includes) {
296 pushed = false;
297 include_files = includes;
298 include(fn);
299 }
300
301
302 TOKEN::~TOKEN() {
303 while (!streams.empty()) pop();
304 }
305
306
307 void TOKEN::pop() {
308 ifstream *is = streams.front();
309 const char *fn = filenames.front();
310 streams.pop_front();
311 filenamess.erase(fn);
312 if (filenames.size() > 1) filenames.pop_front();
313 if (linenumbers.size() > 1) linenumbers.pop_front();
314 is->close();
315 delete is;
316 }
317
318
319 void TOKEN::push_char(u_char c) {
320 pushed = true;
321 pushed_char = c;
322 }
323
324
325 bool TOKEN::next_char(u_char &uc) {
326 if (pushed) {
327 uc = pushed_char;
328 pushed = false;
329 return true;
330 }
331 while (!streams.empty() && streams.front()->eof()) {
332 pop();
333 }
334 if (streams.empty()) return false;
335 ifstream *is = streams.front();
336 uc = (u_char)is->get();
337 if (is->eof()) return next_char(uc);
338 if (uc == (u_char)'\n') {
339 int &line = linenumbers.front();
340 line++;
341 }
342 return true;
343 }
344
345
346 bool TOKEN::include(const char *fn) {
347 string_set::iterator i = filenamess.find(fn);
348 if (i != filenamess.end()) {
349 token_error("redundant or recursive include file detected");
350 return false;
351 }
352 ifstream *is = new ifstream;
353 is->open(fn);
354 if (is->fail()) {
355 char buf[maxlen];
356 snprintf(buf, sizeof(buf), "include file %s not found", fn);
357 token_error(buf);
358 return false;
359 }
360 string_set &inc = *include_files;
361 inc.insert(fn);
362 streams.push_front(is);
363 filenames.push_front(fn);
364 filenamess.insert(fn);
365 linenumbers.push_front(1);
366 return true;
367 }
368
369
370 const char *TOKEN::next() {
371 if (!pending_tokens.empty()) {
372 const char *t = pending_tokens.front();
373 pending_tokens.pop_front();
374 return t;
375 }
376 if (streams.empty()) return NULL;
377 const int PENDING_LIMIT = 1000;
378 u_char buffer[PENDING_LIMIT];
379 int count = 0;
380 state st = s_init;
381 while (true) {
382 if (count == (PENDING_LIMIT-1)) {
383 token_error("token too long");
384 break;
385 }
386 if (st >= end_state) {
387 token_error("finite state machine error");
388 break;
389 }
390 u_char c;
391 if (!next_char(c)) break;
392 st = parse_table[c][st];
393 switch (st) {
394 case s_string:
395 case s_token: {
396 buffer[count++] = c;
397 } break;
398
399 case s_term: {
400 push_char(c);
401 st = s_init;
402 } break;
403
404 case s_string1: {
405 st = s_string;
406 } break;
407
408 case s_string2: {
409 st = s_init;
410 } break;
411
412 case s_single: {
413 buffer[count++] = c;
414 st = s_init;
415 } break;
416
417 case s_ignore:
418 case s_eol: {
419 } break;
420
421
422 case s_slash: {
423 buffer[count++] = c;
424 if (next_char(c)) {
425 if (c == (u_char)'/') {
426 // start of ignore to eol on //
427 count--;
428 st = s_eol;
429 }
430 else {
431 // not a // token, just return this single /
432 push_char(c);
433 st = s_init;
434 }
435 }
436 else {
437 // cannot get another char
438 st = s_init;
439 }
440 } break;
441
442 default: {
443 token_error();
444 token_error("unknown state %d %s", st, " ");
445 } break;
446 }
447 if (st == s_init) break;
448 }
449
450 buffer[count] = '\0';
451 if (count == 0) return NULL;
452 const char *t = register_string((char*)buffer);
453 if (t == token_include) {
454 const char *f = next(); // should be file name
455 const char *s = next(); // should be semicolon
456 if (s == token_semi) {
457 include(f);
458 return next();
459 }
460 else {
461 push(s);
462 push(f);
463 return t;
464 }
465 }
466 return t;
467 }
468
469
470 int TOKEN::nextint() {
471 const char *t = next();
472 char *e;
473 long i = strtol(t, &e, 10);
474 if (*e != '\0') {
475 token_error("integer", t);
476 return 0;
477 }
478 return (int)i;
479 }
480
481
482 void TOKEN::skipeol() {
483 while (true) {
484 u_char c;
485 if (!next_char(c)) break;
486 if (c == (u_char)'\n') break;
487 }
488 }
489
490
491 void TOKEN::token_error(const char *err) {
492 token_error();
493 char buf[maxlen];
494 snprintf(buf, sizeof(buf), "%s \n", err);
495 my_syslog(buf);
496 }
497
498
499 void TOKEN::token_error(const char *fmt, int d, const char *s) {
500 char buf[maxlen];
501 snprintf(buf, sizeof(buf), fmt, d, s);
502 my_syslog(buf);
503 }
504
505
506 void TOKEN::token_error(const char *fmt, const char *t, const char *h) {
507 if (!h) h = "null";
508 char buf[maxlen];
509 snprintf(buf, sizeof(buf), fmt, t, h);
510 my_syslog(buf);
511 }
512
513
514 void TOKEN::token_error(const char *want, const char *have) {
515 token_error();
516 token_error("expecting %s, found %s", want, have);
517 }
518
519
520 void TOKEN::token_error() {
521 token_error("syntax error at line %d in file %s -- ", cur_line(), cur_fn());
522 line_list::iterator j = linenumbers.begin();
523 string_list::const_iterator i = filenames.begin();
524 for (; i!=filenames.end(); i++,j++) {
525 if (i != filenames.begin()) {
526 const char *fn = (*i);
527 int li = (*j);
528 token_error(" included from line %d in file %s -- ", li, fn);
529 }
530 }
531 }
532