# HG changeset patch
# User Carl Byington <carl@five-ten-sg.com>
# Date 1209586292 25200
# Node ID 44ffef730bc4525c2c7487d3876fb42fa03fea93
# Parent  a47f161c76c9c3648a649ba0a17ccd48323eba0f
cleanup tokenizer differences with other projects, update tld list

diff -r a47f161c76c9 -r 44ffef730bc4 src/tokenizer.cpp
--- a/src/tokenizer.cpp	Sat Mar 22 11:25:30 2008 -0700
+++ b/src/tokenizer.cpp	Wed Apr 30 13:11:32 2008 -0700
@@ -303,9 +303,9 @@
 	ifstream *is = streams.front();
 	char *fn = filenames.front();
 	streams.pop_front();
-	filenames.pop_front();
 	filenamess.erase(fn);
-	linenumbers.pop_front();
+	if (filenames.size() > 1)	filenames.pop_front();
+	if (linenumbers.size() > 1) linenumbers.pop_front();
 	is->close();
 	delete is;
 }
@@ -437,7 +437,7 @@
 
 			default: {
 				token_error();
-				token_error("unknown state %d %s \n", st, " ");
+				token_error("unknown state %d %s", st, " ");
 			} break;
 		}
 		if (st == s_init) break;
@@ -509,7 +509,7 @@
 
 void TOKEN::token_error(const char *want, const char *have) {
 	token_error();
-	token_error("expecting %s, found %s \n", want, have);
+	token_error("expecting %s, found %s", want, have);
 }
 
 
@@ -521,7 +521,7 @@
 		if (i != filenames.begin()) {
 			char *fn = (*i);
 			int   li = (*j);
-			token_error("\n    included from line %d in file %s -- ", li, fn);
+			token_error("    included from line %d in file %s -- ", li, fn);
 		}
 	}
 }
diff -r a47f161c76c9 -r 44ffef730bc4 src/tokenizer.h
--- a/src/tokenizer.h	Sat Mar 22 11:25:30 2008 -0700
+++ b/src/tokenizer.h	Wed Apr 30 13:11:32 2008 -0700
@@ -51,8 +51,8 @@
 	int 	nextint();
 	void	skipeol();			// skip to eol
 	void	push(char *token)			{pending_tokens.push_front(token);};
-	char	*cur_fn()					{return filenames.front();};
-	int 	cur_line()					{return linenumbers.front();};
+    const char  *cur_fn()           {return filenames.empty()   ? "" : filenames.front();};
+    int         cur_line()          {return linenumbers.empty() ? 0  : linenumbers.front();};
 	void	token_error(const char *err);
 	void	token_error(const char *fmt, int d, const char *s);
 	void	token_error(const char *fmt, const char *t, const char *h);
diff -r a47f161c76c9 -r 44ffef730bc4 tld.conf
--- a/tld.conf	Sat Mar 22 11:25:30 2008 -0700
+++ b/tld.conf	Wed Apr 30 13:11:32 2008 -0700
@@ -1,5 +1,5 @@
 #
-# icann tlds from http://www.icann.org/registries/listing.html as of 2007-09-08
+# icann tlds from http://www.icann.org/registries/listing.html as of 2008-04-30
 #
 aero
 asia
@@ -23,7 +23,12 @@
 travel
 
 #
-# iana root tlds from http://www.iana.org/cctld/cctld-whois.htm as of 2007-09-08
+# iana root tlds from http://www.iana.org/cctld/cctld-whois.htm as of 2008-04-30
+#
+# curl http://www.iana.org/domains/root/db/ 2>/dev/null | \
+#   grep root.db.*country-code | \
+#     sed -e 's/.*root.db.\(..\)\.html.*code.........\([^<]*\)<.*/\1  # \2/g' | \
+#       sort >yyy
 #
 ac  # Ascension Island
 ad  # Andorra
@@ -52,6 +57,7 @@
 bh  # Bahrain
 bi  # Burundi
 bj  # Benin
+bl  # Saint Barthelemy
 bm  # Bermuda
 bn  # Brunei Darussalam
 bo  # Bolivia
@@ -66,7 +72,7 @@
 cc  # Cocos (Keeling) Islands
 cd  # Congo, The Democratic Republic of the
 cf  # Central African Republic
-cg  # Congo, Republic of
+cg  # Congo
 ch  # Switzerland
 ci  # Cote d'Ivoire
 ck  # Cook Islands
@@ -120,9 +126,9 @@
 gw  # Guinea-Bissau
 gy  # Guyana
 hk  # Hong Kong
-hm  # Heard and McDonald Islands
+hm  # Heard Island and McDonald Islands
 hn  # Honduras
-hr  # Croatia/Hrvatska
+hr  # Croatia
 ht  # Haiti
 hu  # Hungary
 id  # Indonesia
@@ -145,7 +151,7 @@
 ki  # Kiribati
 km  # Comoros
 kn  # Saint Kitts and Nevis
-kp  # Korea, Democratic People's Republic
+kp  # Korea, Democratic People's Republic of
 kr  # Korea, Republic of
 kw  # Kuwait
 ky  # Cayman Islands
@@ -165,6 +171,7 @@
 mc  # Monaco
 md  # Moldova, Republic of
 me  # Montenegro
+mf  # Saint Martin (French part)
 mg  # Madagascar
 mh  # Marshall Islands
 mk  # Macedonia, The Former Yugoslav Republic of
@@ -204,14 +211,14 @@
 pk  # Pakistan
 pl  # Poland
 pm  # Saint Pierre and Miquelon
-pn  # Pitcairn Island
+pn  # Pitcairn
 pr  # Puerto Rico
 ps  # Palestinian Territory, Occupied
 pt  # Portugal
 pw  # Palau
 py  # Paraguay
 qa  # Qatar
-re  # Reunion Island
+re  # Reunion
 ro  # Romania
 rs  # Serbia
 ru  # Russian Federation
@@ -224,8 +231,8 @@
 sg  # Singapore
 sh  # Saint Helena
 si  # Slovenia
-sj  # Svalbard and Jan Mayen Islands
-sk  # Slovak Republic
+sj  # Svalbard and Jan Mayen
+sk  # Slovakia
 sl  # Sierra Leone
 sm  # San Marino
 sn  # Senegal
@@ -247,12 +254,12 @@
 tm  # Turkmenistan
 tn  # Tunisia
 to  # Tonga
-tp  # East Timor
+tp  # Portuguese Timor (being phased out)
 tr  # Turkey
 tt  # Trinidad and Tobago
 tv  # Tuvalu
 tw  # Taiwan
-tz  # Tanzania
+tz  # Tanzania, United Republic of
 ua  # Ukraine
 ug  # Uganda
 uk  # United Kingdom
@@ -265,9 +272,9 @@
 ve  # Venezuela
 vg  # Virgin Islands, British
 vi  # Virgin Islands, U.S.
-vn  # Vietnam
+vn  # Viet Nam
 vu  # Vanuatu
-wf  # Wallis and Futuna Islands
+wf  # Wallis and Futuna
 ws  # Samoa
 ye  # Yemen
 yt  # Mayotte