Mercurial > dnsbl

comparison make-tld-conf.py @ 467:f5b394bec28c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
author Carl Byington <carl@five-ten-sg.com>
date Wed, 16 Dec 2020 10:25:37 -0800
parents 78eedbbce636
children
comparison
equal deleted inserted replaced
466:8ad675a5d448 467:f5b394bec28c
1 #!/usr/bin/python 1 #!/usr/bin/python3
2 2
3 import os, sys, time 3 import os, sys, time
4 4
5 def cmd(c): 5 def cmd(c):
6 print c 6 print(c)
7 return os.popen(c).read().rstrip('\n') 7 return os.popen(c).read().rstrip('\n')
8 8
9 url = 'https://publicsuffix.org/list/effective_tld_names.dat' 9 url = 'https://publicsuffix.org/list/effective_tld_names.dat'
10 c = 'curl %s 2>/dev/null' % url 10 c = 'curl %s 2>/dev/null' % url
11 content = cmd(c) 11 content = cmd(c)
19 prefix = '!' 19 prefix = '!'
20 else: 20 else:
21 prefix = '' 21 prefix = ''
22 yy = [] 22 yy = []
23 for xx in x.split('.'): 23 for xx in x.split('.'):
24 y = unicode(xx, 'utf8') 24 z = xx.encode('punycode').decode('utf8')
25 z = y.encode('punycode')
26 if z[-1:] == '-': yy.append(xx) 25 if z[-1:] == '-': yy.append(xx)
27 else: yy.append('xn--%s' % z) 26 else: yy.append('xn--%s' % z)
28 y = '.'.join(yy) 27 y = '.'.join(yy)
29 if not (y == x): 28 if not (y == x):
30 f.write('// %s%s\n' % (prefix, x)) 29 f.write('// %s%s\n' % (prefix, x))
34 if not rc: 33 if not rc:
35 rc = cmd("sleep 3; dig %s ns +short 2>&1" % x) 34 rc = cmd("sleep 3; dig %s ns +short 2>&1" % x)
36 if not rc: prefix = '// no ns record for ' 35 if not rc: prefix = '// no ns record for '
37 f.write('%s%s\n' % (prefix, x)) 36 f.write('%s%s\n' % (prefix, x))
38 f.write('\n') 37 f.write('\n')
38 f.write('\n')
39 f.write('// add some TLDs from our rpz zone so names under them\n')
40 f.write('// can be added to uribl style lists\n')
41 f.write('\n')
39 42
40 c="grep '^\*.*multi.surbl' /etc/cron.daily/rpz | awk '{print $1}' | cut -c3- | sed -e 's/.multi.surbl.org//g'" 43 c="grep '^\*.*multi.surbl' /etc/cron.daily/rpz | awk '{print $1}' | cut -c3- | sed -e 's/.multi.surbl.org//g' | grep '\.'"
44 content = cmd(c)
45 for x in content.split('\n'):
46 f.write(x + '\n')
47
48 f.write('\n')
49 c="grep '^[^\*].*multi.surbl' /etc/cron.daily/rpz | egrep -v 'CNAME|g.write' | awk '{print $1}' | sed -e 's/.multi.surbl.org//g' | cut -d. -f2- | grep '\.'"
41 content = cmd(c) 50 content = cmd(c)
42 for x in content.split('\n'): 51 for x in content.split('\n'):
43 f.write(x + '\n') 52 f.write(x + '\n')
44 53
45 f.close() 54 f.close()