Repository navigation
Expand file tree
/
Copy pathget_ND.py
More file actions
51 lines (44 loc) · 1.84 KB
/
Copy pathget_ND.py
File metadata and controls
51 lines (44 loc) · 1.84 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import configparser
import urllib.parse
from utils import Crawler as CoreCrawler
class Crawler(CoreCrawler):
abbr = 'ND'
def _get_remote_filename(self, local_filename):
entity_type, name = local_filename[:-4].split('|')
year, name = name[:4], name[5:]
if entity_type == 'Cities':
directory = 'General Purpose'
name = name.split(',')[0]
elif entity_type == 'Counties':
directory = 'General Purpose'
elif entity_type == 'School Districts':
directory = 'School District'
elif entity_type == 'Special Education Districts':
directory = 'School District'
else:
directory = 'Special District'
filename = '{} {} {}.pdf'.format(self.abbr, name, year)
return directory, filename
if __name__ == '__main__':
config = configparser.ConfigParser()
config.read('conf.ini')
crawler = Crawler(config, 'north_dakota')
crawler.get(config.get('north_dakota', 'url'))
for entity_type_url in crawler.get_attr('.content a', 'href', single=False):
crawler.get(entity_type_url)
entity_type = crawler.get_text('h1')
for entity_url in crawler.get_attr('.content a', 'href', single=False):
if entity_url is None or '#' in entity_url:
continue
crawler.get(entity_url)
entity_name = crawler.get_text('h1')
for row in crawler.get_elements('.content .field'):
try:
name = crawler.get_text('h3', root=row)
except Exception:
continue
url = crawler.get_attr('a', 'href', root=row)
filename = '{}|{}.pdf'.format(entity_type, name)
crawler.download(url, filename)
crawler.upload_to_ftp(filename)
crawler.close()