Repository navigation
Expand file tree
/
Copy pathget_IA.py
More file actions
46 lines (39 loc) · 1.53 KB
/
Copy pathget_IA.py
File metadata and controls
46 lines (39 loc) · 1.53 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
import argparse
import configparser
from utils import Crawler as CoreCrawler
class Crawler(CoreCrawler):
abbr = 'IA'
def _get_remote_filename(self, local_filename):
entity_type, name, year = local_filename.split('@#')
if entity_type in ('City', 'County'):
directory = 'General Purpose'
elif entity_type == 'School':
directory = 'School District'
else:
directory = 'Special District'
return directory, '{} {} {}.pdf'.format(self.abbr, name.replace('/', ' '), year), year
if __name__ == '__main__':
config = configparser.ConfigParser()
config.read('conf.ini')
crawler = Crawler(config, 'iowa')
crawler.get(config.get('iowa', 'url'))
crawler.select_option('#edit-field-audit-category-tid', 'Financial Statement')
crawler.click('#edit-submit-audit-reports')
while True:
for row in crawler.get_elements('tbody tr'):
try:
url = crawler.get_attr('.file a', 'href', root=row)
except Exception:
continue
cols = crawler.get_elements('td', root=row)
entity_type = cols[1].text.strip()
entity_name = cols[0].text.strip()
year = cols[3].text.strip().split('/')[-1]
filename = '{}@#{}@#{}'.format(entity_type, entity_name, year)
crawler.download(url, filename)
crawler.upload_to_ftp(filename)
try:
crawler.click('.pager-next a')
except Exception:
break
crawler.close()