Repository navigation
Expand file tree
/
Copy pathget_VA.py
More file actions
96 lines (83 loc) · 3.82 KB
/
Copy pathget_VA.py
File metadata and controls
96 lines (83 loc) · 3.82 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
import argparse
import configparser
import os
import sys
import urllib.parse
import urllib.request
from utils import Crawler as CoreCrawler
from selenium.webdriver.common.keys import Keys
from datetime import datetime
import re
class Crawler(CoreCrawler):
abbr = 'VA'
def _get_remote_filename(self, local_filename):
if local_filename == "Virginia Beach CAFR.pdf":
local_filename = "Virginia Beach CAFR 2017.pdf"
directory = 'School District' if 'Schools' in local_filename else 'General Purpose'
local_filename = re.sub(r'(\s*-?\s*reissue.*)\.pdf','.pdf',local_filename)
filename = '{} {}'.format(
self.abbr, local_filename.replace(' CAFR', '').replace(' Single Audit', '').replace(' Town', '')
)
if not re.search("Gate City", filename, re.IGNORECASE) and not re.search("Chase City", filename, re.IGNORECASE):
filename = filename.replace(' City', '')
r = re.compile(".*(\d{4})\s*\..{3}")
year = r.search(filename).group(1)
return directory, filename, year
if __name__ == '__main__':
script_name = 'get_VA.py'
start_time = datetime.utcnow()
result = 1
error_message = ""
config_file = ""
config = configparser.ConfigParser()
config.read('conf.ini')
crawler = Crawler(config, 'virginia', script_name)
try:
argparser = argparse.ArgumentParser()
argparser.add_argument("--year")
argparser.add_argument("--category")
args = argparser.parse_args()
config_file = str(crawler.dbparams)
crawler.get(crawler.get_property('url','virginia'))
if args.year:
crawler.send_keys('#ASPxPageControl1_Grid1_ob_Grid1FilterContainer_ctl02_ob_Grid1FilterControl0', args.year + Keys.ENTER)
if args.category:
crawler.click('#ob_iDdlddlCategoriesTB')
crawler.click_xpath('//div[@id="ob_iDdlddlCategoriesItemsContainer"]//ul[@class="ob_iDdlICBC"]//li/b[text() = "{}"]/..'.format(args.category))
download_complete = False
while not download_complete:
urls = crawler.get_attr('a.blacklink', 'href', single=False)
if len(urls)<=0:
print("nothing to download")
download_complete = True
for url in urls:
file = urllib.parse.unquote(url).split('/')[-1]
if ' CAFR' not in file or ' memo ' in file:
continue
file_db_record = crawler.db.readFileStatus(script_name=script_name,file_original_name=file)
if file_db_record and file_db_record["file_status"] == 'Uploaded':
print('File {} was uploaded before, skipping download'.format(file))
continue
file_id = None
if file_db_record:
file_id = file_db_record["id"]
downloaded = crawler.download(url, file,file_id)
if downloaded:
crawler.upload_to_file_storage(file)
file_path = os.path.join(crawler.get_property('downloads_path', 'virginia'),file)
if os.path.exists(file_path):
os.remove(file_path)
if not os.path.exists(file_path):
print('Removed {}'.format(file_path))
try:
crawler.assert_exists("img[src*='next_disabled.gif']")
download_complete = True
except:
crawler.click_xpath('//div[@class="ob_gPBC"]/img[contains(@src, "next")]/..')
except Exception as e:
result = 0
error_message = str(e)
finally:
end_time = datetime.utcnow()
crawler.db.log(script_name, start_time, end_time, config_file, result, error_message)
crawler.close()