"""This is the print banner function.

c = randint(1, 4)

if c == 1:

color = '31m'

elif c == 2:

color = '32m'

elif c == 3:

color = '33m'

elif c == 4:

color = '35m'

b = randint(1, 4)

if b == 1:

banner = """\033[%s

""" % (color, version)

elif b == 2:

banner = """\033[%s

""" % (color, version)

elif b == 3:

banner = """\033[%s

""" % (color, version)

elif b == 4:

banner = """\033[%s

""" % (color, version)

sys.stdout.write(banner)

sys.stdout.write('\n')

"""This is the load config function.

configsettings = {}

config = ConfigParser.ConfigParser()

dir_path = os.path.dirname(os.path.realpath(__file__))

# check if env var for config file and use that

try:

configfile = os.environ['DISKOVER_CONFIG']

except KeyError:

configfile = '%s/diskover.cfg' % dir_path

pass

# Check for config file

if not os.path.isfile(configfile):

print('Config file %s not found, exiting.' % configfile)

sys.exit(1)

config.read(configfile)

# Check if any sections missing from config and exit if there is

try:

try:

d = config.get('excludes', 'dirs')

dirs = d.split(',')

configsettings['excluded_dirs'] = set(dirs)

except ConfigParser.NoOptionError:

configsettings['excluded_dirs'] = set([])

try:

f = config.get('excludes', 'files')

files = f.split(',')

configsettings['excluded_files'] = set(files)

except ConfigParser.NoOptionError:

configsettings['excluded_files'] = set([])

try:

d = config.get('includes', 'dirs')

dirs = d.split(',')

configsettings['included_dirs'] = set(dirs)

except (ConfigParser.NoOptionError):

configsettings['included_dirs'] = set([])

try:

f = config.get('includes', 'files')

files = f.split(',')

configsettings['included_files'] = set(files)

except ConfigParser.NoOptionError:

configsettings['included_files'] = set([])

try:

configsettings['ownersgroups_uidgidonly'] = config.get('ownersgroups', 'uidgidonly').lower()

except ConfigParser.NoOptionError:

configsettings['ownersgroups_uidgidonly'] = "false"

try:

configsettings['ownersgroups_domain'] = config.get('ownersgroups', 'domain').lower()

except ConfigParser.NoOptionError:

configsettings['ownersgroups_domain'] = "false"

try:

configsettings['ownersgroups_domainsep'] = config.get('ownersgroups', 'domainsep')

except ConfigParser.NoOptionError:

configsettings['ownersgroups_domainsep'] = "\\"

try:

configsettings['ownersgroups_keepdomain'] = config.get('ownersgroups', 'keepdomain').lower()

except ConfigParser.NoOptionError:

configsettings['ownersgroups_keepdomain'] = "false"

try:

t = config.get('autotag', 'files')

atf = json.loads(t)

configsettings['autotag_files'] = atf

except ValueError as e:

raise ValueError("Error in config autotag files: %s" % e)

except ConfigParser.NoOptionError:

configsettings['autotag_files'] = []

try:

t = config.get('autotag', 'dirs')

atd = json.loads(t)

configsettings['autotag_dirs'] = atd

except ValueError as e:

raise ValueError("Error in config autotag dirs: %s" % e)

except ConfigParser.NoOptionError:

configsettings['autotag_dirs'] = []

try:

configsettings['costpergb'] = float(config.get('storagecost', 'costpergb'))

except ConfigParser.NoOptionError:

configsettings['costpergb'] = 0.03

try:

configsettings['costpergb_base'] = int(config.get('storagecost', 'base'))

except ConfigParser.NoOptionError:

configsettings['costpergb_base'] = 2

try:

s = config.get('storagecost', 'paths')

scp = json.loads(s)

configsettings['costpergb_paths'] = scp

except ValueError as e:

raise ValueError("Error in config storagecost paths: %s" % e)

except ConfigParser.NoOptionError:

configsettings['costpergb_paths'] = []

try:

s = config.get('storagecost', 'times')

sct = json.loads(s)

configsettings['costpergb_times'] = sct

except ValueError as e:

raise ValueError("Error in config storagecost times: %s" % e)

except ConfigParser.NoOptionError:

configsettings['costpergb_times'] = []

try:

configsettings['costpergb_priority'] = config.get('storagecost', 'priority')

except ConfigParser.NoOptionError:

configsettings['costpergb_priority'] = "path"

try:

configsettings['aws'] = config.get('elasticsearch', 'aws').lower()

except ConfigParser.NoOptionError:

configsettings['aws'] = "false"

try:

configsettings['es_host'] = config.get('elasticsearch', 'host')

except ConfigParser.NoOptionError:

configsettings['es_host'] = "localhost"

try:

configsettings['es_port'] = int(config.get('elasticsearch', 'port'))

except ConfigParser.NoOptionError:

configsettings['es_port'] = 9200

try:

configsettings['es_user'] = config.get('elasticsearch', 'user')

except ConfigParser.NoOptionError:

configsettings['es_user'] = ""

try:

configsettings['es_password'] = config.get('elasticsearch', 'password')

except ConfigParser.NoOptionError:

configsettings['es_password'] = ""

try:

configsettings['index'] = config.get('elasticsearch', 'indexname')

except ConfigParser.NoOptionError:

configsettings['index'] = ""

try:

configsettings['es_timeout'] = int(config.get('elasticsearch', 'timeout'))

except ConfigParser.NoOptionError:

configsettings['es_timeout'] = 10

try:

configsettings['es_maxsize'] = int(config.get('elasticsearch', 'maxsize'))

except ConfigParser.NoOptionError:

configsettings['es_maxsize'] = 10

try:

configsettings['es_max_retries'] = int(config.get('elasticsearch', 'maxretries'))

except ConfigParser.NoOptionError:

configsettings['es_max_retries'] = 0

try:

configsettings['es_wait_status_yellow'] = config.get('elasticsearch', 'wait').lower()

except ConfigParser.NoOptionError:

configsettings['es_wait_status_yellow'] = "false"

try:

configsettings['es_chunksize'] = int(config.get('elasticsearch', 'chunksize'))

except ConfigParser.NoOptionError:

configsettings['es_chunksize'] = 500

try:

configsettings['index_shards'] = int(config.get('elasticsearch', 'shards'))

except ConfigParser.NoOptionError:

configsettings['index_shards'] = 5

try:

configsettings['index_replicas'] = int(config.get('elasticsearch', 'replicas'))

except ConfigParser.NoOptionError:

configsettings['index_replicas'] = 1

try:

configsettings['index_refresh'] = config.get('elasticsearch', 'indexrefresh')

except ConfigParser.NoOptionError:

configsettings['index_refresh'] = "1s"

try:

configsettings['disable_replicas'] = config.get('elasticsearch', 'disablereplicas').lower()

except ConfigParser.NoOptionError:

configsettings['disable_replicas'] = "false"

try:

configsettings['index_translog_size'] = config.get('elasticsearch', 'translogsize')

except ConfigParser.NoOptionError:

configsettings['index_translog_size'] = "512mb"

try:

configsettings['es_scrollsize'] = int(config.get('elasticsearch', 'scrollsize'))

except ConfigParser.NoOptionError:

configsettings['es_scrollsize'] = 100

try:

configsettings['redis_host'] = config.get('redis', 'host')

except ConfigParser.NoOptionError:

configsettings['redis_host'] = "localhost"

try:

configsettings['redis_port'] = int(config.get('redis', 'port'))

except ConfigParser.NoOptionError:

configsettings['redis_port'] = 6379

try:

configsettings['redis_password'] = config.get('redis', 'password')

except ConfigParser.NoOptionError:

configsettings['redis_password'] = ""

try:

configsettings['redis_cachedirtimes'] = config.get('redis', 'cachedirtimes').lower()

except ConfigParser.NoOptionError:

configsettings['redis_cachedirtimes'] = "false"

try:

configsettings['redis_dirtimesttl'] = int(config.get('redis', 'dirtimesttl'))

except ConfigParser.NoOptionError:

configsettings['redis_dirtimesttl'] = 604800

try:

configsettings['redis_db'] = int(config.get('redis', 'db'))

except ConfigParser.NoOptionError:

configsettings['redis_db'] = 0

try:

configsettings['redis_rq_timeout'] = int(config.get('redis', 'timeout'))

except ConfigParser.NoOptionError:

configsettings['redis_rq_timeout'] = 180

try:

configsettings['redis_ttl'] = int(config.get('redis', 'ttl'))

except ConfigParser.NoOptionError:

configsettings['redis_ttl'] = 500

try:

configsettings['redis_queue'] = config.get('redis', 'queue')

except ConfigParser.NoOptionError:

configsettings['redis_queue'] = "diskover"

try:

configsettings['redis_queue_crawl'] = config.get('redis', 'queuecrawl')

except ConfigParser.NoOptionError:

configsettings['redis_queue_crawl'] = "diskover_crawl"

try:

configsettings['redis_queue_calcdir'] = config.get('redis', 'queuecalcdir')

except ConfigParser.NoOptionError:

configsettings['redis_queue_calcdir'] = "diskover_calcdir"

try:

configsettings['adaptivebatch_startsize'] = int(config.get('adaptivebatch', 'startsize'))

except ConfigParser.NoOptionError:

configsettings['adaptivebatch_startsize'] = 50

try:

configsettings['adaptivebatch_maxsize'] = int(config.get('adaptivebatch', 'maxsize'))

except ConfigParser.NoOptionError:

configsettings['autobatch_maxsize'] = 500

try:

configsettings['adaptivebatch_stepsize'] = int(config.get('adaptivebatch', 'stepsize'))

except ConfigParser.NoOptionError:

configsettings['adaptivebatch_stepsize'] = 10

try:

configsettings['adaptivebatch_maxfiles'] = int(config.get('adaptivebatch', 'maxfiles'))

except ConfigParser.NoOptionError:

configsettings['adaptivebatch_maxfiles'] = 50000

try:

configsettings['listener_host'] = config.get('socketlistener', 'host')

except ConfigParser.NoOptionError:

configsettings['listener_host'] = "localhost"

try:

configsettings['listener_port'] = int(config.get('socketlistener', 'port'))

except ConfigParser.NoOptionError:

configsettings['listener_port'] = 9999

try:

configsettings['listener_maxconnections'] = int(config.get('socketlistener', 'maxconnections'))

except ConfigParser.NoOptionError:

configsettings['listener_maxconnections'] = 5

try:

configsettings['listener_twcport'] = int(config.get('socketlistener', 'twcport'))

except ConfigParser.NoOptionError:

configsettings['listener_twcport'] = 9998

try:

configsettings['diskover_path'] = config.get('paths', 'diskoverpath')

except ConfigParser.NoOptionError:

configsettings['diskover_path'] = "./diskover.py"

try:

configsettings['python_path'] = config.get('paths', 'pythonpath')

except ConfigParser.NoOptionError:

configsettings['python_path'] = "python"

try:

configsettings['md5_readsize'] = int(config.get('dupescheck', 'readsize'))

except ConfigParser.NoOptionError:

configsettings['md5_readsize'] = 65536

try:

configsettings['dupes_maxsize'] = int(config.get('dupescheck', 'maxsize'))

except ConfigParser.NoOptionError:

configsettings['dupes_maxsize'] = 1073741824

try:

configsettings['dupes_checkbytes'] = int(config.get('dupescheck', 'checkbytes'))

except ConfigParser.NoOptionError:

configsettings['dupes_checkbytes'] = 64

try:

configsettings['crawlbot_botsleep'] = float(config.get('crawlbot', 'sleeptime'))

except ConfigParser.NoOptionError:

configsettings['crawlbot_botsleep'] = 0.1

try:

configsettings['crawlbot_botthreads'] = int(config.get('crawlbot', 'botthreads'))

except ConfigParser.NoOptionError:

configsettings['crawlbot_botthreads'] = 8

try:

configsettings['crawlbot_dirlisttime'] = int(config.get('crawlbot', 'dirlisttime'))

except ConfigParser.NoOptionError:

configsettings['crawlbot_dirlisttime'] = 3600

try:

configsettings['gource_maxfilelag'] = float(config.get('gource', 'maxfilelag'))

except ConfigParser.NoOptionError:

configsettings['gource_maxfilelag'] = 5

try:

configsettings['qumulo_host'] = config.get('qumulo', 'cluster')

except ConfigParser.NoOptionError:

configsettings['qumulo_host'] = ""

try:

configsettings['qumulo_api_user'] = config.get('qumulo', 'api_user')

except ConfigParser.NoOptionError:

configsettings['qumulo_api_user'] = ""

try:

configsettings['qumulo_api_password'] = config.get('qumulo', 'api_password')

except ConfigParser.NoOptionError:

configsettings['qumulo_api_password'] = ""

except ConfigParser.NoSectionError as e:

print('Missing section from diskover.cfg, check diskover.cfg.sample and copy over, exiting. (%s)' % e)

sys.exit(1)

"""This is the get plugins info function.

plugin_dir = os.path.dirname(os.path.realpath(__file__)) + "/plugins"

main_module = "__init__"

plugins_info = []

possible_plugins = os.listdir(plugin_dir)

for i in possible_plugins:

location = os.path.join(plugin_dir, i)

if not os.path.isdir(location) or not main_module + ".py" \

in os.listdir(location):

continue

info = imp.find_module(main_module, [location])

plugins_info.append({"name": i, "info": info})

"""This is the load plugins function.

loaded_plugins = []

plugins_info = get_plugins_info()

for plugin_info in plugins_info:

plugin_module = imp.load_module(plugin_info["name"], *plugin_info["info"])

loaded_plugins.append(plugin_module)

"""This is the list plugins function.

plugins_info = get_plugins_info()

for plugin_info in plugins_info:

"""This is the es index create function.

logger.info('Checking es index: %s', indexname)

# check for existing es index

if es.indices.exists(index=indexname):

# check if crawlbot or reindex cli argument and don't delete existing index

if cliargs['reindex']:

logger.info('Reindexing (non-recursive, preserving tags)')

return

elif cliargs['reindexrecurs']:

logger.info('Reindexing (recursive, preserving tags)')

return

elif cliargs['crawlbot']:

return

# delete existing index

else:

logger.warning('es index exists, deleting')

es.indices.delete(index=indexname, ignore=[400, 404])

# set up es index mappings and create new index

if cliargs['qumulo']:

from diskover_qumulo import get_qumulo_mappings

mappings = get_qumulo_mappings(config)

elif cliargs['s3']:

from diskover_s3 import get_s3_mappings

mappings = get_s3_mappings(config)

else:

mappings = {

"settings": {

"index" : {

"number_of_shards": config['index_shards'],

"number_of_replicas": config['index_replicas']

}

},

"mappings": {

"diskspace": {

"properties": {

"path": {

"type": "keyword"

},

"total": {

"type": "long"

},

"used": {

"type": "long"

},

"free": {

"type": "long"

},

"available": {

"type": "long"

},

"indexing_date": {

"type": "date"

}

}

},

"crawlstat": {

"properties": {

"path": {

"type": "keyword"

},

"state": {

"type": "text"

},

"crawl_time": {

"type": "float"

},

"indexing_date": {

"type": "date"

}

}

},

"worker": {

"properties": {

"worker_name": {

"type": "keyword"

},

"dir_count": {

"type": "integer"

},

"file_count": {

"type": "integer"

},

"bulk_time": {

"type": "float"

},

"crawl_time": {

"type": "float"

},

"indexing_date": {

"type": "date"

}

}

},

"directory": {

"properties": {

"filename": {

"type": "keyword"

},

"path_parent": {

"type": "keyword"

},

"filesize": {

"type": "long"

},

"items": {

"type": "long"

},

"items_files": {

"type": "long"

},

"items_subdirs": {

"type": "long"

},

"owner": {

"type": "keyword"

},

"group": {

"type": "keyword"

},

"last_modified": {

"type": "date"

},

"last_access": {

"type": "date"

},

"last_change": {

"type": "date"

},

"hardlinks": {

"type": "integer"

},

"inode": {

"type": "float"

},

"tag": {

"type": "keyword"

},

"tag_custom": {

"type": "keyword"

},

"crawl_time": {

"type": "float"

},

"change_percent_filesize": {

"type": "float"

},

"change_percent_items": {

"type": "float"

},

"change_percent_items_files": {

"type": "float"

},

"change_percent_items_subdirs": {

"type": "float"

},

"costpergb": {

"type": "scaled_float",

"scaling_factor": 100

},

"worker_name": {

"type": "keyword"

},

"indexing_date": {

"type": "date"

}

}

},

"file": {

"properties": {

"filename": {

"type": "keyword"

},

"extension": {

"type": "keyword"

},

"path_parent": {

"type": "keyword"

},

"filesize": {

"type": "long"

},

"owner": {

"type": "keyword"

},

"group": {

"type": "keyword"

},

"last_modified": {

"type": "date"

},

"last_access": {

"type": "date"

},

"last_change": {

"type": "date"

},

"hardlinks": {

"type": "integer"

},

"inode": {

"type": "float"

},

"filehash": {

"type": "keyword"

},

"tag": {

"type": "keyword"

},

"tag_custom": {

"type": "keyword"

},

"dupe_md5": {

"type": "keyword"

},

"costpergb": {

"type": "scaled_float",

"scaling_factor": 100

},

"worker_name": {

"type": "keyword"

},

"indexing_date": {

"type": "date"

}

}

}

}

}

# check plugins for additional mappings

for plugin in plugins:

mappings = (plugin.add_mappings(mappings))

logger.info('Creating es index')

es.indices.create(index=indexname, body=mappings)

"""This is the es index bulk add function.

if len(doclist) == 0:

return

if config['es_wait_status_yellow'] == "true":

# wait for es health to be at least yellow

es.cluster.health(wait_for_status='yellow',

request_timeout=config['es_timeout'])

# bulk load data to Elasticsearch index

diskover_connections.helpers.bulk(es, doclist, index=cliargs['index'], chunk_size=config['es_chunksize'],

"""This is the es delete path bulk function.

file_id_list = []

dir_id_list = []

file_delete_list = []

dir_delete_list = []

# refresh index

es.indices.refresh(index=cliargs['index'])

# escape special characters

newpath = escape_chars(path)

# create wildcard string and check for / (root) path

if newpath == '\/':

newpathwildcard = '\/*'

else:

newpathwildcard = newpath + '\/*'

# file doc search

if recursive:

data = {

"query": {

"query_string": {

"query": "path_parent: " + newpath + " OR "

"path_parent: " + newpathwildcard,

"analyze_wildcard": "true"

}

}

}

else:

data = {

"query": {

"query_string": {

"query": "path_parent: " + newpath

}

}

}

logger.info('Searching for all files in %s' % path)

# search es and start scroll

res = es.search(index=cliargs['index'], doc_type='file', scroll='1m',

size=config['es_scrollsize'], body=data,

request_timeout=config['es_timeout'])

while res['hits']['hits'] and len(res['hits']['hits']) > 0:

for hit in res['hits']['hits']:

# add doc id to file_id_list

file_id_list.append(hit['_id'])

# add file path info inc. tags to reindex_file_list

reindex_dict['file'].append((hit['_source']['path_parent'] +

'/' + hit['_source']['filename'],

hit['_source']['tag'],

hit['_source']['tag_custom']))

# get es scroll id

scroll_id = res['_scroll_id']

# use es scroll api

res = es.scroll(scroll_id=scroll_id, scroll='1m',

request_timeout=config['es_timeout'])

logger.info('Found %s files for %s' % (len(file_id_list), path))

# add file id's to delete_list

for i in file_id_list:

d = {

'_op_type': 'delete',

'_index': cliargs['index'],

'_type': 'file',

'_id': i

}

file_delete_list.append(d)

if len(file_delete_list) > 0:

# bulk delete files in es

logger.info('Bulk deleting files in es index')

index_bulk_add(es, file_delete_list, config, cliargs)

# directory doc search

if recursive:

data = {

'query': {

'query_string': {

'query': '(path_parent: ' + newpath + ') OR '

'(path_parent: ' + newpathwildcard + ') OR (filename: "'

+ os.path.basename(path) + '" AND path_parent: "'

+ os.path.abspath(os.path.join(path, os.pardir)) + '")',

'analyze_wildcard': 'true'

}

}

}

else:

data = {

'query': {

'query_string': {

'query': '(path_parent: ' + newpath + ') OR (filename: "'

+ os.path.basename(path) + '" AND path_parent: "'

+ os.path.abspath(os.path.join(path, os.pardir)) + '")'

}

}

}

logger.info('Searching for all directories in %s' % path)

# search es and start scroll

res = es.search(index=cliargs['index'], doc_type='directory', scroll='1m',

size=config['es_scrollsize'], body=data, request_timeout=config['es_timeout'])

while res['hits']['hits'] and len(res['hits']['hits']) > 0:

for hit in res['hits']['hits']:

# add directory doc id to dir_id_list

dir_id_list.append(hit['_id'])

# add directory path info inc. tags, filesize, items to reindex_dir_list

reindex_dict['directory'].append((hit['_source']['path_parent'] +

'/' + hit['_source']['filename'],

hit['_source']['tag'],

hit['_source']['tag_custom']))

# get es scroll id

scroll_id = res['_scroll_id']

# use es scroll api

res = es.scroll(scroll_id=scroll_id, scroll='1m',

request_timeout=config['es_timeout'])

logger.info('Found %s directories for %s' % (len(dir_id_list), path))

# add dir id's to delete_list

for i in dir_id_list:

d = {

'_op_type': 'delete',

'_index': cliargs['index'],

'_type': 'directory',

'_id': i

}

dir_delete_list.append(d)

if len(dir_delete_list) > 0:

# bulk delete directories in es

logger.info('Bulk deleting directories in es index')

index_bulk_add(es, dir_delete_list, config, cliargs)

index=None, path=None, sort=False, maxdepth=None, pathid=False):

"""This is the es get docs function.

if index is None:

index = cliargs['index']

data = _index_get_docs_data(index, copytags, hotdirs, doctype, path, maxdepth, sort, logger)

# refresh index

es.indices.refresh(index)

# search es and start scroll

res = es.search(index=index, doc_type=doctype, scroll='1m',

size=config['es_scrollsize'], body=data, request_timeout=config['es_timeout'])

doclist = []

pathdict = {}

doccount = 0

while res['hits']['hits'] and len(res['hits']['hits']) > 0:

for hit in res['hits']['hits']:

fullpath = os.path.abspath(os.path.join(hit['_source']['path_parent'], hit['_source']['filename']))

rel_path = fullpath.replace(rootdir_path, ".")

if copytags:

doclist.append((fullpath, hit['_source']['tag'], hit['_source']['tag_custom'], doctype))

elif hotdirs:

doclist.append((hit['_id'], fullpath, hit['_source']['filesize'], hit['_source']['items'],

hit['_source']['items_files'], hit['_source']['items_subdirs']))

elif pathid:

pathdict[rel_path] = hit['_id']

else:

# convert es time to unix time format

mtime = time.mktime(datetime.strptime(

hit['_source']['last_modified'],

'%Y-%m-%dT%H:%M:%S').timetuple())

doclist.append((hit['_id'], fullpath, mtime, doctype))

doccount += 1

# use es scroll api

res = es.scroll(scroll_id=res['_scroll_id'], scroll='1m',

request_timeout=config['es_timeout'])

logger.info('Found %s %s docs' % (str(doccount), doctype))

if pathid:

return pathdict

else:

index=None, path=None, sort=False, maxdepth=None):

"""This is the es get docs generator function.

if index is None:

index = cliargs['index']

data = _index_get_docs_data(index, copytags, hotdirs, doctype, path, maxdepth, sort, logger)

# refresh index

es.indices.refresh(index)

# search es and start scroll

res = es.search(index=index, doc_type=doctype, scroll='1m',

size=config['es_scrollsize'], body=data, request_timeout=config['es_timeout'])

doclist = []

doccount = 0

while res['hits']['hits'] and len(res['hits']['hits']) > 0:

for hit in res['hits']['hits']:

fullpath = os.path.abspath(os.path.join(hit['_source']['path_parent'], hit['_source']['filename']))

if copytags:

doclist.append((fullpath, hit['_source']['tag'], hit['_source']['tag_custom'], doctype))

elif hotdirs:

doclist.append((hit['_id'], fullpath, hit['_source']['filesize'], hit['_source']['items'],

hit['_source']['items_files'], hit['_source']['items_subdirs']))

else:

# convert es time to unix time format

mtime = time.mktime(datetime.strptime(hit['_source']['last_modified'],

'%Y-%m-%dT%H:%M:%S').timetuple())

atime = time.mktime(datetime.strptime(hit['_source']['last_access'],

'%Y-%m-%dT%H:%M:%S').timetuple())

ctime = time.mktime(datetime.strptime(hit['_source']['last_change'],

'%Y-%m-%dT%H:%M:%S').timetuple())

doclist.append((hit['_id'], fullpath, mtime, atime, ctime, doctype))

doccount += 1

# yield results before loop

yield doclist

del doclist[:]

# use es scroll api

res = es.scroll(scroll_id=res['_scroll_id'], scroll='1m',

request_timeout=config['es_timeout'])

logger.info('Found %s %s docs' % (str(doccount), doctype))

if copytags:

logger.info('Searching for all %s docs with tags in %s...', doctype, index)

data = {

'_source': ['path_parent', 'filename', 'tag', 'tag_custom'],

'query': {

'query_string': {

'query': 'tag:(NOT "") OR tag_custom:(NOT "")'

}

}

}

elif hotdirs:

logger.info('Searching for all %s docs in %s...', doctype, index)