Sandbox::https://github.com/apify/crawlee-python/blob/master/scripts/update_public_suffix

Repository navigation

#!/usr/bin/env python3

"""Refresh the bundled snapshot of the Public Suffix List.

The `same-domain` enqueue strategy matches only against the ICANN section of the list, so the snapshot keeps the

license notice and the ICANN rules without comments. It then changes only when the ICANN rules do.

Single-purpose: run with no arguments from anywhere in the repository.

"""

from __future__ import annotations

import urllib.request

from itertools import takewhile

from pathlib import Path

SNAPSHOT = Path(__file__).resolve().parent.parent / 'src/crawlee/_utils/resources/public_suffix_list.dat'

LIST_URL = 'https://publicsuffix.org/list/public_suffix_list.dat'

ICANN_BEGIN = '// ===BEGIN ICANN DOMAINS==='

ICANN_END = '// ===END ICANN DOMAINS==='

def main() -> None:

with urllib.request.urlopen(LIST_URL, timeout=30) as response:

lines = [line.strip() for line in response.read().decode('utf-8').splitlines()]

# The leading comment block is the MPL 2.0 license notice, which has to ship with the list.

license_notice = list(takewhile(lambda line: line.startswith('//'), lines))

begin, end = lines.index(ICANN_BEGIN), lines.index(ICANN_END)

rules = [line for line in lines[begin + 1 : end] if line and not line.startswith('//')]

SNAPSHOT.write_text(

'\n'.join(

[

*license_notice,

'',

f'// The ICANN section of {LIST_URL}, generated by scripts/update_public_suffix_list.py.',

'',

ICANN_BEGIN,

*rules,

ICANN_END,

'',

]

encoding='utf-8',

)

if __name__ == '__main__':

main()