Visitar URL original
Cleanup further (#4963) · schemaorg/schemaorg@703fe56 · GitHub
Skip to content

Commit 703fe56

Browse files
authored
Cleanup further (#4963)
* Clean up imports, headers, and formatting in scripts Modernize imports and type annotations for Python 3.11+, drop legacy header boilerplate and UTF-8 stanzas, and standardize lazy logging and f-strings across software/scripts/. * Fix bugs, revive legacy tools, and use SdoUri in scripts Standardize CLI entry points across software/scripts/, consolidate URI handling onto SdoUri, and revive the bit-rotted compareterms, compare_health, scansite, and inputconvert utilities for Python 3.11 and rdflib 7. Also fix several bugs: preserve custom output paths and --shacltests behavior in buildsite, fail fast on invalid arguments in reorg, fix CORS headers and .html routing in devserv, stop double-printing errors in runtests, limit --sourceoutput to failing examples in validate_examples_shacl, and emit valid Turtle in bmdc2sdo.
1 parent 0c41dce commit 703fe56

16 files changed

Lines changed: 1539 additions & 1546 deletions

‎software/scripts/bmdc2sdo/mcf2turtle.py‎

Lines changed: 169 additions & 201 deletions
Large diffs are not rendered by default.

‎software/scripts/bmdc2sdo/syntaxdebug.py‎

Lines changed: 37 additions & 326 deletions
Large diffs are not rendered by default.
Lines changed: 102 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,90 +1,126 @@
11
#!/usr/bin/env python3
2-
# -*- coding: utf-8 -*-
2+
# Based on a CC0 licensed script shared by Gabriel Romualdo <https://xtrp.io/>
33

4-
# Note: links below are broken.
5-
# Based on a CC0 licensed script
64
# <https://xtrp.io/blog/2019/11/09/a-quick-python-script-to-find-broken-links/>
7-
# Shared by Gabriel Romualdo <https://xtrp.io/>
5+
"""Basic command line tool to find broken links in a website."""
86

9-
10-
"""Basic command line tool to find broken links in a web-site."""
7+
from __future__ import annotations
118

129
import argparse
10+
from collections.abc import Iterator, Sequence
11+
from dataclasses import dataclass
1312
import sys
1413
from urllib.parse import urldefrag, urljoin, urlparse
1514

1615
import bs4
1716
import requests
1817

18+
_SKIPPED_EXTENSIONS: tuple[str, ...] = (".png", ".jpg", ".jpeg", ".gif", ".ico")
19+
20+
21+
@dataclass(frozen=True)
1922
class BadLink:
20-
def __init__(self, bad_link, source, error):
21-
self.bad_link = bad_link
22-
self.source = source
23-
self.error = error
23+
bad_link: str
24+
source: str
25+
error: str
2426

25-
def __str__(self):
27+
def __str__(self) -> str:
2628
return f"{self.error}: link {self.bad_link} from {self.source}"
2729

2830

29-
searched_links = set()
30-
31-
32-
def getLinksFromHTML(html):
33-
return (
34-
el["href"]
35-
for el in bs4.BeautifulSoup(html, features="html.parser").select("a[href]")
36-
)
31+
def getLinksFromHTML(html: str) -> Iterator[str]:
32+
for el in bs4.BeautifulSoup(html, features="html.parser").select("a[href]"):
33+
href = el.get("href")
34+
if isinstance(href, str):
35+
yield href
3736

3837

3938
def find_broken_links(
40-
domainToSearch, target_url, parent_url="", depth=0, targetonly=False
41-
):
42-
broken_links = []
43-
error_links = []
39+
domain_to_search: str,
40+
target_url: str,
41+
parent_url: str = "",
42+
depth: int = 0,
43+
targetonly: bool = False,
44+
timeout: float = 10.0,
45+
session: requests.Session | None = None,
46+
searched_links: set[str] | None = None,
47+
) -> tuple[list[BadLink], list[BadLink]]:
48+
if searched_links is None:
49+
searched_links = set()
50+
if session is None:
51+
with requests.Session() as new_session:
52+
return find_broken_links(
53+
domain_to_search=domain_to_search,
54+
target_url=target_url,
55+
parent_url=parent_url,
56+
depth=depth,
57+
targetonly=targetonly,
58+
timeout=timeout,
59+
session=new_session,
60+
searched_links=searched_links,
61+
)
62+
63+
broken_links: list[BadLink] = []
64+
error_links: list[BadLink] = []
4465
target_url = urldefrag(target_url).url
4566

4667
if (
4768
target_url not in searched_links
4869
and not target_url.startswith("mailto:")
4970
and "javascript:" not in target_url
50-
and not target_url.endswith((".png", ".jpg", ".jpeg"))
71+
and not target_url.lower().endswith(_SKIPPED_EXTENSIONS)
5172
):
73+
searched_links.add(target_url)
5274
try:
53-
searched_links.add(target_url)
54-
requestObj = requests.get(target_url)
55-
if requestObj.status_code == 404:
75+
response = session.get(target_url, timeout=timeout)
76+
if response.status_code >= 400:
5677
broken_links.append(
5778
BadLink(
58-
bad_link=target_url, source=parent_url, error="Broken (404)"
79+
bad_link=target_url,
80+
source=parent_url,
81+
error=f"Broken ({response.status_code})",
5982
)
6083
)
61-
else:
62-
if urlparse(target_url).netloc == domainToSearch:
63-
for link in getLinksFromHTML(requestObj.text):
64-
if not targetonly or not depth:
65-
b, e = find_broken_links(
66-
domainToSearch=domainToSearch,
67-
target_url=urljoin(target_url, link),
68-
parent_url=target_url,
69-
depth=depth + 1,
70-
targetonly=targetonly,
71-
)
72-
broken_links.extend(b)
73-
error_links.extend(e)
74-
except Exception as e:
84+
elif (
85+
urlparse(target_url).netloc == domain_to_search
86+
and "text/html"
87+
in response.headers.get("Content-Type", "").lower()
88+
and (not targetonly or depth == 0)
89+
):
90+
for link in getLinksFromHTML(response.text):
91+
b, e = find_broken_links(
92+
domain_to_search=domain_to_search,
93+
target_url=urljoin(target_url, link),
94+
parent_url=target_url,
95+
depth=depth + 1,
96+
targetonly=targetonly,
97+
timeout=timeout,
98+
session=session,
99+
searched_links=searched_links,
100+
)
101+
broken_links.extend(b)
102+
error_links.extend(e)
103+
except requests.RequestException as exc:
75104
error_links.append(
76-
BadLink(bad_link=target_url, source=parent_url, error=f"Error: {e}")
105+
BadLink(
106+
bad_link=target_url, source=parent_url, error=f"Error: {exc}"
107+
)
77108
)
78109

79-
return (broken_links, error_links)
110+
return broken_links, error_links
80111

81112

82-
def main(target_url, targetonly):
113+
def run_check(
114+
target_url: str, targetonly: bool = False, timeout: float = 10.0
115+
) -> int:
83116
print(f"Looking for broken links in {target_url}")
117+
searched_links: set[str] = set()
84118
broken_links, error_links = find_broken_links(
85-
domainToSearch=urlparse(target_url).netloc,
119+
domain_to_search=urlparse(target_url).netloc,
86120
target_url=target_url,
87121
targetonly=targetonly,
122+
timeout=timeout,
123+
searched_links=searched_links,
88124
)
89125
print("\n--- DONE! ---\n")
90126
print("The following links returned errors:")
@@ -97,20 +133,33 @@ def main(target_url, targetonly):
97133

98134
print(
99135
f"\nLinks Checked: {len(searched_links)} "
100-
f"Broken: {len(broken_links)} Error: {len(error_links)} "
136+
f"Broken: {len(broken_links)} Error: {len(error_links)}"
101137
)
102138
return len(error_links) + len(broken_links)
103139

104140

105-
if __name__ == "__main__":
106-
parser = argparse.ArgumentParser()
141+
def main(argv: Sequence[str] | None = None) -> int:
142+
parser = argparse.ArgumentParser(description=__doc__)
107143
parser.add_argument(
108144
"-t",
109145
"--targetonly",
110-
default=False,
111146
action="store_true",
112147
help="Only check links in target URL",
113148
)
114-
parser.add_argument("target")
115-
args = parser.parse_args()
116-
sys.exit(main(target_url=args.target, targetonly=args.targetonly))
149+
parser.add_argument(
150+
"--timeout",
151+
type=float,
152+
default=10.0,
153+
help="Per-request HTTP timeout in seconds (default: 10)",
154+
)
155+
parser.add_argument("target", help="Target URL to check")
156+
args = parser.parse_args(argv)
157+
return run_check(
158+
target_url=args.target,
159+
targetonly=args.targetonly,
160+
timeout=args.timeout,
161+
)
162+
163+
164+
if __name__ == "__main__":
165+
sys.exit(main())

0 commit comments

Comments
 (0)