11#!/usr/bin/env python3
2- # -*- coding: utf-8 -*-
2+ # Based on a CC0 licensed script shared by Gabriel Romualdo <https://xtrp.io/>
33
4- # Note: links below are broken.
5- # Based on a CC0 licensed script
64# <https://xtrp.io/blog/2019/11/09/a-quick-python-script-to-find-broken-links/>
7- # Shared by Gabriel Romualdo <https://xtrp.io/>
5+ """Basic command line tool to find broken links in a website."""
86
9-
10- """Basic command line tool to find broken links in a web-site."""
7+ from __future__ import annotations
118
129import argparse
10+ from collections .abc import Iterator , Sequence
11+ from dataclasses import dataclass
1312import sys
1413from urllib .parse import urldefrag , urljoin , urlparse
1514
1615import bs4
1716import requests
1817
18+ _SKIPPED_EXTENSIONS : tuple [str , ...] = (".png" , ".jpg" , ".jpeg" , ".gif" , ".ico" )
19+
20+
21+ @dataclass (frozen = True )
1922class BadLink :
20- def __init__ (self , bad_link , source , error ):
21- self .bad_link = bad_link
22- self .source = source
23- self .error = error
23+ bad_link : str
24+ source : str
25+ error : str
2426
25- def __str__ (self ):
27+ def __str__ (self ) -> str :
2628 return f"{ self .error } : link { self .bad_link } from { self .source } "
2729
2830
29- searched_links = set ()
30-
31-
32- def getLinksFromHTML (html ):
33- return (
34- el ["href" ]
35- for el in bs4 .BeautifulSoup (html , features = "html.parser" ).select ("a[href]" )
36- )
31+ def getLinksFromHTML (html : str ) -> Iterator [str ]:
32+ for el in bs4 .BeautifulSoup (html , features = "html.parser" ).select ("a[href]" ):
33+ href = el .get ("href" )
34+ if isinstance (href , str ):
35+ yield href
3736
3837
3938def find_broken_links (
40- domainToSearch , target_url , parent_url = "" , depth = 0 , targetonly = False
41- ):
42- broken_links = []
43- error_links = []
39+ domain_to_search : str ,
40+ target_url : str ,
41+ parent_url : str = "" ,
42+ depth : int = 0 ,
43+ targetonly : bool = False ,
44+ timeout : float = 10.0 ,
45+ session : requests .Session | None = None ,
46+ searched_links : set [str ] | None = None ,
47+ ) -> tuple [list [BadLink ], list [BadLink ]]:
48+ if searched_links is None :
49+ searched_links = set ()
50+ if session is None :
51+ with requests .Session () as new_session :
52+ return find_broken_links (
53+ domain_to_search = domain_to_search ,
54+ target_url = target_url ,
55+ parent_url = parent_url ,
56+ depth = depth ,
57+ targetonly = targetonly ,
58+ timeout = timeout ,
59+ session = new_session ,
60+ searched_links = searched_links ,
61+ )
62+
63+ broken_links : list [BadLink ] = []
64+ error_links : list [BadLink ] = []
4465 target_url = urldefrag (target_url ).url
4566
4667 if (
4768 target_url not in searched_links
4869 and not target_url .startswith ("mailto:" )
4970 and "javascript:" not in target_url
50- and not target_url .endswith (( ".png" , ".jpg" , ".jpeg" ) )
71+ and not target_url .lower (). endswith ( _SKIPPED_EXTENSIONS )
5172 ):
73+ searched_links .add (target_url )
5274 try :
53- searched_links .add (target_url )
54- requestObj = requests .get (target_url )
55- if requestObj .status_code == 404 :
75+ response = session .get (target_url , timeout = timeout )
76+ if response .status_code >= 400 :
5677 broken_links .append (
5778 BadLink (
58- bad_link = target_url , source = parent_url , error = "Broken (404)"
79+ bad_link = target_url ,
80+ source = parent_url ,
81+ error = f"Broken ({ response .status_code } )" ,
5982 )
6083 )
61- else :
62- if urlparse (target_url ).netloc == domainToSearch :
63- for link in getLinksFromHTML (requestObj .text ):
64- if not targetonly or not depth :
65- b , e = find_broken_links (
66- domainToSearch = domainToSearch ,
67- target_url = urljoin (target_url , link ),
68- parent_url = target_url ,
69- depth = depth + 1 ,
70- targetonly = targetonly ,
71- )
72- broken_links .extend (b )
73- error_links .extend (e )
74- except Exception as e :
84+ elif (
85+ urlparse (target_url ).netloc == domain_to_search
86+ and "text/html"
87+ in response .headers .get ("Content-Type" , "" ).lower ()
88+ and (not targetonly or depth == 0 )
89+ ):
90+ for link in getLinksFromHTML (response .text ):
91+ b , e = find_broken_links (
92+ domain_to_search = domain_to_search ,
93+ target_url = urljoin (target_url , link ),
94+ parent_url = target_url ,
95+ depth = depth + 1 ,
96+ targetonly = targetonly ,
97+ timeout = timeout ,
98+ session = session ,
99+ searched_links = searched_links ,
100+ )
101+ broken_links .extend (b )
102+ error_links .extend (e )
103+ except requests .RequestException as exc :
75104 error_links .append (
76- BadLink (bad_link = target_url , source = parent_url , error = f"Error: { e } " )
105+ BadLink (
106+ bad_link = target_url , source = parent_url , error = f"Error: { exc } "
107+ )
77108 )
78109
79- return ( broken_links , error_links )
110+ return broken_links , error_links
80111
81112
82- def main (target_url , targetonly ):
113+ def run_check (
114+ target_url : str , targetonly : bool = False , timeout : float = 10.0
115+ ) -> int :
83116 print (f"Looking for broken links in { target_url } " )
117+ searched_links : set [str ] = set ()
84118 broken_links , error_links = find_broken_links (
85- domainToSearch = urlparse (target_url ).netloc ,
119+ domain_to_search = urlparse (target_url ).netloc ,
86120 target_url = target_url ,
87121 targetonly = targetonly ,
122+ timeout = timeout ,
123+ searched_links = searched_links ,
88124 )
89125 print ("\n --- DONE! ---\n " )
90126 print ("The following links returned errors:" )
@@ -97,20 +133,33 @@ def main(target_url, targetonly):
97133
98134 print (
99135 f"\n Links Checked: { len (searched_links )} "
100- f"Broken: { len (broken_links )} Error: { len (error_links )} "
136+ f"Broken: { len (broken_links )} Error: { len (error_links )} "
101137 )
102138 return len (error_links ) + len (broken_links )
103139
104140
105- if __name__ == "__main__" :
106- parser = argparse .ArgumentParser ()
141+ def main ( argv : Sequence [ str ] | None = None ) -> int :
142+ parser = argparse .ArgumentParser (description = __doc__ )
107143 parser .add_argument (
108144 "-t" ,
109145 "--targetonly" ,
110- default = False ,
111146 action = "store_true" ,
112147 help = "Only check links in target URL" ,
113148 )
114- parser .add_argument ("target" )
115- args = parser .parse_args ()
116- sys .exit (main (target_url = args .target , targetonly = args .targetonly ))
149+ parser .add_argument (
150+ "--timeout" ,
151+ type = float ,
152+ default = 10.0 ,
153+ help = "Per-request HTTP timeout in seconds (default: 10)" ,
154+ )
155+ parser .add_argument ("target" , help = "Target URL to check" )
156+ args = parser .parse_args (argv )
157+ return run_check (
158+ target_url = args .target ,
159+ targetonly = args .targetonly ,
160+ timeout = args .timeout ,
161+ )
162+
163+
164+ if __name__ == "__main__" :
165+ sys .exit (main ())
0 commit comments