Audit Title and Meta Description Lengths from a Sitemap with Python (Standard Library Only)

#!/usr/bin/env python3 """Audit <title> and meta description lengths for every URL in an XML sitemap. Standard library only. Usage: python3 title_meta_audit.py https://example.com/sitemap.xml --out report.csv python3 title_meta_audit.py /path/to/sitemap.xml --out report.csv python3 title_meta_audit.py sitemap.xml --limit 20 Practical length bands (NOT Google rules; Google documents no fixed character limit and truncates to fit the device). Defaults: title: 30-60 characters meta description: 70-160 characters Override with --title-min/--title-max/--desc-min/--desc-max. """ import argparse import csv import gzip import sys import threading import time import urllib.error import urllib.request import xml.etree.ElementTree as ET from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from html.parser import HTMLParser NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}" UA = "title-meta-audit/1.0 (+https://example.com/contact)" class TitleMetaParser(HTMLParser): """Collect <title> text and name=description meta content from <head>.""" def __init__(self): super().__init__() self.title_parts = [] self.description = None self._in_head = False self._in_title = False self._done = False def handle_starttag(self, tag, attrs): if self._done: return name = tag.lower() attr = {k.lower(): (v or "") for k, v in attrs} if name == "head": self._in_head = True return if name == "body": self._done = True return if not self._in_head: return if name == "title": self._in_title = True return if name == "meta": meta_name = (attr.get("name") or "").strip().lower() if meta_name == "description" and self.description is None: self.description = attr.get("content", "") def handle_endtag(self, tag): name = tag.lower() if name == "title": self._in_title = False if name == "head": self._in_head = False self._done = True def handle_data(self, data): if self._in_title and not self._done: self.title_parts.append(data) @property def title(self): return "".join(self.title_parts).strip() def fetch_bytes(src, timeout): if src.startswith(("http://", "https://")): req = urllib.request.Request(src, headers={"User-Agent": UA}) with urllib.request.urlopen(req, timeout=timeout) as resp: data = resp.read() else: with open(src, "rb") as fh: data = fh.read() return gzip.decompress(data) if data[:2] == b"\x1f\x8b" else data def sitemap_urls(src, timeout, seen=None, depth=0): seen = seen if seen is not None else set() if src in seen or depth > 3: return [] seen.add(src) root = ET.fromstring(fetch_bytes(src, timeout)) locs = [el.text.strip() for el in root.iter(NS + "loc") if el.text] if root.tag == NS + "sitemapindex": urls = [] for child in locs: urls += sitemap_urls(child, timeout, seen, depth + 1) return urls return locs def fetch_html(url, timeout): req = urllib.request.Request(url, headers={"User-Agent": UA}) with urllib.request.urlopen(req, timeout=timeout) as resp: charset = resp.headers.get_content_charset() or "utf-8" return resp.read().decode(charset, errors="replace"), resp.geturl(), resp.status def classify(title, description, title_min, title_max, desc_min, desc_max): issues = [] title_len = len(title) if title is not None else 0 desc_len = len(description) if description is not None else 0 if not title: issues.append("missing title") else: if title_len < title_min: issues.append(f"title too short ({title_len} < {title_min})") elif title_len > title_max: issues.append(f"title too long ({title_len} > {title_max})") if description is None: issues.append("missing meta description") elif description.strip() == "": issues.append("empty meta description") else: if desc_len < desc_min: issues.append(f"description too short ({desc_len} < {desc_min})") elif desc_len > desc_max: issues.append(f"description too long ({desc_len} > {desc_max})") return issues, title_len, desc_len def audit_one(url, timeout, bands): title_min, title_max, desc_min, desc_max = bands try: html, final_url, status = fetch_html(url, timeout) except urllib.error.HTTPError as e: return { "url": url, "final_url": "", "status": e.code, "title": "", "title_len": 0, "description": "", "desc_len": 0, "issues": f"HTTP error {e.code}", } except Exception as e: return { "url": url, "final_url": "", "status": 0, "title": "", "title_len": 0, "description": "", "desc_len": 0, "issues": f"error: {e}", } parser = TitleMetaParser() parser.feed(html) title = parser.title description = parser.description issues, title_len, desc_len = classify( title, description, title_min, title_max, desc_min, desc_max ) return { "url": url, "final_url": final_url, "status": status, "title": title, "title_len": title_len, "description": description if description is not None else "", "desc_len": desc_len, "issues": "; ".join(issues) if issues else "", } def main(argv=None): p = argparse.ArgumentParser(description=__doc__) p.add_argument("sitemap", help="Sitemap URL or local XML path") p.add_argument("--out", default="title_meta_report.csv") p.add_argument("--timeout", type=float, default=15.0) p.add_argument("--workers", type=int, default=4) p.add_argument("--delay", type=float, default=0.25, help="Seconds between request starts") p.add_argument("--limit", type=int, default=0, help="Audit at most N URLs (0 = all)") p.add_argument("--title-min", type=int, default=30) p.add_argument("--title-max", type=int, default=60) p.add_argument("--desc-min", type=int, default=70) p.add_argument("--desc-max", type=int, default=160) args = p.parse_args(argv) bands = (args.title_min, args.title_max, args.desc_min, args.desc_max) try: urls = sitemap_urls(args.sitemap, args.timeout) except Exception as e: print(f"Failed to read sitemap: {e}", file=sys.stderr) return 2 if args.limit and args.limit > 0: urls = urls[: args.limit] print( f"auditing {len(urls)} URLs with {args.workers} workers " f"(title {args.title_min}-{args.title_max}, " f"description {args.desc_min}-{args.desc_max}; practical bands)" ) lock = threading.Lock() last_start = [0.0] rows = [] def gated(url): with lock: now = time.monotonic() wait = args.delay - (now - last_start[0]) if wait > 0: time.sleep(wait) last_start[0] = time.monotonic() return audit_one(url, args.timeout, bands) with ThreadPoolExecutor(max_workers=args.workers) as pool: futures = {pool.submit(gated, u): u for u in urls} for fut in as_completed(futures): rows.append(fut.result()) # Preserve sitemap order  by_url = {r["url"]: r for r in rows} ordered = [by_url[u] for u in urls if u in by_url] # Duplicate title / description across pages (after all fetches)  title_map = defaultdict(list) desc_map = defaultdict(list) for r in ordered: if r["title"]: title_map[r["title"]].append(r["url"]) if r["description"]: desc_map[r["description"]].append(r["url"]) for r in ordered: extras = [] if r["title"] and len(title_map[r["title"]]) > 1: extras.append("duplicate title") if r["description"] and len(desc_mapSource：https://dev.to/appierun_66/audit-title-and-meta-description-lengths-from-a-sitemap-with-python-standard-library-only-eni
My favorites