クロールは「リンク探索」から卒業する ― Sitemap起点のURL発見設計

import re
import requests


def discover_sitemaps(base_url: str):
    robots_url = base_url.rstrip("/") + "/robots.txt"
    r = requests.get(robots_url, timeout=20)
    r.raise_for_status()

    sitemaps = []
    for line in r.text.splitlines():
        m = re.match(r"(?i)^sitemap:\s*(\S+)", line.strip())
        if m:
            sitemaps.append(m.group(1))
    return sitemaps


print(discover_sitemaps("https://example.com"))

import requests
import xml.etree.ElementTree as ET

NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}


def fetch_xml(url: str) -> ET.Element:
    r = requests.get(url, timeout=30)
    r.raise_for_status()
    return ET.fromstring(r.content)


def parse_sitemap(url: str):
    root = fetch_xml(url)

    # sitemapindex
    if root.tag.endswith("sitemapindex"):
        locs = [e.find("sm:loc", NS).text for e in root.findall("sm:sitemap", NS)]
        urls = []
        for loc in locs:
            urls.extend(parse_sitemap(loc))
        return urls

    # urlset
    if root.tag.endswith("urlset"):
        out = []
        for u in root.findall("sm:url", NS):
            loc = u.find("sm:loc", NS).text
            lastmod_el = u.find("sm:lastmod", NS)
            lastmod = lastmod_el.text if lastmod_el is not None else None
            out.append({"loc": loc, "lastmod": lastmod})
        return out

    raise ValueError(f"Unknown sitemap root: {root.tag}")


items = parse_sitemap("https://example.com/sitemap.xml")
print(len(items), items[:3])

from urllib.parse import urlparse, urlunparse


def normalize_url(u: str) -> str:
    p = urlparse(u)
    # クエリやフラグメントを落とす（要件次第で保持）
    p = p._replace(query="", fragment="")
    # 末尾スラッシュ統一はサイト仕様に合わせる
    return urlunparse(p)


def filter_urls(items, allow_prefixes):
    out = []
    for it in items:
        loc = normalize_url(it["loc"])
        path = urlparse(loc).path
        if any(path.startswith(prefix) for prefix in allow_prefixes):
            out.append({**it, "loc": loc})
    return out


filtered = filter_urls(items, allow_prefixes=["/product/", "/item/"])
print(len(filtered))