"""爬取中医方剂大全的方剂列表：方剂名、详情页链接、所属分类，保存到 csv。

页面结构（2026-07 实测）：和中药大全一致——div.p_content 下按 h2 分成
20 个分类（解表剂、泻下剂、和解剂 ...），每个 h2 之后跟着若干
<a href="/wiki/xxx">方剂名</a>。共 721 条，其中 18 条是同一方剂
挂在两个分类下。

用法（在项目根目录执行）：
    python -m _001__clawler.03_formula_list
"""

import csv
import time

import requests
from bs4 import BeautifulSoup

from common.path_utils import get_file_path

BASE_URL = "https://zhongyibaike.com"
LIST_URL = "https://zhongyibaike.com/wiki/%E4%B8%AD%E5%8C%BB%E6%96%B9%E5%89%82"
OUTPUT_CSV = get_file_path("_001__clawler/formula_list.csv")

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    )
}


def fetch_html(url, retries=3, timeout=30):
    """下载页面 html，失败时重试。"""
    for attempt in range(1, retries + 1):
        try:
            response = requests.get(url, headers=HEADERS, timeout=timeout)
            response.raise_for_status()
            # 站点返回 utf-8，显式指定避免 requests 猜错编码导致中文乱码
            response.encoding = "utf-8"
            return response.text
        except requests.RequestException as exc:
            print(f"第 {attempt}/{retries} 次请求失败: {exc}")
            if attempt == retries:
                raise
            time.sleep(2 * attempt)


def parse_formula_list(html):
    """从列表页 html 中解析出 [{分类, 方剂名, 链接}, ...]，并按链接去重。

    用 descendants 顺序遍历而不是 find_all('a')，这样才能知道每个链接
    落在哪个 h2 分类下面。
    """
    soup = BeautifulSoup(html, "html.parser")
    content = soup.find("div", class_="p_content")
    if content is None:
        raise RuntimeError("找不到 div.p_content，页面结构可能已经改版")

    formulas = []
    seen = set()
    category = None
    for element in content.descendants:
        name = getattr(element, "name", None)
        if name == "h2":
            category = element.get_text(strip=True)
        elif name == "a" and element.get("href", "").startswith("/wiki/"):
            formula_name = element.get_text(strip=True)
            href = element["href"]
            if not formula_name or href in seen:
                continue
            seen.add(href)
            formulas.append(
                {
                    "分类": category or "",
                    "方剂名": formula_name,
                    "链接": BASE_URL + href,
                }
            )
    return formulas


def save_to_csv(formulas, output_path):
    """写 csv。utf-8-sig 让 Excel 打开中文不乱码。"""
    with open(output_path, "w", encoding="utf-8-sig", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=["分类", "方剂名", "链接"])
        writer.writeheader()
        writer.writerows(formulas)


def main():
    print(f"正在抓取列表页: {LIST_URL}")
    html = fetch_html(LIST_URL)
    #分类 ：方剂  链接
    formulas = parse_formula_list(html)
    categories = {formula["分类"] for formula in formulas}
    print(f"解析到 {len(formulas)} 个方剂，{len(categories)} 个分类")

    save_to_csv(formulas, OUTPUT_CSV)
    print(f"已保存到: {OUTPUT_CSV}")


if __name__ == "__main__":
    main()