如何抓取维基百科. 完整初学者教程

# Create virtual environment
python -m venv wikipedia-env

# Activate it
# macOS/Linux:
source wikipedia-env/bin/activate

# Windows Command Prompt:
wikipedia-env\Scripts\activate.bat

# Windows PowerShell:
wikipedia-env\Scripts\Activate.ps1

pip install requests beautifulsoup4 lxml html2text pandas

pip freeze > requirements.txt

import requests
from bs4 import BeautifulSoup
import html2text
import pandas as pd
import io, os, re, json, random
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

# Rotate user agents to mimic different browsers
USER_AGENTS = [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
]


def get_session():
    """Create a requests session with automatic retry on server errors"""
    session = requests.Session()
    # Retry on server errors (5xx) and rate limit errors (429)
    retry = Retry(
        total=3,
        backoff_factor=1,
        status_forcelist=[429, 500, 502, 503, 504],
        allowed_methods=["GET"],
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount("https://", adapter)
    return session


SESSION = get_session()

def extract_infobox(soup):
    """Extract structured data from Wikipedia infobox"""
    box = soup.select_one("table.infobox")
    if not box:
        return None

    data = {}

    # Extract title
    title = box.select_one(".infobox-above, .fn")
    if title:
        data["title"] = title.get_text(strip=True)

    # Extract key-value pairs
    for row in box.find_all("tr"):
        label = row.select_one("th.infobox-label, th.infobox-header")
        value = row.select_one("td.infobox-data, td")
        if label and value:
            # Regex cleaning: Remove special chars to make a valid JSON key
            key = re.sub(r"[^\w\s-]", "", label.get_text(strip=True)).strip()
            if key:
                data[key] = value.get_text(separator=" ", strip=True)

    return data

def extract_tables(soup, folder):
    """Extract Wikipedia tables and save as CSV files"""
    tables = soup.select("table.wikitable")
    if not tables:
        return 0

    os.makedirs(f"{folder}/tables", exist_ok=True)
    table_count = 0

    for table in tables:
        try:
            dfs = pd.read_html(io.StringIO(str(table)))
            if dfs:
                table_count += 1
                dfs[0].to_csv(f"{folder}/tables/table_{table_count}.csv", index=False)
        except Exception:
            pass
    return table_count

def scrape_page(url):
    # 1. Fetch
    headers = {"User-Agent": random.choice(USER_AGENTS)}
    try:
        response = SESSION.get(url, headers=headers, timeout=15)
    except Exception as e:
        print(f"Error: {e}")
        return None

    # 2. Parse
    soup = BeautifulSoup(response.content, "lxml")

    # 3. Setup output folder
    title_elem = soup.find("h1", id="firstHeading")
    title = title_elem.get_text(strip=True) if title_elem else "Unknown"

    safe_title = re.sub(r"[^\w\-_]", "_", title)
    output_folder = f"Output_{safe_title}"
    os.makedirs(output_folder, exist_ok=True)

    # 4. Extract structured data (before cleaning!)
    infobox = extract_infobox(soup)
    if infobox:
        with open(f"{output_folder}/infobox.json", "w", encoding="utf-8") as f:
            json.dump(infobox, f, indent=2, ensure_ascii=False)

    extract_tables(soup, output_folder)

    # 5. Clean and convert to Markdown
    content = soup.select_one("#mw-content-text .mw-parser-output")

    # Remove noise elements so they don't appear in the text
    junk = [".navbox", ".reflist", ".reference", ".hatnote", ".ambox"]
    for selector in junk:
        for el in content.select(selector):
            el.decompose()

    h = html2text.HTML2Text()
    h.body_width = 0  # No wrapping
    markdown_content = h.handle(str(content))

    with open(f"{output_folder}/content.md", "w", encoding="utf-8") as f:
        f.write(f"# {title}\n\n{markdown_content}")

    print(f"Scraped: {title}")
    return {"soup": soup, "title": title}


if __name__ == "__main__":
    scrape_page("https://en.wikipedia.org/wiki/Google")

python wiki_scraper.py

{
  "title": "Google LLC",
  "Formerly": "Google Inc. (1998-2017)",
  "Company type": "Subsidiary",
  "Traded as": "Nasdaq : GOOGL Nasdaq : GOOG",
  "Industry": "Internet Cloud computing Computer software Computer hardware Artificial intelligence Advertising",
  "Founded": "September 4, 1998 ; 27 years ago ( 1998-09-04 ) [ a ] in Menlo Park , California , United States",
  "Founders": "Larry Page Sergey Brin",
  "Headquarters": "Googleplex , Mountain View, California , U.S.",
  "Area served": "Worldwide",
  "Key people": "John L. Hennessy ( Chairman ) Sundar Pichai ( CEO ) Ruth Porat ( President and CIO ) Anat Ashkenazi ( CFO )",
  "Products": "Google Search Android Nest Pixel Workspace Fitbit Waze YouTube Gemini Full list",
  "Number of employees": "187,000 (2022)",
  "Parent": "Alphabet Inc.",
  "Subsidiaries": "Adscape Cameyo Charleston Road Registry Endoxon FeedBurner ImageAmerica Kaltix Nest Labs reCAPTCHA X Development YouTube ZipDash",
  "ASN": "15169",
  "Website": "about .google"
}

SN,City,Country or U.S. state
1.0,Ann Arbor,Michigan
2.0,Atlanta,Georgia
3.0,Austin,Texas
4.0,Boulder,Colorado
5.0,Boulder - Pearl Place,Colorado
6.0,Boulder - Walnut,Colorado
7.0,Cambridge,Massachusetts
8.0,Chapel Hill,North Carolina
9.0,Chicago - Carpenter,Illinois
10.0,Chicago - Fulton Market,Illinois

# Google

**Google LLC** ([/ˈɡuː.ɡəl/](/wiki/Help:IPA/English "Help:IPA/English") [](//upload.wikimedia.org/wikipedia/commons/transcoded/3/3d/En-us-googol.ogg/En-us-googol.ogg.mp3 "Play audio")[ⓘ](/wiki/File:En-us-googol.ogg "File:En-us-googol.ogg"), [_GOO -gəl_](/wiki/Help:Pronunciation_respelling_key "Help:Pronunciation respelling key")) is an American multinational technology corporation focused on...

Google was founded on September 4, 1998, by American computer scientists [Larry Page](/wiki/Larry_Page "Larry Page") and [Sergey Brin](/wiki/Sergey_Brin "Sergey Brin"). Together, they own about 14% of its publicly listed shares and control 56% of its stockholder voting power through [super-voting stock](/wiki/Super-voting_stock "Super-voting stock"). The company went [public](/wiki/Public_company "Public company") via an [initial public offering](/wiki/Initial_public_offering "Initial public offering") (IPO) in 2004...

## History

### Early years

Google began in January 1996 as a research project by [Larry Page](/wiki/Larry_Page "Larry Page") and [Sergey Brin](/wiki/Sergey_Brin "Sergey Brin") while they were both [PhD](/wiki/PhD "PhD") students at [Stanford University](/wiki/Stanford_University "Stanford University") in [California](/wiki/California "California"), United States...

import requests
from bs4 import BeautifulSoup
import html2text
import pandas as pd
import io, os, re, json, random, time
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from urllib.parse import urlparse, urljoin
from collections import deque
import argparse

def normalize_url(url):
    """Standardizes URLs (removes fragments like #history)"""
    if not url:
        return None
    # Handle protocol-relative URLs (common on Wikipedia)
    if url.startswith("//"):
        url = "https:" + url
    parsed = urlparse(url)
    return f"{parsed.scheme}://{parsed.netloc.lower()}{parsed.path}"

def is_valid_wikipedia_link(url):
    """Filters out special pages (Files, Talk, Help)"""
    if not url:
        return False
    parsed = urlparse(url)
    if "wikipedia.org" not in parsed.netloc:
        return False

    # We only want articles, not maintenance pages
    skip = [
        "/wiki/Special:",
        "/wiki/File:",
        "/wiki/Help:",
        "/wiki/User:",
        "/wiki/Talk:",
    ]
    return not any(parsed.path.startswith(p) for p in skip)

def extract_links(soup, base_url):
    links = set()
    # Target the main article body
    content = soup.select_one("#mw-content-text .mw-parser-output")
    if not content:
        return links

    # 1. Early paragraphs: Scan only first 3 paragraphs for high-level concepts
    for p in content.find_all("p", recursive=False, limit=3):
        for link in p.find_all("a", href=True):
            url = urljoin(base_url, link["href"])
            if is_valid_wikipedia_link(url):
                links.add(normalize_url(url))

    # 2. "See Also": Find the header and grab all links in that section
    for heading in soup.find_all(["h2", "h3"]):
        if "see also" in heading.get_text().lower():
            # Get all elements after the heading until the next heading
            current = heading.find_next_sibling()
            while current and current.name not in ["h2", "h3"]:
                for link in current.find_all("a", href=True):
                    url = urljoin(base_url, link["href"])
                    if is_valid_wikipedia_link(url):
                        links.add(normalize_url(url))
                current = current.find_next_sibling()
            break  # Stop once we've processed the section
    return links

class WikipediaCrawler:
    def __init__(self, start_url, max_pages=5, max_depth=2):
        # The queue stores tuples: (URL, Depth)
        self.queue = deque([(normalize_url(start_url), 0)])
        self.visited = set()
        self.max_pages = max_pages
        self.max_depth = max_depth

    def crawl(self):
        count = 0
        while self.queue and count < self.max_pages:
            # Get the next URL from the front of the queue
            url, depth = self.queue.popleft()

            # Skip if we've already scraped this
            if url in self.visited:
                continue

            # Skip if we've exceeded max depth
            if depth > self.max_depth:
                continue

            print(f"[{count+1}/{self.max_pages}] [Depth {depth}] Crawling: {url}")

            # 1. Scrape the page
            data = scrape_page(url)
            self.visited.add(url)
            count += 1

            # 2. Find new links (if the scrape was successful)
            if data and data.get("soup"):
                new_links = extract_links(data["soup"], url)
                for link in new_links:
                    if link not in self.visited:
                        self.queue.append((link, depth + 1))

            # 3. Rate limiting
            time.sleep(1.5)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("url", help="Wikipedia article URL")
    parser.add_argument("--crawl", action="store_true", help="Enable crawler mode")
    parser.add_argument(
        "--max-pages", type=int, default=5, help="Maximum pages to scrape"
    )
    parser.add_argument("--max-depth", type=int, default=2, help="Maximum crawl depth")
    args = parser.parse_args()

    if args.crawl:
        WikipediaCrawler(
            args.url, max_pages=args.max_pages, max_depth=args.max_depth
        ).crawl()
    else:
        scrape_page(args.url)

python wiki_scraper.py "https://en.wikipedia.org/wiki/Google"

python wiki_scraper.py "https://en.wikipedia.org/wiki/Google" --crawl --max-pages 10 --max-depth 2

[1/10] [Depth 0] Crawling: https://en.wikipedia.org/wiki/Google
Scraped: Google
[2/10] [Depth 1] Crawling: https://en.wikipedia.org/wiki/Initial_public_offering
Scraped: Initial public offering
[3/10] [Depth 1] Crawling: https://en.wikipedia.org/wiki/E-commerce
Scraped: E-commerce
[4/10] [Depth 1] Crawling: https://en.wikipedia.org/wiki/Search_engine
Scraped: Search engine
[5/10] [Depth 1] Crawling: https://en.wikipedia.org/wiki/BBC
Scraped: BBC
[6/10] [Depth 1] Crawling: https://en.wikipedia.org/wiki/Public_company
Scraped: Public company
[7/10] [Depth 1] Crawling: https://en.wikipedia.org/wiki/Super-voting_stock
Scraped: Super-voting stock
[8/10] [Depth 1] Crawling: https://en.wikipedia.org/wiki/Larry_Page
Scraped: Larry Page
[9/10] [Depth 1] Crawling: https://en.wikipedia.org/wiki/Sundar_Pichai
Scraped: Sundar Pichai
[10/10] [Depth 1] Crawling: https://en.wikipedia.org/wiki/Sergey_Brin
Scraped: Sergey Brin

import requests
from bs4 import BeautifulSoup
import html2text
import pandas as pd
import io, os, re, json, random, time
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from urllib.parse import urlparse, urljoin
from collections import deque
import argparse

# Rotate user agents to mimic different browsers
USER_AGENTS = [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
]


def get_session():
    """Create a requests session with automatic retry on server errors"""
    session = requests.Session()
    # Retry on server errors (5xx) and rate limit errors (429)
    retry = Retry(
        total=3,
        backoff_factor=1,
        status_forcelist=[429, 500, 502, 503, 504],
        allowed_methods=["GET"],
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount("https://", adapter)
    return session


SESSION = get_session()


def extract_infobox(soup):
    """Extract structured data from Wikipedia infobox"""
    box = soup.select_one("table.infobox")
    if not box:
        return None

    data = {}

    # Extract title
    title = box.select_one(".infobox-above, .fn")
    if title:
        data["title"] = title.get_text(strip=True)

    # Extract key-value pairs
    for row in box.find_all("tr"):
        label = row.select_one("th.infobox-label, th.infobox-header")
        value = row.select_one("td.infobox-data, td")
        if label and value:
            # Regex cleaning: Remove special chars to make a valid JSON key
            key = re.sub(r"[^\w\s-]", "", label.get_text(strip=True)).strip()
            if key:
                data[key] = value.get_text(separator=" ", strip=True)

    return data


def extract_tables(soup, folder):
    """Extract Wikipedia tables and save as CSV files"""
    tables = soup.select("table.wikitable")
    if not tables:
        return 0

    os.makedirs(f"{folder}/tables", exist_ok=True)
    table_count = 0

    for table in tables:
        try:
            dfs = pd.read_html(io.StringIO(str(table)))
            if dfs:
                table_count += 1
                dfs[0].to_csv(f"{folder}/tables/table_{table_count}.csv", index=False)
        except Exception:
            pass
    return table_count


def normalize_url(url):
    """Standardizes URLs (removes fragments like #history)"""
    if not url:
        return None
    # Handle protocol-relative URLs (common on Wikipedia)
    if url.startswith("//"):
        url = "https:" + url
    parsed = urlparse(url)
    return f"{parsed.scheme}://{parsed.netloc.lower()}{parsed.path}"


def is_valid_wikipedia_link(url):
    """Filters out special pages (Files, Talk, Help)"""
    if not url:
        return False
    parsed = urlparse(url)
    if "wikipedia.org" not in parsed.netloc:
        return False

    # We only want articles, not maintenance pages
    skip = [
        "/wiki/Special:",
        "/wiki/File:",
        "/wiki/Help:",
        "/wiki/User:",
        "/wiki/Talk:",
    ]
    return not any(parsed.path.startswith(p) for p in skip)


def extract_links(soup, base_url):
    links = set()
    # Target the main article body
    content = soup.select_one("#mw-content-text .mw-parser-output")
    if not content:
        return links

    # 1. Early paragraphs: Scan only first 3 paragraphs for high-level concepts
    for p in content.find_all("p", recursive=False, limit=3):
        for link in p.find_all("a", href=True):
            url = urljoin(base_url, link["href"])
            if is_valid_wikipedia_link(url):
                links.add(normalize_url(url))

    # 2. "See Also": Find the header and grab all links in that section
    for heading in soup.find_all(["h2", "h3"]):
        if "see also" in heading.get_text().lower():
            # Get all elements after the heading until the next heading
            current = heading.find_next_sibling()
            while current and current.name not in ["h2", "h3"]:
                for link in current.find_all("a", href=True):
                    url = urljoin(base_url, link["href"])
                    if is_valid_wikipedia_link(url):
                        links.add(normalize_url(url))
                current = current.find_next_sibling()
            break  # Stop once we've processed the section
    return links


class WikipediaCrawler:
    def __init__(self, start_url, max_pages=5, max_depth=2):
        # The queue stores tuples: (URL, Depth)
        self.queue = deque([(normalize_url(start_url), 0)])
        self.visited = set()
        self.max_pages = max_pages
        self.max_depth = max_depth

    def crawl(self):
        count = 0
        while self.queue and count < self.max_pages:
            # Get the next URL from the front of the queue
            url, depth = self.queue.popleft()

            # Skip if we've already scraped this
            if url in self.visited:
                continue

            # Skip if we've exceeded max depth
            if depth > self.max_depth:
                continue

            print(f"[{count+1}/{self.max_pages}] [Depth {depth}] Crawling: {url}")

            # 1. Scrape the page
            data = scrape_page(url)
            self.visited.add(url)
            count += 1

            # 2. Find new links (if the scrape was successful)
            if data and data.get("soup"):
                new_links = extract_links(data["soup"], url)
                for link in new_links:
                    if link not in self.visited:
                        self.queue.append((link, depth + 1))

            # 3. Rate limiting
            time.sleep(1.5)


def scrape_page(url):
    # 1. Fetch
    headers = {"User-Agent": random.choice(USER_AGENTS)}
    try:
        response = SESSION.get(url, headers=headers, timeout=15)
    except Exception as e:
        print(f"Error: {e}")
        return None

    # 2. Parse
    soup = BeautifulSoup(response.content, "lxml")

    # 3. Setup output folder
    title_elem = soup.find("h1", id="firstHeading")
    title = title_elem.get_text(strip=True) if title_elem else "Unknown"

    safe_title = re.sub(r"[^\w\-_]", "_", title)
    output_folder = f"Output_{safe_title}"
    os.makedirs(output_folder, exist_ok=True)

    # 4. Extract structured data (before cleaning!)
    infobox = extract_infobox(soup)
    if infobox:
        with open(f"{output_folder}/infobox.json", "w", encoding="utf-8") as f:
            json.dump(infobox, f, indent=2, ensure_ascii=False)

    extract_tables(soup, output_folder)

    # 5. Clean and convert to Markdown
    content = soup.select_one("#mw-content-text .mw-parser-output")

    # Remove noise elements so they don't appear in the text
    junk = [".navbox", ".reflist", ".reference", ".hatnote", ".ambox"]
    for selector in junk:
        for el in content.select(selector):
            el.decompose()

    h = html2text.HTML2Text()
    h.body_width = 0  # No wrapping
    markdown_content = h.handle(str(content))

    with open(f"{output_folder}/content.md", "w", encoding="utf-8") as f:
        f.write(f"# {title}\n\n{markdown_content}")

    print(f"Scraped: {title}")
    return {"soup": soup, "title": title}


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("url", help="Wikipedia article URL")
    parser.add_argument("--crawl", action="store_true", help="Enable crawler mode")
    parser.add_argument(
        "--max-pages", type=int, default=5, help="Maximum pages to scrape"
    )
    parser.add_argument("--max-depth", type=int, default=2, help="Maximum crawl depth")
    args = parser.parse_args()

    if args.crawl:
        WikipediaCrawler(
            args.url, max_pages=args.max_pages, max_depth=args.max_depth
        ).crawl()
    else:
        scrape_page(args.url)