如何使用Python抓取网页表格：完整指南

pip install requests beautifulsoup4 pandas selenium lxml

import pandas as pd
import requests
from io import StringIO  # Needed to wrap the HTML string

# The URL of the Wikipedia page
url = "https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_population"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}

# Fetch the HTML content
response = requests.get(url, headers=headers)
html_content = response.text

# Parse all tables from the HTML content
tables = pd.read_html(StringIO(html_content))

print(f"Total tables found: {len(tables)}")

# In this case, the table we want is the 1st one
df = tables[0]

# Display the first 5 rows
print(df.head())

# Save the data to a CSV file
df.to_csv("population_data.csv", index=False)
print("Data saved to population_data.csv")

import requests
from bs4 import BeautifulSoup
import pandas as pd

url = "https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_population"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
}
response = requests.get(url, headers=headers)

if response.status_code == 200:
    print("Successfully fetched the page")
    html_content = response.text
else:
    print(f"Error fetching page: Status code {response.status_code}")
    exit()

soup = BeautifulSoup(html_content, 'html.parser')

table = soup.find('table', attrs={'class': 'wikitable'})

if table is None:
    print("Could not find the table. Check your selectors.")
    exit()

# --- Get Headers ---
headers_html = table.find_all('th')
headers = [th.text.strip() for th in headers_html]
print(f"Headers found: {headers}")

# --- Get All Rows ---
rows_html = table.find('tbody').find_all('tr')
all_rows = []

for row in rows_html:
    cells_html = row.find_all('td')
    row_data = [cell.text.strip() for cell in cells_html]
    if row_data:
        all_rows.append(row_data)

df = pd.DataFrame(all_rows, columns=headers)
print(df.head())

df.to_csv('scraped_data_manual.csv', index=False)

import pandas as pd
import requests
from io import StringIO

# Configuration
WIKI_URL = 'https://en.wikipedia.org/wiki/Comparison_of_web_browsers'
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36'
OUTPUT_FILE = 'browser_comparison.csv'

# Fetch page
response = requests.get(WIKI_URL, headers={'User-Agent': USER_AGENT})
html_content = response.text

# Extract all tables
all_tables = pd.read_html(StringIO(html_content))
print(f"Found {len(all_tables)} tables")

# Find the correct table by checking its columns
for index, table in enumerate(all_tables):
    if 'Browser' in str(table.columns):
        print(f"\nBrowser table found at index {index}")
        
        # Clean the multi-level headers from pandas
        table.columns = ['_'.join(col).strip() for col in table.columns.values]
        print(table.head(10))

        table.to_csv(OUTPUT_FILE, index=False)
        print(f"\nSaved to {OUTPUT_FILE}")
        break

pip install html-table-extractor

import requests
import csv
from bs4 import BeautifulSoup
from html_table_extractor.extractor import Extractor

# Configuration
WIKI_URL = "https://en.wikipedia.org/wiki/Comparison_of_web_browsers"
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"
OUTPUT_FILE = "browser_comparison_extractor.csv"

print(f"Fetching page: {WIKI_URL}")
response = requests.get(WIKI_URL, headers={"User-Agent": USER_AGENT})
html_content = response.text

soup = BeautifulSoup(html_content, "html.parser")
all_tables = soup.find_all("table")

print(f"Found {len(all_tables)} tables. Searching for the correct one...")

for index, table in enumerate(all_tables):
    try:
        # We convert the 'table' object back to a string
        extractor = Extractor(str(table))
        extractor.parse()

        table_data = extractor.return_list()

        if table_data:
            # Clean header values (e.g., "Browser\n" becomes "Browser")
            headers = [str(header).strip() for header in table_data[0]]

            if "Browser" in headers:
                print(f"\nBrowser table found at index {index}")

                with open(OUTPUT_FILE, "w", newline="", encoding="utf-8") as f:
                    writer = csv.writer(f)
                    writer.writerows(table_data)

                print(f"Successfully saved to {OUTPUT_FILE}")

                # Break the loop since we found the first one
                break

    except Exception as e:
        print(f"Could not parse table at index {index}: {e}")

# This code extends the Method 2 (BeautifulSoup) example.
# It assumes 'df' is the DataFrame we already created.

# The 'Population' column is a string: '1,417,492,000'

# Remove commas
df['Population_Clean'] = df['Population'].str.replace(',', '', regex=False)

# Convert to numeric
df['Population_Clean'] = pd.to_numeric(df['Population_Clean'])

# Now you can perform calculations
print(f"Total population of top 5: {df.head(5)['Population_Clean'].sum()}")

# Save to CSV
df.to_csv('population_data_clean.csv', index=False)

# Save to JSON
df.to_json('population_data_clean.json', orient='records')

{
    "marketCap": {
        "raw": 4.580887901397705E12,
        "fmt": "4.581T",
        "longFmt": "4,580,887,901,397"
    },
    "ticker": "NVDA",
    "avgDailyVol3m": {
        "raw": 1.818218125E8,
        "fmt": "181.822M",
        "longFmt": "181,821,812"
    },
...
}

import pandas as pd
import requests
from io import StringIO
from time import sleep

all_data = []
page = 90
headers = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"
}

while True:
    print(f"Page {page}...", end=" ")
    response = requests.get(f"https://coinmarketcap.com/?page={page}", headers=headers)

    if response.status_code != 200:
        break

    try:
        all_data.append(pd.read_html(StringIO(response.text))[0])
    except:
        break

    if (
        'aria-disabled="true"' in response.text
        or "?page=" + str(page + 1) not in response.text
    ):
        print("Last page!")
        break

    page += 1
    sleep(2)

pd.concat(all_data, ignore_index=True).to_csv("coinmarketcap_data.csv", index=False)
print(f"\nDone! Saved {len(all_data)} pages")

import pandas as pd
from io import StringIO
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from time import sleep

url = "https://datatables.net/examples/data_sources/ajax.html"
all_data = []

options = Options()
options.add_argument("--headless")
driver = webdriver.Chrome(options=options)
driver.get(url)
sleep(2)  # Wait for the page and initial JavaScript to load

page = 90
while True:
    print(f"Scraping page {page}...", end=" ")

    table_html = driver.find_element(By.ID, "example").get_attribute("outerHTML")
    if not table_html:
        break

    df = pd.read_html(StringIO(table_html))[0]
    all_data.append(df)
    print("Done.")

    # Check if the "Next" button is disabled
    try:
        next_btn = driver.find_element(By.CSS_SELECTOR, "button.dt-paging-button.next")
        btn_class = next_btn.get_attribute("class") or ""

        if "disabled" in btn_class:
            print("Last page reached!")
            break

        # If not disabled, click it and wait
        next_btn.click()
        sleep(1)  # Wait for the table to reload
        page += 1
    except:
        print("Could not find 'Next' button or it broke.")
        break

driver.quit()

final_df = pd.concat(all_data, ignore_index=True)
final_df.to_csv("datatables_data.csv", index=False)
print(f"\nDone! Saved {len(all_data)} pages to datatables_data.csv")

# Your selector has a typo or wrong class name
table = soup.find('table', attrs={'class': 'my-typo-class'})  # Returns None

# This next line is where it crashes
rows = table.find_all('tr')  # Can't call .find_all() on None!