C#网页抓取指南: 从零到生产代码（2025版）

dotnet --version

dotnet new console -n WebScraper
cd WebScraper

WebScraper/ 
├── Program.cs # Your main entry point 
├── WebScraper.csproj # Project configuration file 
├── obj/ # Intermediate build files (ignore this) 
└── bin/ # Compiled output goes here

dotnet add package HtmlAgilityPack

<ItemGroup>
  <PackageReference Include="HtmlAgilityPack" Version="1.11.61" />
</ItemGroup>

dotnet add package CsvHelper

using HtmlAgilityPack;

var web = new HtmlWeb();
var doc = web.Load("https://quotes.toscrape.com/");

Console.WriteLine("Page loaded successfully!");
Console.WriteLine($"Title: {doc.DocumentNode.SelectSingleNode("//title").InnerText}");

dotnet run

// Select ALL matching elements
var quoteNodes = doc.DocumentNode.SelectNodes("//div[@class='quote']");

// Select the FIRST matching element
var firstQuote = doc.DocumentNode.SelectSingleNode("//div[@class='quote']");

using HtmlAgilityPack;

var web = new HtmlWeb();
var doc = web.Load("https://quotes.toscrape.com/");

// Select all quote containers
var quoteNodes = doc.DocumentNode.SelectNodes("//div[@class='quote']");

foreach (var quoteNode in quoteNodes)
{
    // Extract nested elements using relative XPath (starts with .)
    var text = quoteNode.SelectSingleNode(".//span[@class='text']").InnerText;
    var author = quoteNode.SelectSingleNode(".//small[@class='author']").InnerText;   

    Console.WriteLine($"Quote: {text}");
    Console.WriteLine($"Author: {author}");
    Console.WriteLine("---");
}

引用: "I have not failed. I've just found 10,000 ways that won't work."

using HtmlAgilityPack;

var web = new HtmlWeb();
var doc = web.Load("https://quotes.toscrape.com/");

var quoteNodes = doc.DocumentNode.SelectNodes("//div[@class='quote']");

foreach (var quoteNode in quoteNodes)
{
    var text = quoteNode.SelectSingleNode(".//span[@class='text']").InnerText;
    var author = quoteNode.SelectSingleNode(".//small[@class='author']").InnerText;    
    // Decode HTML entities to readable text
    text = HtmlEntity.DeEntitize(text);    

    Console.WriteLine($"Quote: {text}");
    Console.WriteLine($"Author: {author}");
    Console.WriteLine("---");
}

dotnet add package Selenium.WebDriver
dotnet add package Selenium.WebDriver.ChromeDriver

using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;

// Configure Chrome options
var options = new ChromeOptions();
options.AddArgument("--headless");  // Run without GUI
options.AddArgument("--disable-gpu");  // Disable GPU acceleration (recommended for headless)
options.AddArgument("--no-sandbox");  // Bypass OS security model (needed in some environments)

// Launch Chrome with these options
var driver = new ChromeDriver(options);

try
{
    driver.Navigate().GoToUrl("https://quotes.toscrape.com/js/");
    Console.WriteLine($"Page title: {driver.Title}");
}
finally
{
    driver.Quit();  // ALWAYS close the browser
}

dotnet run

页面标题: Quotes to Scrape

using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;

var options = new ChromeOptions();
options.AddArgument("--headless");
options.AddArgument("--disable-gpu");

var driver = new ChromeDriver(options);

try
{
    driver.Navigate().GoToUrl("https://quotes.toscrape.com/js/");    

    // Wait for JavaScript to load content (important!)
    Thread.Sleep(2000);  // Simple wait    

    // Find all quote containers
    var quoteElements = driver.FindElements(By.CssSelector("div.quote"));    

    Console.WriteLine($"Found {quoteElements.Count} quotes\n");    

    foreach (var quoteElement in quoteElements)
    {
        // Extract text from nested elements
        var text = quoteElement.FindElement(By.CssSelector("span.text")).Text;
        var author = quoteElement.FindElement(By.CssSelector("small.author")).Text;        

        // Extract tags (multiple elements)
        var tagElements = quoteElement.FindElements(By.CssSelector("a.tag"));
        var tags = tagElements.Select(t => t.Text).ToList();        

        Console.WriteLine($"Quote: {text}");
        Console.WriteLine($"Author: {author}");
        Console.WriteLine($"Tags: {string.Join(", ", tags)}");
        Console.WriteLine("---");
    }
}
finally
{
    driver.Quit();  // Clean up browser process
}

driver.FindElement(By.CssSelector("div.quote"));
driver.FindElement(By.CssSelector("span.text"));
driver.FindElement(By.CssSelector("a[href='/author/Albert-Einstein']"));

driver.FindElement(By.XPath("//div[@class='quote']"));
driver.FindElement(By.XPath("//span[@class='text']"));
driver.FindElement(By.XPath("//a[contains(@href, 'Einstein')]"));

var authorLink = quoteElement.FindElement(By.CssSelector("a"));
var authorUrl = authorLink.GetAttribute("href");
Console.WriteLine($"Author URL: {authorUrl}");

using OpenQA.Selenium.Support.UI;

var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(10));
wait.Until(d => d.FindElements(By.CssSelector("div.quote")).Count > 0);

public class Quote
{
    public string Text { get; set; }
    public string Author { get; set; }
    public string Tags { get; set; }
    public string Url { get; set; }
}

using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using CsvHelper;
using System.Globalization;

public class Quote
{
    public string Text { get; set; }
    public string Author { get; set; }
    public string Tags { get; set; }
    public string Url { get; set; }
}

class Program
{
    static void Main()
    {
        var options = new ChromeOptions();
        options.AddArgument("--headless");
        options.AddArgument("--disable-gpu");

        var driver = new ChromeDriver(options);
        var quotes = new List<Quote>();

        try
        {
            driver.Navigate().GoToUrl("https://quotes.toscrape.com/js/");
            Thread.Sleep(2000);  // Wait for JavaScript to load

            var quoteElements = driver.FindElements(By.CssSelector("div.quote"));

            foreach (var quoteElement in quoteElements)
            {
                var text = quoteElement.FindElement(By.CssSelector("span.text")).Text;
                var author = quoteElement.FindElement(By.CssSelector("small.author")).Text; 
                var tagElements = quoteElement.FindElements(By.CssSelector("a.tag"));
                var tags = string.Join(", ", tagElements.Select(t => t.Text));   

                var authorLink = quoteElement.FindElement(By.CssSelector("a"));
                var url = authorLink.GetAttribute("href");

                quotes.Add(new Quote
                {
                    Text = text,
                    Author = author,
                    Tags = tags,
                    Url = url
                });
            }

            Console.WriteLine($"Scraped {quotes.Count} quotes. Writing to CSV...");

            // Write to CSV
            using (var writer = new StreamWriter("quotes.csv"))
            using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
            {
                csv.WriteRecords(quotes);
            }

            Console.WriteLine("Export complete! Check quotes.csv");
        }
        finally
        {
            driver.Quit();
        }
    }
}

using (var writer = new StreamWriter("quotes.csv", false, new UTF8Encoding(true)))
using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
{
    csv.WriteRecords(quotes);
}