掌握Java网页抓取：技术与示例完整指南

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>decodo-web-scraper</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>21</maven.compiler.source>
        <maven.compiler.target>21</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <!-- Jsoup for parsing -->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.21.2</version>
        </dependency>

        <!-- Gson for JSON export -->
        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.13.2</version>
        </dependency>

        <!-- OpenCSV for CSV export (optional) -->
        <dependency>
            <groupId>com.opencsv</groupId>
            <artifactId>opencsv</artifactId>
            <version>5.7.1</version>
        </dependency>

        <!-- Playwright for browser automation with proxies -->
        <dependency>
            <groupId>com.microsoft.playwright</groupId>
            <artifactId>playwright</artifactId>
            <version>1.45.0</version>
        </dependency>

        <!-- Selenium for browser automation with proxies -->
            <dependency>
                <groupId>org.seleniumhq.selenium</groupId>
                <artifactId>selenium-java</artifactId>
                <version>4.25.0</version>
            </dependency>
    </dependencies>

</project>

import java.net.http.*;
import java.net.*;
import java.time.Duration;

// Create an HTTP client
HttpClient client = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(10))
.build();


// Build a `GET` request that targets the website you want to scrape
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create("https://quotes.toscrape.com/"))
.header("User-Agent", "Java/HttpClient")
.GET()
.build();

// Send the request and capture the response body as a string
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
String html = response.body();

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

// Parse the fetched HTML into a Jsoup `Document`
Document doc = Jsoup.parse(html, "https://quotes.toscrape.com");
// Select all quote blocks with the CSS selector
Elements quotes = doc.select(".quote");

// For each quote, read the `.text` and `.author` elements
for (Element quote : quotes) {
System.out.println("*** QUOTE:");
System.out.println(quote.selectFirst(".text").text());
System.out.println("- Author: " + quote.selectFirst(".author").text());
}

import java.util.ArrayList;
import java.util.List;

// ...

// Parse the fetched HTML into a Jsoup `Document`
Document doc = Jsoup.parse(html, "https://quotes.toscrape.com");
// Select all quote blocks with the CSS selector
Elements quotes = doc.select(".quote");

// Collect parsed quotes
List<Quote> allQuotes = new ArrayList<>();

// For each quote, read the `.text` and `.author` elements
for (Element quote : quotes) {
String quoteText = quote.selectFirst(".text").text();
String quoteAuthor = quote.selectFirst(".author").text();
// Add the current quote to the list of all quotes
allQuotes.add(new Quote(quoteText, quoteAuthor));
}

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import java.nio.file.Files;
import java.nio.file.Path;

// ... After you've built `List<Quote> allQuotes`

Gson gson = new GsonBuilder()
.setPrettyPrinting()
.create();

String json = gson.toJson(allQuotes);
Files.writeString(Path.of("quotes.json"), json);

System.out.println("Wrote " + allQuotes.size() + " quotes to quotes.json");

import com.opencsv.CSVWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.charset.StandardCharsets;
import java.io.Writer;

// ... After you've built `List<Quote> allQuotes`

Path out = Path.of("quotes.csv");
try (Writer w = Files.newBufferedWriter(out, StandardCharsets.UTF_8);
CSVWriter csv = new CSVWriter(w)) {

// Header
csv.writeNext(new String[] { "quoteText", "quoteAuthor" });

// Rows
for (Quote q : allQuotes) {
csv.writeNext(new String[] { q.getQuoteText(), q.getQuoteAuthor() });
}
}

System.out.println("Wrote " + allQuotes.size() + " quotes to quotes.csv");

import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;

public class DynamicExample {
public static void main(String[] args) {
WebDriver driver = new ChromeDriver();
driver.get("https://example.com/quotes");

// Example: click a button before data appears
WebElement button = driver.findElement(By.cssSelector(".load-more"));
button.click();

// Wait for new elements to load (simple sleep for demo)
try { Thread.sleep(2000); } catch (InterruptedException ignored) {}

System.out.println(driver.getPageSource());
driver.quit();
}
}

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

public class MainPagination {
public static void main(String[] args) throws Exception {
int page = 1;
boolean hasNext = true;

while (hasNext) {
String url = "https://quotes.toscrape.com/page/" + page;
Document doc = Jsoup.connect(url)
.userAgent("Mozilla/5.0 (compatible; JavaScraper/1.0)")
.timeout(10000)
.get();

Elements quotes = doc.select(".quote");
if (quotes.isEmpty()) break; // Stop when no more products

// Print the page number and number of scraped quotes
System.out.println("Page " + page + ": " + quotes.size() + " items");

// Extract and store data here
page++;

// Small delay helps avoid hitting rate limits
Thread.sleep(1500);
}
}
}

import java.net.http.*;
import java.net.*;
import java.util.*;
import java.util.concurrent.*;

public class ParallelScraper {
public static void main(String[] args) throws Exception {
List<String> urls = List.of(
"https://quotes.toscrape.com/page/1",
"https://quotes.toscrape.com/page/2",
"https://quotes.toscrape.com/page/3"
);

HttpClient client = HttpClient.newHttpClient();
ExecutorService executor = Executors.newFixedThreadPool(5);

for (String url : urls) {
executor.submit(() -> {
try {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.header("User-Agent", "JavaScraper/1.0")
.GET()
.build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(url + " - " + response.statusCode());
} catch (Exception e) {
System.err.println("Error fetching " + url + ": " + e.getMessage());
}
});
}

executor.shutdown();
executor.awaitTermination(1, TimeUnit.MINUTES);
}
}

package org.example;

import org.openqa.selenium.Proxy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;

public class SeleniumProxyTest {
    public static void main(String[] args) {
        // Proxy with username:password@host:port
        String proxyUser = "YOUR_PROXY_USERNAME";
        String proxyPass = "YOUR_PROXY_PASSWORD";
        String proxyHost = "gate.decodo.com";
        int proxyPort = 7000;

        String proxyWithAuth = proxyUser + ":" + proxyPass + "@" + proxyHost + ":" + proxyPort;

        Proxy seleniumProxy = new Proxy();
        seleniumProxy.setProxyType(Proxy.ProxyType.MANUAL);
        seleniumProxy.setHttpProxy(proxyWithAuth);
        seleniumProxy.setSslProxy(proxyWithAuth);

        ChromeOptions options = new ChromeOptions();
        options.setProxy(seleniumProxy);
        // Optional – avoid some SSL/proxy issues
        options.addArguments("--ignore-certificate-errors");

        // If you have a separate chromedriver, you may need:
        // System.setProperty("webdriver.chrome.driver", "/path/to/chromedriver");

        WebDriver driver = new ChromeDriver(options);

        try {
            driver.get("https://quotes.toscrape.com");
            System.out.println(driver.getPageSource());
        } finally {
            driver.quit();
        }
    }
}

package org.example;

import com.microsoft.playwright.*;
import com.microsoft.playwright.options.Proxy;

public class PlaywrightTest {
    public static void main(String[] args) {
        Proxy proxy = new Proxy("http://gate.decodo.com:7000")
                .setUsername("YOUR_PROXY_USERNAME")
                .setPassword("YOUR_PROXY_PASSWORD");

        try (Playwright playwright = Playwright.create()) {
            Browser browser = playwright.chromium().launch(
                    new BrowserType.LaunchOptions()
                            .setHeadless(true)
                            .setProxy(proxy)
            );

            Page page = browser.newPage();
            page.navigate("https://quotes.toscrape.com");
            System.out.println(page.content());

            browser.close();
        }
    }
}