diff --git a/build.gradle b/build.gradle index bdfb9dd..b2f2a2f 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ repositories { } dependencies { - implementation platform('run.halo.tools.platform:plugin:2.24.0') + implementation platform('run.halo.tools.platform:plugin:2.25.0') compileOnly 'run.halo.app:api' implementation('com.rometools:rome:2.1.0') { exclude group: 'org.slf4j', module: 'slf4j-api' @@ -63,7 +63,7 @@ tasks.named('generatePluginComponentsIdx') { } halo { - version = '2.24' + version = '2.25' port = 8091 debug = true } diff --git a/openspec/changes/use-halo-http-security-utils/.openspec.yaml b/openspec/changes/use-halo-http-security-utils/.openspec.yaml new file mode 100644 index 0000000..c86b1d7 --- /dev/null +++ b/openspec/changes/use-halo-http-security-utils/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-06-14 diff --git a/openspec/changes/use-halo-http-security-utils/design.md b/openspec/changes/use-halo-http-security-utils/design.md new file mode 100644 index 0000000..17ab74a --- /dev/null +++ b/openspec/changes/use-halo-http-security-utils/design.md @@ -0,0 +1,168 @@ +## Context + +`SafeUrlFetcher` is the shared outbound fetch boundary for link detail scraping, +RSS/Atom discovery and fetching, and link verification. It currently combines +several concerns in plugin-owned code: + +- URL scheme and resolved-address validation in `LinkSecurityUtils`. +- Manual redirect handling and redirect target validation. +- HTTP fetching through Jsoup for HTTP URLs. +- A custom HTTPS socket path that pins the connection to a validated address. +- Response body size checks, charset decoding, status/header capture, and + optional Jsoup document parsing. + +Halo 2.25 moved `run.halo.app.infra.utils.HttpSecurityUtils` into the API module, +so plugins can now use the same secure Reactor Netty client and response-size +filter as Halo itself. The utility exposes `secureHttpClient()` and +`maxResponseSizeFilter(long)`. + +## Goals / Non-Goals + +**Goals:** + +- Upgrade the plugin build and local dev runtime to Halo 2.25.0. +- Raise the plugin runtime requirement to `>=2.25.0`. +- Replace plugin-maintained SSRF networking code with Halo API-provided HTTP + security utilities. +- Preserve the current `SafeUrlFetcher.FetchOptions` and `FetchResult` contract + for link detail, RSS, and verification callers. +- Preserve redirect validation, redirect limits, body-size limits, timeouts, + status code handling, final URL reporting, `ETag`, `Last-Modified`, and HTML + document parsing where callers rely on them. + +**Non-Goals:** + +- No Console UI changes. +- No public endpoint or generated TypeScript API client changes. +- No new outbound proxy, allowlist, or operator-configurable HTTP policy. +- No change to Halo's `HttpSecurityUtils` behavior inside this plugin change. + +## Decisions + +### 1. Require Halo 2.25.0 directly + +The implementation should update the platform dependency, dev runtime, and +plugin manifest requirement together: + +- `run.halo.tools.platform:plugin:2.25.0` +- `halo.version = '2.25'` +- `spec.requires: ">=2.25.0"` + +**Rationale:** importing `HttpSecurityUtils` from the API module is only safe for +plugin runtime environments that provide Halo 2.25 or newer. + +**Alternative considered:** keep `>=2.22.5` and use reflection or a fallback +fetcher. That would preserve compatibility but keep two security implementations +alive, which is the maintenance problem this change is meant to remove. + +### 2. Keep `SafeUrlFetcher` as the plugin adapter + +`LinkRequest`, `LinkFeedFetcher`, and `LinkVerificationFetcher` should keep using +`SafeUrlFetcher.fetch(...)`. The internals of `SafeUrlFetcher` should move to a +WebFlux `WebClient` backed by: + +- `ReactorClientHttpConnector(HttpSecurityUtils.secureHttpClient())` +- `HttpSecurityUtils.maxResponseSizeFilter(options.maxBodySize)` + +**Rationale:** callers already depend on the fetcher's small result model rather +than raw HTTP APIs. Keeping that adapter makes the refactor surgical and avoids +cross-service churn. + +**Alternative considered:** inject `WebClient` into every caller and let each +service interpret responses. That spreads redirect, timeout, response-size, and +header handling across unrelated services. + +### 3. Preserve per-operation timeout behavior + +The secure client returned by Halo sets secure defaults, including disabled +automatic redirects. `SafeUrlFetcher` should derive the Reactor Netty client from +`HttpSecurityUtils.secureHttpClient()` and then apply the current operation's +timeout from `FetchOptions` so link verification can keep its shorter timeout. + +**Rationale:** RSS fetching currently uses the default 10s timeout, while link +verification uses 5s. The security utility should replace the SSRF mechanism, not +silently change verification latency. + +**Alternative considered:** accept Halo's default timeout for every operation. +That is simpler but changes verification behavior. + +### 4. Keep manual redirect handling in the fetcher + +`HttpSecurityUtils.secureHttpClient()` disables automatic redirects. The fetcher +should continue to inspect redirect responses, resolve relative `Location` +headers against the current URL, enforce the redirect limit, and re-run each hop +through the same secure WebClient. + +**Rationale:** redirect targets are separate outbound connections. Sending every +hop through the secure client preserves the existing safety boundary without +maintaining plugin-local IP filtering code. + +**Alternative considered:** enable automatic redirects in the Reactor Netty +client. That would make redirect behavior harder to reason about and bypass the +plugin's explicit hop limit. + +### 5. Use response exchange APIs, not `retrieve()` + +The fetcher should use an API that exposes status and headers before body +decoding, such as `exchangeToMono`, then build `FetchResult` from: + +- final URL +- HTTP status code +- response body +- parsed Jsoup document when `FetchOptions.parseDocument` is true +- `ETag` +- `Last-Modified` + +**Rationale:** callers use non-2xx status codes as data. For example, RSS refresh +handles `304 Not Modified`, and verification records HTTP status in link status. +`retrieve()` would turn many of these responses into exceptions. + +**Alternative considered:** use `retrieve()` and map exceptions back to status +results. That is more fragile and obscures normal non-2xx outcomes. + +### 6. Normalize fetch errors at the adapter boundary + +`SafeUrlFetcher` should continue throwing `ServerErrorException` for invalid +URLs, blocked URLs, redirects, body-size violations, and IO/client failures. For +link detail requests, the endpoint should continue converting invalid or blocked +URL failures into HTTP 400. + +When `FetchOptions.allowOversizedBody` is true, the fetcher should preserve the +status code and return an empty body if the response body exceeds the configured +limit. This preserves the current reachability-check behavior. + +**Rationale:** the refactor changes the lower-level exception types from custom +validation/IO errors to WebClient/Reactor/DataBuffer errors. Callers should not +need to learn those details. + +**Alternative considered:** let WebClient exceptions propagate directly. That +would make endpoint and status error messages less stable. + +## Risks / Trade-offs + +| Risk | Mitigation | +|------|------------| +| Halo's blocked address set is stricter than the plugin-local filter and may block additional special-use ranges | Accept the stricter platform policy because these are external friend-link requests, not intranet fetches | +| Response decoding may change if the refactor uses WebClient string decoding directly | Preserve the current charset-aware body decoding by reading bytes and decoding from `Content-Type` where practical | +| Redirect handling could accidentally lose `ETag` or `Last-Modified` from the final response | Add tests around RSS fetch results and `304 Not Modified` behavior | +| Tests that mocked Jsoup/static URL utilities will no longer cover the new path | Replace them with tests at the `SafeUrlFetcher` boundary and service-level tests that assert behavior rather than implementation details | + +## Migration Plan + +1. Upgrade Halo dependencies and plugin runtime metadata. +2. Refactor `SafeUrlFetcher` internals to use Halo's secure HTTP utilities while + retaining the existing public nested types. +3. Delete plugin-local `LinkSecurityUtils` and `RedirectHandler` once no code or + tests reference them. +4. Update tests to cover blocked private/local targets, redirect blocking, + response size limits, RSS conditional headers, and verification status + preservation. +5. Run `./gradlew test` and `./gradlew build`. + +Rollback is a normal code rollback of this change. Because the manifest minimum +Halo version changes, the dependency and manifest updates should be reverted +together if the refactor is rolled back. + +## Open Questions + +- None. diff --git a/openspec/changes/use-halo-http-security-utils/proposal.md b/openspec/changes/use-halo-http-security-utils/proposal.md new file mode 100644 index 0000000..1fd47b4 --- /dev/null +++ b/openspec/changes/use-halo-http-security-utils/proposal.md @@ -0,0 +1,46 @@ +## Why + +Halo 2.25 exposes `HttpSecurityUtils` from the API module so plugins can reuse the +same SSRF-safe HTTP client and response-size filter used by Halo itself. The +links plugin currently maintains its own SSRF-safe fetcher stack, which duplicates +platform logic and increases long-term maintenance risk. + +## What Changes + +- Upgrade the Halo plugin platform/API dependency and local dev runtime to + Halo 2.25.0. +- **BREAKING**: raise the plugin's minimum supported Halo version to `>=2.25.0` + because the implementation will import API classes first exposed to plugins in + Halo 2.25. +- Refactor external URL fetching for link details, RSS discovery/fetching, and + link verification to use `run.halo.app.infra.utils.HttpSecurityUtils`: + - `secureHttpClient()` for SSRF-safe outbound connections. + - `maxResponseSizeFilter(long)` for response body bounds. +- Preserve the plugin-facing fetch behavior needed by existing services: + status code, final URL, response body, parsed HTML document when requested, + `ETag`, and `Last-Modified`. +- Remove the plugin-local SSRF utility and manual socket/redirect code after the + Halo utility-backed fetcher covers the same safety boundaries. + +## Capabilities + +### New Capabilities + +- *(none)* + +### Modified Capabilities + +- `ssrf-protection`: external HTTP requests continue to be protected from + private, local, and special-use network targets, but the security enforcement + is delegated to Halo 2.25's API-provided HTTP security utilities instead of + plugin-maintained URL/IP validation and pinned socket code. + +## Impact + +- **Backend**: `build.gradle`, `plugin.yaml`, `run.halo.links.security`, and the + tests around link detail, RSS fetching, and verification. +- **Frontend**: no Console UI changes expected. +- **API**: no endpoint contract or generated TypeScript client changes expected. +- **Dependencies**: Halo platform/API moves from 2.24.0 to 2.25.0; no new + third-party dependency should be needed because WebFlux/Reactor Netty are + already available through Halo. diff --git a/openspec/changes/use-halo-http-security-utils/specs/ssrf-protection/spec.md b/openspec/changes/use-halo-http-security-utils/specs/ssrf-protection/spec.md new file mode 100644 index 0000000..54ec023 --- /dev/null +++ b/openspec/changes/use-halo-http-security-utils/specs/ssrf-protection/spec.md @@ -0,0 +1,43 @@ +## ADDED Requirements + +### Requirement: External fetches use Halo HTTP security utilities +The system SHALL execute plugin-originated external HTTP(S) requests through +Halo API-provided HTTP security utilities. + +#### Scenario: Link detail fetch uses the secure client +- **WHEN** the system fetches an external URL to extract link detail metadata +- **THEN** it uses an HTTP client based on `HttpSecurityUtils.secureHttpClient()` + +#### Scenario: RSS feed fetch uses the secure client +- **WHEN** the system fetches an external URL for RSS/Atom discovery or feed refresh +- **THEN** it uses an HTTP client based on `HttpSecurityUtils.secureHttpClient()` + +#### Scenario: Link verification fetch uses the secure client +- **WHEN** the system fetches an external URL for reachability or backlink verification +- **THEN** it uses an HTTP client based on `HttpSecurityUtils.secureHttpClient()` + +#### Scenario: Response size filter is applied +- **WHEN** the system consumes an external HTTP response body +- **THEN** it applies `HttpSecurityUtils.maxResponseSizeFilter(long)` using the + fetch operation's configured maximum response size + +#### Scenario: Plugin runtime requirement matches the security utility dependency +- **WHEN** the plugin is packaged +- **THEN** the plugin manifest declares Halo `>=2.25.0` as the minimum runtime + version + +## MODIFIED Requirements + +### Requirement: URL validation prevents DNS Rebinding attacks +The system SHALL use Halo's SSRF-safe HTTP client for outbound external HTTP(S) +connections so private, local, and special-use resolved addresses are rejected +before a connection is opened. + +#### Scenario: DNS Rebinding attack is blocked +- **WHEN** a link detail request is made with a URL whose hostname resolves to a + private, local, or special-use address at connection time +- **THEN** the system rejects the request before connecting + +#### Scenario: Direct private IP access is blocked +- **WHEN** a link detail request is made with a URL pointing to `http://192.168.1.1/` +- **THEN** the system rejects the request with an error indicating private address access is not allowed diff --git a/openspec/changes/use-halo-http-security-utils/tasks.md b/openspec/changes/use-halo-http-security-utils/tasks.md new file mode 100644 index 0000000..f9f0e91 --- /dev/null +++ b/openspec/changes/use-halo-http-security-utils/tasks.md @@ -0,0 +1,39 @@ +## 1. Upgrade Halo runtime contract + +- [x] 1.1 Update `build.gradle` to use `run.halo.tools.platform:plugin:2.25.0`. +- [x] 1.2 Update the `halo { version = ... }` local dev runtime to Halo 2.25. +- [x] 1.3 Update `src/main/resources/plugin.yaml` to require Halo `>=2.25.0`. +- [x] 1.4 Compile enough of the backend to confirm `run.halo.app.infra.utils.HttpSecurityUtils` is available from the plugin API dependency. + +## 2. Refactor the shared external URL fetcher + +- [x] 2.1 Refactor `SafeUrlFetcher` to build its HTTP client from `HttpSecurityUtils.secureHttpClient()`. +- [x] 2.2 Apply `HttpSecurityUtils.maxResponseSizeFilter(options.maxBodySize)` whenever response bodies are consumed. +- [x] 2.3 Preserve the existing `FetchOptions` and `FetchResult` contract, including status code, final URL, body, parsed document, `ETag`, and `Last-Modified`. +- [x] 2.4 Preserve current request headers, including `User-Agent`, `Accept`, `Referer`, `If-None-Match`, and `If-Modified-Since`. +- [x] 2.5 Preserve per-operation timeout behavior, including the shorter verification timeout. +- [x] 2.6 Keep explicit redirect handling with relative `Location` resolution, a 3-hop limit, and secure-client execution for every hop. +- [x] 2.7 Normalize invalid URL, blocked URL, redirect, response-size, and client failures to the existing `ServerErrorException` boundary. +- [x] 2.8 Preserve `allowOversizedBody` behavior for reachability checks by returning the response status with an empty body when the body exceeds the configured limit. + +## 3. Remove duplicate plugin-local security code + +- [x] 3.1 Remove `LinkSecurityUtils` after `SafeUrlFetcher` no longer depends on plugin-local address validation. +- [x] 3.2 Remove `RedirectHandler` after redirect handling is consolidated in the Halo utility-backed fetcher. +- [x] 3.3 Search the backend for remaining external fetch paths and verify link detail, RSS, and verification all go through the shared fetcher. +- [x] 3.4 Confirm no OpenAPI client regeneration is required because endpoint shapes, extension models, and DTO contracts did not change. + +## 4. Update tests + +- [x] 4.1 Replace tests that mock `LinkSecurityUtils`, `RedirectHandler`, or `Jsoup.connect` internals with tests against the `SafeUrlFetcher` behavior boundary. +- [x] 4.2 Cover direct private/local target blocking for link detail, RSS, and verification fetches. +- [x] 4.3 Cover non-HTTP(S) URL rejection and bad URL error normalization. +- [x] 4.4 Cover redirect-to-private target rejection and redirect limit enforcement. +- [x] 4.5 Cover response-size enforcement and the reachability `allowOversizedBody` status-preservation path. +- [x] 4.6 Cover RSS response metadata preservation, including `ETag`, `Last-Modified`, and `304 Not Modified`. + +## 5. Verify + +- [x] 5.1 Run `./gradlew test`. +- [x] 5.2 Run `./gradlew build`. +- [x] 5.3 Run `openspec validate use-halo-http-security-utils --strict`. diff --git a/src/main/java/run/halo/links/security/LinkSecurityUtils.java b/src/main/java/run/halo/links/security/LinkSecurityUtils.java deleted file mode 100644 index 8d03d4d..0000000 --- a/src/main/java/run/halo/links/security/LinkSecurityUtils.java +++ /dev/null @@ -1,137 +0,0 @@ -package run.halo.links.security; - -import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.UnknownHostException; - -/** - * Utility class for validating URLs to prevent SSRF attacks. - * Blocks private/reserved IP ranges and non-HTTP(S) schemes. - */ -public final class LinkSecurityUtils { - - private static final int MAX_REDIRECTS = 3; - - private LinkSecurityUtils() { - } - - /** - * Validates that the given URL string is safe to connect to. - * - * @param urlString the URL to validate - * @throws IllegalArgumentException if the URL is malformed, uses an invalid scheme, - * or resolves to a private/reserved address - */ - public static InetAddress validateUrl(String urlString) { - URL url; - try { - url = new URL(urlString); - } catch (MalformedURLException e) { - throw new IllegalArgumentException("Invalid URL: " + urlString, e); - } - return validateUrl(url); - } - - /** - * Validates that the given URL is safe to connect to. - * - * @param url the URL to validate - * @return the validated public {@link InetAddress} - * @throws IllegalArgumentException if the URL uses an invalid scheme - * or resolves to a private/reserved address - */ - public static InetAddress validateUrl(URL url) { - String protocol = url.getProtocol().toLowerCase(); - if (!"http".equals(protocol) && !"https".equals(protocol)) { - throw new IllegalArgumentException( - "Only HTTP and HTTPS protocols are allowed: " + url); - } - - String host = url.getHost(); - if (host == null || host.isBlank()) { - throw new IllegalArgumentException("URL must have a host: " + url); - } - - InetAddress[] addresses; - try { - addresses = InetAddress.getAllByName(host); - } catch (UnknownHostException e) { - throw new IllegalArgumentException( - "Unable to resolve host: " + host, e); - } - - for (InetAddress address : addresses) { - if (!isPrivateAddress(address)) { - return address; - } - } - throw new IllegalArgumentException( - "Access to private/reserved address is not allowed"); - } - - /** - * Checks if the given IP address is private, loopback, link-local, or otherwise reserved. - * - * @param address the address to check - * @return true if the address is private/reserved - */ - public static boolean isPrivateAddress(InetAddress address) { - if (address.isAnyLocalAddress() || address.isLoopbackAddress() - || address.isLinkLocalAddress() || address.isMulticastAddress()) { - return true; - } - - if (address.isSiteLocalAddress()) { - return true; - } - - // 169.254.0.0/16 (APIPA / IPv4 link-local) — not covered by isSiteLocalAddress - if (address.getAddress().length == 4) { - byte[] bytes = address.getAddress(); - int first = bytes[0] & 0xFF; - int second = bytes[1] & 0xFF; - if (first == 169 && second == 254) { - return true; - } - } - - // fc00::/7 (IPv6 Unique Local Address / ULA) — not covered by isSiteLocalAddress - if (address.getAddress().length == 16) { - byte[] bytes = address.getAddress(); - if ((bytes[0] & 0xFE) == 0xFC) { - return true; - } - } - - return false; - } - - public static int getMaxRedirects() { - return MAX_REDIRECTS; - } - - /** - * Builds a connection URL using the validated IP address while preserving - * the original protocol, port, and path. IPv6 addresses are wrapped in brackets. - * - * @param originalUrl the original URL - * @param address the validated public IP address - * @return a URL string suitable for direct IP connection - */ - public static String toConnectUrl(URL originalUrl, InetAddress address) { - StringBuilder sb = new StringBuilder(); - sb.append(originalUrl.getProtocol()).append("://"); - if (address instanceof java.net.Inet6Address) { - sb.append('[').append(address.getHostAddress()).append(']'); - } else { - sb.append(address.getHostAddress()); - } - if (originalUrl.getPort() != -1) { - sb.append(':').append(originalUrl.getPort()); - } - sb.append(originalUrl.getFile()); - return sb.toString(); - } - -} diff --git a/src/main/java/run/halo/links/security/RedirectHandler.java b/src/main/java/run/halo/links/security/RedirectHandler.java deleted file mode 100644 index 58a0244..0000000 --- a/src/main/java/run/halo/links/security/RedirectHandler.java +++ /dev/null @@ -1,101 +0,0 @@ -package run.halo.links.security; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; -import org.jsoup.Connection; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.springframework.web.server.ServerErrorException; - -/** - * Handles HTTP redirects with SSRF validation. - * Validates each redirect target before following, limits total hops, and - * rebuilds connections with consistent settings. - */ -public class RedirectHandler { - - private final Map headers; - private final int timeout; - private final int maxBodySize; - private int hopsRemaining; - - public RedirectHandler(Map headers, int timeout, int maxBodySize) { - this.headers = headers; - this.timeout = timeout; - this.maxBodySize = maxBodySize; - this.hopsRemaining = LinkSecurityUtils.getMaxRedirects(); - } - - /** - * Follows redirects starting from the given response until a non-redirect - * response is received or the hop limit is exceeded. - * - * @param response the initial Jsoup connection response - * @return the final Document - * @throws ServerErrorException if a redirect is blocked or hop limit exceeded - * @throws IOException on connection errors - */ - public Document followRedirects(Connection.Response response) throws IOException { - Connection.Response current = response; - while (isRedirect(current.statusCode())) { - if (hopsRemaining <= 0) { - throw new ServerErrorException( - "Too many redirects", new IllegalStateException( - "Exceeded maximum redirect limit of " - + LinkSecurityUtils.getMaxRedirects())); - } - hopsRemaining--; - - String location = current.header("Location"); - if (location == null || location.isBlank()) { - throw new ServerErrorException( - "Redirect missing Location header", - new IllegalStateException( - "HTTP " + current.statusCode() + " without Location")); - } - - URL currentUrl = current.url(); - URL redirectUrl; - try { - redirectUrl = new URL(currentUrl, location); - } catch (MalformedURLException e) { - throw new ServerErrorException( - "Invalid redirect URL: " + location, e); - } - - InetAddress validatedAddress; - try { - validatedAddress = LinkSecurityUtils.validateUrl(redirectUrl); - } catch (IllegalArgumentException e) { - throw new ServerErrorException( - "Redirect blocked for security reasons", e); - } - - String connectUrl = "http".equalsIgnoreCase(redirectUrl.getProtocol()) - ? LinkSecurityUtils.toConnectUrl(redirectUrl, validatedAddress) - : redirectUrl.toExternalForm(); - - Map requestHeaders = new HashMap<>(headers); - if ("http".equalsIgnoreCase(redirectUrl.getProtocol())) { - requestHeaders.put("Host", redirectUrl.getHost()); - } - - current = Jsoup.connect(connectUrl) - .followRedirects(false) - .timeout(timeout) - .maxBodySize(maxBodySize) - .headers(requestHeaders) - .execute(); - } - return current.parse(); - } - - private boolean isRedirect(int statusCode) { - return statusCode == 301 || statusCode == 302 - || statusCode == 303 || statusCode == 307 || statusCode == 308; - } -} diff --git a/src/main/java/run/halo/links/security/SafeUrlFetcher.java b/src/main/java/run/halo/links/security/SafeUrlFetcher.java index b3078e1..7c0c530 100644 --- a/src/main/java/run/halo/links/security/SafeUrlFetcher.java +++ b/src/main/java/run/halo/links/security/SafeUrlFetcher.java @@ -1,32 +1,35 @@ package run.halo.links.security; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetAddress; -import java.net.InetSocketAddress; +import io.netty.channel.ChannelOption; import java.net.MalformedURLException; -import java.net.Socket; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; +import java.time.Duration; import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Locale; import java.util.Map; -import javax.net.ssl.SSLParameters; -import javax.net.ssl.SSLSocket; -import javax.net.ssl.SSLSocketFactory; import lombok.Getter; -import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.core.io.buffer.DataBufferLimitException; +import org.springframework.core.io.buffer.DataBufferUtils; +import org.springframework.http.HttpHeaders; +import org.springframework.web.reactive.function.BodyExtractors; +import org.springframework.web.reactive.function.client.ClientResponse; +import org.springframework.web.reactive.function.client.ExchangeFunction; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.web.server.ServerErrorException; +import reactor.core.publisher.Mono; +import reactor.netty.http.client.HttpClient; +import run.halo.app.infra.utils.HttpSecurityUtils; /** - * Fetches remote HTTP(S) resources after validating URL targets against SSRF rules. + * Fetches remote HTTP(S) resources using Halo's SSRF-safe HTTP client. */ public final class SafeUrlFetcher { @@ -35,9 +38,8 @@ public final class SafeUrlFetcher { + "Chrome/58.0.3029.110 Safari/537.3"; public static final int DEFAULT_TIMEOUT_MS = 10_000; public static final int DEFAULT_MAX_BODY_SIZE = 1024 * 1024 * 20; - private static final int MAX_HEADER_LINE_SIZE = 8192; - private static PinnedHttpsConnector pinnedHttpsConnector = - SafeUrlFetcher::connectPinnedHttpsSocket; + private static final int MAX_REDIRECTS = 3; + private static volatile ExchangeFunction exchangeFunctionForTesting; private SafeUrlFetcher() { } @@ -51,19 +53,21 @@ public static FetchResult fetch(String urlString, FetchOptions options) { } try { return fetch(url, options == null ? FetchOptions.html(urlString) : options); - } catch (IOException e) { + } catch (ServerErrorException e) { + throw e; + } catch (Exception e) { throw new ServerErrorException("Failed to fetch URL", e); } } - private static FetchResult fetch(URL url, FetchOptions options) throws IOException { + private static FetchResult fetch(URL url, FetchOptions options) { FetchResponse current = execute(url, options); - int hopsRemaining = LinkSecurityUtils.getMaxRedirects(); + int hopsRemaining = MAX_REDIRECTS; while (isRedirect(current.statusCode())) { if (hopsRemaining <= 0) { throw new ServerErrorException("Too many redirects", new IllegalStateException("Exceeded maximum redirect limit of " - + LinkSecurityUtils.getMaxRedirects())); + + MAX_REDIRECTS)); } hopsRemaining--; @@ -89,221 +93,87 @@ private static FetchResult fetch(URL url, FetchOptions options) throws IOExcepti current.etag(), current.lastModified()); } - private static FetchResponse execute(URL url, FetchOptions options) - throws IOException { - InetAddress validatedAddress; - try { - validatedAddress = LinkSecurityUtils.validateUrl(url); - } catch (IllegalArgumentException e) { - throw new ServerErrorException("URL blocked for security reasons", e); - } - - if ("https".equalsIgnoreCase(url.getProtocol())) { - return executePinnedHttps(url, validatedAddress, options); - } - return executeHttp(url, validatedAddress, options); - } - - private static FetchResponse executeHttp(URL url, InetAddress validatedAddress, - FetchOptions options) throws IOException { - String connectUrl = "http".equalsIgnoreCase(url.getProtocol()) - ? LinkSecurityUtils.toConnectUrl(url, validatedAddress) - : url.toExternalForm(); - - Map headers = requestHeaders(url, options); - - Connection.Response response = Jsoup.connect(connectUrl) - .followRedirects(false) - .ignoreHttpErrors(true) - .ignoreContentType(options.ignoreContentType) - .maxBodySize(options.maxBodySize + 1) - .timeout(options.timeout) - .headers(headers) - .execute(); - String body = bodyWithinLimit(response, options); - return new FetchResponse(url, response.statusCode(), body, - response.header("ETag"), response.header("Last-Modified"), - response.header("Location")); - } - - private static FetchResponse executePinnedHttps(URL url, InetAddress validatedAddress, - FetchOptions options) throws IOException { - try (Socket socket = pinnedHttpsConnector.connect(url, validatedAddress, options.timeout)) { - writeRequest(socket.getOutputStream(), url, options); - return readResponse(socket.getInputStream(), url, options); - } - } - - private static Socket connectPinnedHttpsSocket(URL url, InetAddress address, int timeout) - throws IOException { - int port = effectivePort(url); - Socket rawSocket = new Socket(); - try { - rawSocket.connect(new InetSocketAddress(address, port), timeout); - rawSocket.setSoTimeout(timeout); - SSLSocketFactory sslSocketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault(); - SSLSocket sslSocket = (SSLSocket) sslSocketFactory - .createSocket(rawSocket, url.getHost(), port, true); - try { - SSLParameters parameters = sslSocket.getSSLParameters(); - parameters.setEndpointIdentificationAlgorithm("HTTPS"); - sslSocket.setSSLParameters(parameters); - sslSocket.startHandshake(); - return sslSocket; - } catch (IOException e) { - sslSocket.close(); - throw e; - } - } catch (IOException e) { - rawSocket.close(); - throw e; - } - } - - private static void writeRequest(OutputStream outputStream, URL url, FetchOptions options) - throws IOException { - String target = url.getFile(); - if (target == null || target.isBlank()) { - target = "/"; - } - StringBuilder request = new StringBuilder(); - request.append("GET ").append(target).append(" HTTP/1.1\r\n"); - requestHeaders(url, options).forEach((name, value) -> - request.append(name).append(": ").append(value).append("\r\n")); - request.append("Connection: close\r\n\r\n"); - outputStream.write(request.toString().getBytes(StandardCharsets.ISO_8859_1)); - outputStream.flush(); + private static FetchResponse execute(URL url, FetchOptions options) { + validateHttpUrl(url); + URI uri = toUri(url); + FetchResponse response = webClient(options).get() + .uri(uri) + .headers(headers -> requestHeaders(url, options).forEach(headers::set)) + .exchangeToMono(clientResponse -> toFetchResponse(url, options, clientResponse)) + .block(); + if (response == null) { + throw new ServerErrorException("Failed to fetch URL", + new IllegalStateException("Empty response")); + } + return response; } - private static FetchResponse readResponse(InputStream inputStream, URL url, FetchOptions options) - throws IOException { - String statusLine = readAsciiLine(inputStream); - if (statusLine == null || !statusLine.startsWith("HTTP/")) { - throw new IOException("Invalid HTTP response status line"); - } - String[] statusParts = statusLine.split(" ", 3); - if (statusParts.length < 2) { - throw new IOException("Invalid HTTP response status line"); - } - int statusCode = Integer.parseInt(statusParts[1]); - Map> headers = readHeaders(inputStream); - byte[] body; - try { - body = readBody(inputStream, headers, options.maxBodySize); - } catch (ServerErrorException e) { - if (!options.allowOversizedBody || !isResponseTooLarge(e)) { - throw e; - } - body = new byte[0]; - } - String contentType = header(headers, "content-type"); - return new FetchResponse(url, statusCode, decodeBody(body, contentType), - header(headers, "etag"), header(headers, "last-modified"), - header(headers, "location")); + private static WebClient webClient(FetchOptions options) { + WebClient.Builder builder = WebClient.builder() + .filter(HttpSecurityUtils.maxResponseSizeFilter(options.maxBodySize)); + ExchangeFunction exchangeFunction = exchangeFunctionForTesting; + if (exchangeFunction != null) { + return builder.exchangeFunction(exchangeFunction).build(); + } + HttpClient httpClient = HttpSecurityUtils.secureHttpClient() + .responseTimeout(Duration.ofMillis(options.timeout)) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, options.timeout); + return builder.clientConnector(new ReactorClientHttpConnector(httpClient)).build(); } - private static Map> readHeaders(InputStream inputStream) - throws IOException { - Map> headers = new LinkedHashMap<>(); - String line; - while ((line = readAsciiLine(inputStream)) != null && !line.isEmpty()) { - int separator = line.indexOf(':'); - if (separator <= 0) { - continue; - } - String name = line.substring(0, separator).trim().toLowerCase(Locale.ROOT); - String value = line.substring(separator + 1).trim(); - headers.computeIfAbsent(name, ignored -> new ArrayList<>()).add(value); - } - return headers; + private static Mono toFetchResponse(URL url, FetchOptions options, + ClientResponse response) { + HttpHeaders headers = response.headers().asHttpHeaders(); + return bodyBytes(response, options) + .map(body -> new FetchResponse(url, response.statusCode().value(), + decodeBody(body, headers.getFirst(HttpHeaders.CONTENT_TYPE)), + headers.getFirst(HttpHeaders.ETAG), headers.getFirst(HttpHeaders.LAST_MODIFIED), + headers.getFirst(HttpHeaders.LOCATION))); } - private static byte[] readBody(InputStream inputStream, Map> headers, - int maxBodySize) throws IOException { - String transferEncoding = header(headers, "transfer-encoding"); - if (transferEncoding != null - && transferEncoding.toLowerCase(Locale.ROOT).contains("chunked")) { - return readChunkedBody(inputStream, maxBodySize); - } - String contentLength = header(headers, "content-length"); - if (contentLength != null && !contentLength.isBlank()) { - try { - long length = Long.parseLong(contentLength); - if (length > maxBodySize) { - throw responseTooLarge("Content-Length exceeds " + maxBodySize); + private static Mono bodyBytes(ClientResponse response, FetchOptions options) { + return DataBufferUtils.join(response.body(BodyExtractors.toDataBuffers())) + .map(SafeUrlFetcher::readAndRelease) + .defaultIfEmpty(new byte[0]) + .onErrorResume(error -> { + if (!isResponseTooLarge(error)) { + return Mono.error(error); } - return inputStream.readNBytes((int) length); - } catch (NumberFormatException ignored) { - // Ignore malformed Content-Length and fall back to reading until EOF. - } - } - return readUntilEof(inputStream, maxBodySize); - } - - private static byte[] readChunkedBody(InputStream inputStream, int maxBodySize) - throws IOException { - var body = new java.io.ByteArrayOutputStream(); - while (true) { - String chunkHeader = readAsciiLine(inputStream); - if (chunkHeader == null) { - throw new IOException("Unexpected EOF in chunked body"); - } - int separator = chunkHeader.indexOf(';'); - String sizeText = separator >= 0 ? chunkHeader.substring(0, separator) : chunkHeader; - int size = Integer.parseInt(sizeText.trim(), 16); - if (size == 0) { - while (true) { - String trailer = readAsciiLine(inputStream); - if (trailer == null || trailer.isEmpty()) { - return body.toByteArray(); - } + if (options.allowOversizedBody) { + return Mono.just(new byte[0]); } - } - appendWithinLimit(body, inputStream.readNBytes(size), maxBodySize); - readAsciiLine(inputStream); - } + return Mono.error(responseTooLarge(error.getMessage())); + }); } - private static byte[] readUntilEof(InputStream inputStream, int maxBodySize) - throws IOException { - var body = new java.io.ByteArrayOutputStream(); - byte[] buffer = new byte[8192]; - int read; - while ((read = inputStream.read(buffer)) != -1) { - if (body.size() + read > maxBodySize) { - throw responseTooLarge("Body exceeds " + maxBodySize); - } - body.write(buffer, 0, read); + private static byte[] readAndRelease(DataBuffer dataBuffer) { + try { + byte[] bytes = new byte[dataBuffer.readableByteCount()]; + dataBuffer.read(bytes); + return bytes; + } finally { + DataBufferUtils.release(dataBuffer); } - return body.toByteArray(); } - private static void appendWithinLimit(java.io.ByteArrayOutputStream output, byte[] bytes, - int maxBodySize) { - if (output.size() + bytes.length > maxBodySize) { - throw responseTooLarge("Body exceeds " + maxBodySize); + private static void validateHttpUrl(URL url) { + String protocol = url.getProtocol().toLowerCase(Locale.ROOT); + if (!"http".equals(protocol) && !"https".equals(protocol)) { + throw new ServerErrorException("URL blocked for security reasons", + new IllegalArgumentException("Only HTTP and HTTPS protocols are allowed: " + url)); + } + if (url.getHost() == null || url.getHost().isBlank()) { + throw new ServerErrorException("Invalid URL", + new IllegalArgumentException("URL must have a host: " + url)); } - output.writeBytes(bytes); } - private static String readAsciiLine(InputStream inputStream) throws IOException { - var line = new java.io.ByteArrayOutputStream(); - int value; - while ((value = inputStream.read()) != -1) { - if (value == '\n') { - break; - } - if (value != '\r') { - line.write(value); - if (line.size() > MAX_HEADER_LINE_SIZE) { - throw new IOException("HTTP header line is too large"); - } - } - } - if (value == -1 && line.size() == 0) { - return null; + private static URI toUri(URL url) { + try { + return url.toURI(); + } catch (URISyntaxException e) { + throw new ServerErrorException("Invalid URL", e); } - return line.toString(StandardCharsets.ISO_8859_1); } private static String decodeBody(byte[] body, String contentType) { @@ -326,24 +196,19 @@ private static String decodeBody(byte[] body, String contentType) { return new String(body, charset); } - private static String header(Map> headers, String name) { - List values = headers.get(name.toLowerCase(Locale.ROOT)); - return values == null || values.isEmpty() ? null : values.get(0); - } - private static Map requestHeaders(URL url, FetchOptions options) { Map headers = new HashMap<>(); - headers.put("Host", hostHeader(url)); - headers.put("User-Agent", USER_AGENT); - headers.put("Accept", options.accept); + headers.put(HttpHeaders.HOST, hostHeader(url)); + headers.put(HttpHeaders.USER_AGENT, USER_AGENT); + headers.put(HttpHeaders.ACCEPT, options.accept); if (options.referer != null && !options.referer.isBlank()) { - headers.put("Referer", options.referer); + headers.put(HttpHeaders.REFERER, options.referer); } if (options.etag != null && !options.etag.isBlank()) { - headers.put("If-None-Match", options.etag); + headers.put(HttpHeaders.IF_NONE_MATCH, options.etag); } if (options.lastModified != null && !options.lastModified.isBlank()) { - headers.put("If-Modified-Since", options.lastModified); + headers.put(HttpHeaders.IF_MODIFIED_SINCE, options.lastModified); } return headers; } @@ -356,17 +221,24 @@ private static String hostHeader(URL url) { return url.getHost() + ":" + port; } - private static int effectivePort(URL url) { - return url.getPort() == -1 ? url.getDefaultPort() : url.getPort(); - } - private static ServerErrorException responseTooLarge(String message) { return new ServerErrorException("Response exceeds maximum size", new IllegalStateException(message)); } - private static boolean isResponseTooLarge(ServerErrorException e) { - return "Response exceeds maximum size".equals(e.getReason()); + private static boolean isResponseTooLarge(Throwable error) { + Throwable current = error; + while (current != null) { + if (current instanceof DataBufferLimitException) { + return true; + } + if (current instanceof ServerErrorException serverErrorException + && "Response exceeds maximum size".equals(serverErrorException.getReason())) { + return true; + } + current = current.getCause(); + } + return false; } private static boolean isRedirect(int statusCode) { @@ -374,30 +246,6 @@ private static boolean isRedirect(int statusCode) { || statusCode == 303 || statusCode == 307 || statusCode == 308; } - private static String bodyWithinLimit(Connection.Response response, FetchOptions options) { - String contentLength = response.header("Content-Length"); - if (contentLength != null && !contentLength.isBlank()) { - try { - if (Long.parseLong(contentLength) > options.maxBodySize) { - if (options.allowOversizedBody) { - return ""; - } - throw responseTooLarge("Content-Length exceeds " + options.maxBodySize); - } - } catch (NumberFormatException ignored) { - // Ignore malformed Content-Length and fall back to body length. - } - } - String body = response.body(); - if (body != null && body.getBytes(StandardCharsets.UTF_8).length > options.maxBodySize) { - if (options.allowOversizedBody) { - return ""; - } - throw responseTooLarge("Body exceeds " + options.maxBodySize); - } - return body; - } - public record FetchResult(URL url, int statusCode, String body, Document document, String etag, String lastModified) { } @@ -406,15 +254,8 @@ record FetchResponse(URL url, int statusCode, String body, String etag, String l String location) { } - @FunctionalInterface - interface PinnedHttpsConnector { - Socket connect(URL url, InetAddress address, int timeout) throws IOException; - } - - static void setPinnedHttpsConnectorForTesting(PinnedHttpsConnector connector) { - pinnedHttpsConnector = connector == null - ? SafeUrlFetcher::connectPinnedHttpsSocket - : connector; + static void setExchangeFunctionForTesting(ExchangeFunction exchangeFunction) { + exchangeFunctionForTesting = exchangeFunction; } @Getter @@ -450,9 +291,9 @@ public static FetchOptions html(String referer) { } public static FetchOptions feed(String referer, String etag, String lastModified) { - return new FetchOptions("application/rss+xml,application/atom+xml,application/xml,text/xml", - referer, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_BODY_SIZE, true, false, etag, - lastModified, false); + return new FetchOptions("application/rss+xml,application/atom+xml,application/xml," + + "text/xml", referer, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_BODY_SIZE, true, false, + etag, lastModified, false); } public static FetchOptions verification(String referer, int maxBodySize) { diff --git a/src/main/resources/plugin.yaml b/src/main/resources/plugin.yaml index 5d3b34e..59a45a0 100644 --- a/src/main/resources/plugin.yaml +++ b/src/main/resources/plugin.yaml @@ -5,7 +5,7 @@ metadata: annotations: "store.halo.run/app-id": "app-hfbQg" spec: - requires: ">=2.22.5" + requires: ">=2.25.0" author: name: Halo website: https://github.com/halo-dev diff --git a/src/test/java/run/halo/links/dto/LinkRequestTest.java b/src/test/java/run/halo/links/dto/LinkRequestTest.java deleted file mode 100644 index 7768d1e..0000000 --- a/src/test/java/run/halo/links/dto/LinkRequestTest.java +++ /dev/null @@ -1,91 +0,0 @@ -package run.halo.links.dto; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.URL; -import org.jsoup.Connection; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; -import org.springframework.web.server.ServerErrorException; -import run.halo.links.security.LinkSecurityUtils; - -class LinkRequestTest { - - @Test - void shouldBlockDirectAccessToPrivateIp() { - assertThatThrownBy(() -> LinkRequest.getLinkDetail("http://192.168.1.1/")) - .isInstanceOf(ServerErrorException.class) - .hasMessageContaining("blocked"); - } - - @Test - void shouldBlockDirectAccessToLoopback() { - assertThatThrownBy(() -> LinkRequest.getLinkDetail("http://127.0.0.1:8090/actuator")) - .isInstanceOf(ServerErrorException.class) - .hasMessageContaining("blocked"); - } - - @Test - void shouldBlockNonHttpScheme() { - assertThatThrownBy(() -> LinkRequest.getLinkDetail("file:///etc/passwd")) - .isInstanceOf(ServerErrorException.class) - .hasMessageContaining("blocked"); - } - - @Test - void shouldFetchPublicUrlSuccessfully() { - LinkDetailDTO detail = LinkRequest.getLinkDetail("https://example.com"); - assertThat(detail.getTitle()).isNotBlank(); - } - - @Test - void shouldConnectUsingValidatedIpForHttp() throws Exception { - URL url = new URL("http://example.com"); - InetAddress mockAddr = mock(InetAddress.class); - when(mockAddr.getHostAddress()).thenReturn("93.184.216.34"); - - Connection mockConn = mock(Connection.class); - Connection.Response mockResponse = mock(Connection.Response.class); - when(mockResponse.statusCode()).thenReturn(200); - Document doc = new Document("http://example.com"); - when(mockResponse.parse()).thenReturn(doc); - when(mockResponse.header("Content-Length")).thenReturn(null); - when(mockResponse.body()).thenReturn(""); - - when(mockConn.followRedirects(false)).thenReturn(mockConn); - when(mockConn.ignoreHttpErrors(true)).thenReturn(mockConn); - when(mockConn.ignoreContentType(false)).thenReturn(mockConn); - when(mockConn.maxBodySize(anyInt())).thenReturn(mockConn); - when(mockConn.timeout(anyInt())).thenReturn(mockConn); - when(mockConn.headers(anyMap())).thenReturn(mockConn); - when(mockConn.execute()).thenReturn(mockResponse); - - try (MockedStatic security = mockStatic(LinkSecurityUtils.class); - MockedStatic jsoup = mockStatic(Jsoup.class)) { - - security.when(() -> LinkSecurityUtils.validateUrl(url)).thenReturn(mockAddr); - security.when(() -> LinkSecurityUtils.toConnectUrl(url, mockAddr)) - .thenReturn("http://93.184.216.34"); - security.when(LinkSecurityUtils::getMaxRedirects).thenReturn(3); - - jsoup.when(() -> Jsoup.connect("http://93.184.216.34")).thenReturn(mockConn); - jsoup.when(() -> Jsoup.parse("", "http://example.com")).thenReturn(doc); - - LinkDetailDTO detail = LinkRequest.getLinkDetail("http://example.com"); - - assertThat(detail).isNotNull(); - jsoup.verify(() -> Jsoup.connect("http://93.184.216.34")); - } - } -} diff --git a/src/test/java/run/halo/links/security/LinkRequestTest.java b/src/test/java/run/halo/links/security/LinkRequestTest.java new file mode 100644 index 0000000..56b46c7 --- /dev/null +++ b/src/test/java/run/halo/links/security/LinkRequestTest.java @@ -0,0 +1,71 @@ +package run.halo.links.security; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.function.Consumer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.web.reactive.function.client.ClientResponse; +import org.springframework.web.server.ServerErrorException; +import reactor.core.publisher.Mono; +import run.halo.links.dto.LinkDetailDTO; +import run.halo.links.dto.LinkRequest; + +class LinkRequestTest { + + @AfterEach + void tearDown() { + SafeUrlFetcher.setExchangeFunctionForTesting(null); + } + + @Test + void shouldBlockDirectAccessToPrivateIp() { + assertThatThrownBy(() -> LinkRequest.getLinkDetail("http://192.168.1.1/")) + .isInstanceOf(ServerErrorException.class); + } + + @Test + void shouldBlockDirectAccessToLoopback() { + assertThatThrownBy(() -> LinkRequest.getLinkDetail("http://127.0.0.1:8090/actuator")) + .isInstanceOf(ServerErrorException.class); + } + + @Test + void shouldBlockNonHttpScheme() { + assertThatThrownBy(() -> LinkRequest.getLinkDetail("file:///etc/passwd")) + .isInstanceOf(ServerErrorException.class) + .hasMessageContaining("blocked"); + } + + @Test + void shouldFetchPublicUrlSuccessfully() { + SafeUrlFetcher.setExchangeFunctionForTesting(request -> Mono.just( + response(HttpStatus.OK, """ + + + Example Site + + + + + + """, builder -> builder.header(HttpHeaders.CONTENT_TYPE, "text/html")))); + + LinkDetailDTO detail = LinkRequest.getLinkDetail("https://example.com"); + + assertThat(detail.getTitle()).isEqualTo("Example Site"); + assertThat(detail.getDescription()).isEqualTo("Example description"); + assertThat(detail.getIcon()).isEqualTo("https://example.com/favicon.ico"); + assertThat(detail.getImage()).isEqualTo("https://example.com/preview.png"); + } + + private static ClientResponse response(HttpStatus status, String body, + Consumer customizer) { + ClientResponse.Builder builder = ClientResponse.create(status); + customizer.accept(builder); + return builder.body(body).build(); + } +} diff --git a/src/test/java/run/halo/links/security/LinkSecurityUtilsTest.java b/src/test/java/run/halo/links/security/LinkSecurityUtilsTest.java deleted file mode 100644 index 4e3217b..0000000 --- a/src/test/java/run/halo/links/security/LinkSecurityUtilsTest.java +++ /dev/null @@ -1,133 +0,0 @@ -package run.halo.links.security; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import java.net.InetAddress; -import org.junit.jupiter.api.Test; - -class LinkSecurityUtilsTest { - - @Test - void shouldBlockIpv4Loopback() { - assertThatThrownBy(() -> LinkSecurityUtils.validateUrl("http://127.0.0.1/actuator")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("private/reserved"); - } - - @Test - void shouldBlockIpv4PrivateRange10() { - assertThatThrownBy(() -> LinkSecurityUtils.validateUrl("http://10.0.0.1/")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("private/reserved"); - } - - @Test - void shouldBlockIpv4PrivateRange172() { - assertThatThrownBy(() -> LinkSecurityUtils.validateUrl("http://172.16.0.1/")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("private/reserved"); - } - - @Test - void shouldBlockIpv4PrivateRange192() { - assertThatThrownBy(() -> LinkSecurityUtils.validateUrl("http://192.168.1.1/")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("private/reserved"); - } - - @Test - void shouldBlockIpv6Loopback() { - assertThatThrownBy(() -> LinkSecurityUtils.validateUrl("http://[::1]/")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("private/reserved"); - } - - @Test - void shouldBlockIpv6LinkLocal() { - assertThatThrownBy(() -> LinkSecurityUtils.validateUrl("http://[fe80::1]/")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("private/reserved"); - } - - @Test - void shouldAllowPublicUrl() { - assertThatCode(() -> LinkSecurityUtils.validateUrl("https://example.com")) - .doesNotThrowAnyException(); - } - - @Test - void shouldAllowPublicIp() { - assertThatCode(() -> LinkSecurityUtils.validateUrl("http://8.8.8.8")) - .doesNotThrowAnyException(); - } - - @Test - void shouldBlockFileScheme() { - assertThatThrownBy(() -> LinkSecurityUtils.validateUrl("file:///etc/passwd")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Only HTTP and HTTPS"); - } - - @Test - void shouldBlockFtpScheme() { - assertThatThrownBy(() -> LinkSecurityUtils.validateUrl("ftp://internal.ftp.server/resource")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Only HTTP and HTTPS"); - } - - @Test - void isPrivateAddressShouldDetectLoopback() throws Exception { - InetAddress loopback = InetAddress.getByName("127.0.0.1"); - assertThat(LinkSecurityUtils.isPrivateAddress(loopback)).isTrue(); - } - - @Test - void isPrivateAddressShouldDetectSiteLocal() throws Exception { - InetAddress siteLocal = InetAddress.getByName("192.168.1.1"); - assertThat(LinkSecurityUtils.isPrivateAddress(siteLocal)).isTrue(); - } - - @Test - void isPrivateAddressShouldReturnFalseForPublic() throws Exception { - InetAddress publicAddr = InetAddress.getByName("8.8.8.8"); - assertThat(LinkSecurityUtils.isPrivateAddress(publicAddr)).isFalse(); - } - - @Test - void shouldBlock169254Range() throws Exception { - InetAddress apipa = InetAddress.getByName("169.254.1.1"); - assertThat(LinkSecurityUtils.isPrivateAddress(apipa)).isTrue(); - } - - @Test - void shouldBlockIpv6UlaFc00() throws Exception { - InetAddress ula = InetAddress.getByName("fc00::1"); - assertThat(LinkSecurityUtils.isPrivateAddress(ula)).isTrue(); - } - - @Test - void shouldBlockIpv6UlaFd00() throws Exception { - InetAddress ula = InetAddress.getByName("fd00::1"); - assertThat(LinkSecurityUtils.isPrivateAddress(ula)).isTrue(); - } - - @Test - void shouldBlockAnyLocalAddress() throws Exception { - InetAddress anyLocal = InetAddress.getByName("0.0.0.0"); - assertThat(LinkSecurityUtils.isPrivateAddress(anyLocal)).isTrue(); - } - - @Test - void shouldBlockMulticastAddress() throws Exception { - InetAddress multicast = InetAddress.getByName("224.0.0.1"); - assertThat(LinkSecurityUtils.isPrivateAddress(multicast)).isTrue(); - } - - @Test - void shouldAllowIpv6PublicAddress() throws Exception { - InetAddress publicAddr = InetAddress.getByName("2001:db8::1"); - assertThat(LinkSecurityUtils.isPrivateAddress(publicAddr)).isFalse(); - } -} diff --git a/src/test/java/run/halo/links/security/RedirectHandlerTest.java b/src/test/java/run/halo/links/security/RedirectHandlerTest.java deleted file mode 100644 index 1014d7c..0000000 --- a/src/test/java/run/halo/links/security/RedirectHandlerTest.java +++ /dev/null @@ -1,144 +0,0 @@ -package run.halo.links.security; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.net.InetAddress; -import java.util.Map; -import org.jsoup.Connection; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; -import org.springframework.web.server.ServerErrorException; - -class RedirectHandlerTest { - - private static final Map HEADERS = Map.of("User-Agent", "test"); - private static final int TIMEOUT = 10000; - private static final int MAX_BODY = 1024 * 1024 * 20; - - @Test - void shouldBlockRedirectToPrivateIp() throws IOException { - Connection.Response firstResponse = mockResponse(302, "http://192.168.1.1/secret"); - - try (MockedStatic security = mockStatic(LinkSecurityUtils.class)) { - security.when(() -> LinkSecurityUtils.validateUrl(any(java.net.URL.class))) - .thenThrow(new IllegalArgumentException("private")); - security.when(LinkSecurityUtils::getMaxRedirects).thenReturn(3); - - RedirectHandler handler = new RedirectHandler(HEADERS, TIMEOUT, MAX_BODY); - - assertThatThrownBy(() -> handler.followRedirects(firstResponse)) - .isInstanceOf(ServerErrorException.class) - .hasMessageContaining("Redirect blocked"); - } - } - - @Test - void shouldBlockTooManyRedirects() throws IOException { - Connection.Response response = mockResponse(302, "http://example.com/next"); - - try (MockedStatic jsoup = mockStatic(Jsoup.class); - MockedStatic security = mockStatic(LinkSecurityUtils.class)) { - Connection mockConn = mock(Connection.class); - when(mockConn.followRedirects(false)).thenReturn(mockConn); - when(mockConn.timeout(anyInt())).thenReturn(mockConn); - when(mockConn.maxBodySize(anyInt())).thenReturn(mockConn); - when(mockConn.headers(anyMap())).thenReturn(mockConn); - when(mockConn.execute()).thenReturn(response); - - jsoup.when(() -> Jsoup.connect(anyString())).thenReturn(mockConn); - - InetAddress mockAddr = mock(InetAddress.class); - when(mockAddr.getHostAddress()).thenReturn("1.2.3.4"); - security.when(() -> LinkSecurityUtils.validateUrl(any(java.net.URL.class))) - .thenReturn(mockAddr); - security.when(() -> LinkSecurityUtils.toConnectUrl( - any(java.net.URL.class), any(InetAddress.class))) - .thenReturn("http://1.2.3.4/next"); - security.when(LinkSecurityUtils::getMaxRedirects).thenReturn(3); - - RedirectHandler handler = new RedirectHandler(HEADERS, TIMEOUT, MAX_BODY); - - assertThatThrownBy(() -> handler.followRedirects(response)) - .isInstanceOf(ServerErrorException.class) - .hasMessageContaining("Too many redirects"); - } - } - - @Test - void shouldFollowValidRedirectsAndReturnDocument() throws IOException { - Connection.Response redirectResponse = - mockResponse(302, "https://example.com/final"); - Document expectedDoc = new Document("https://example.com/final"); - Connection.Response finalResponse = mock(Response.class); - when(finalResponse.statusCode()).thenReturn(200); - when(finalResponse.parse()).thenReturn(expectedDoc); - - InetAddress mockAddr = mock(InetAddress.class); - when(mockAddr.getHostAddress()).thenReturn("93.184.216.34"); - - try (MockedStatic jsoup = mockStatic(Jsoup.class); - MockedStatic security = mockStatic(LinkSecurityUtils.class)) { - Connection mockConn = mock(Connection.class); - when(mockConn.followRedirects(false)).thenReturn(mockConn); - when(mockConn.timeout(anyInt())).thenReturn(mockConn); - when(mockConn.maxBodySize(anyInt())).thenReturn(mockConn); - when(mockConn.headers(anyMap())).thenReturn(mockConn); - when(mockConn.execute()).thenReturn(finalResponse); - - jsoup.when(() -> Jsoup.connect(anyString())).thenReturn(mockConn); - security.when(() -> LinkSecurityUtils.validateUrl(any(java.net.URL.class))) - .thenReturn(mockAddr); - security.when(() -> LinkSecurityUtils.toConnectUrl( - any(java.net.URL.class), eq(mockAddr))) - .thenReturn("https://93.184.216.34/final"); - security.when(LinkSecurityUtils::getMaxRedirects).thenReturn(3); - - RedirectHandler handler = new RedirectHandler(HEADERS, TIMEOUT, MAX_BODY); - Document result = handler.followRedirects(redirectResponse); - - assertThat(result).isEqualTo(expectedDoc); - } - } - - @Test - void shouldBlockSchemeDowngradeInRedirect() throws IOException { - Connection.Response firstResponse = mockResponse(302, "ftp://evil.com/file"); - - try (MockedStatic security = mockStatic(LinkSecurityUtils.class)) { - security.when(() -> LinkSecurityUtils.validateUrl(any(java.net.URL.class))) - .thenThrow(new IllegalArgumentException("scheme")); - security.when(LinkSecurityUtils::getMaxRedirects).thenReturn(3); - - RedirectHandler handler = new RedirectHandler(HEADERS, TIMEOUT, MAX_BODY); - - assertThatThrownBy(() -> handler.followRedirects(firstResponse)) - .isInstanceOf(ServerErrorException.class) - .hasMessageContaining("Redirect blocked"); - } - } - - private Connection.Response mockResponse(int statusCode, String location) - throws IOException { - Connection.Response response = mock(Response.class); - when(response.statusCode()).thenReturn(statusCode); - when(response.header("Location")).thenReturn(location); - when(response.url()).thenReturn(new java.net.URL("https://example.com/start")); - return response; - } - - // Helper interface to avoid mocking the full Connection.Response - private interface Response extends Connection.Response { - } -} diff --git a/src/test/java/run/halo/links/security/SafeUrlFetcherTest.java b/src/test/java/run/halo/links/security/SafeUrlFetcherTest.java index c41006c..246295f 100644 --- a/src/test/java/run/halo/links/security/SafeUrlFetcherTest.java +++ b/src/test/java/run/halo/links/security/SafeUrlFetcherTest.java @@ -2,242 +2,152 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.when; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetAddress; -import java.net.Socket; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicReference; -import org.jsoup.Connection; -import org.jsoup.Jsoup; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.web.reactive.function.client.ClientRequest; +import org.springframework.web.reactive.function.client.ClientResponse; import org.springframework.web.server.ServerErrorException; +import reactor.core.publisher.Mono; class SafeUrlFetcherTest { + @AfterEach + void tearDown() { + SafeUrlFetcher.setExchangeFunctionForTesting(null); + } + @Test void shouldBlockPrivateFeedUrlBeforeConnecting() { assertThatThrownBy(() -> SafeUrlFetcher.fetch("http://127.0.0.1/feed.xml", - SafeUrlFetcher.FetchOptions.feed("http://example.com", null, null))) - .isInstanceOf(ServerErrorException.class) - .hasMessageContaining("blocked"); + SafeUrlFetcher.FetchOptions.verification("http://example.com", 64, 200))) + .isInstanceOf(ServerErrorException.class); } @Test - void shouldBlockRedirectToPrivateAddress() throws Exception { - URL startUrl = new URL("http://example.com/feed.xml"); - InetAddress publicAddress = mock(InetAddress.class); - when(publicAddress.getHostAddress()).thenReturn("93.184.216.34"); - - Connection mockConn = mock(Connection.class); - Connection.Response redirectResponse = mockResponse(302, "http://192.168.1.1/feed.xml"); - - when(mockConn.followRedirects(false)).thenReturn(mockConn); - when(mockConn.ignoreHttpErrors(true)).thenReturn(mockConn); - when(mockConn.ignoreContentType(true)).thenReturn(mockConn); - when(mockConn.maxBodySize(anyInt())).thenReturn(mockConn); - when(mockConn.timeout(anyInt())).thenReturn(mockConn); - when(mockConn.headers(anyMap())).thenReturn(mockConn); - when(mockConn.execute()).thenReturn(redirectResponse); - - try (MockedStatic security = mockStatic(LinkSecurityUtils.class); - MockedStatic jsoup = mockStatic(Jsoup.class)) { - security.when(() -> LinkSecurityUtils.validateUrl(any(URL.class))) - .thenAnswer(invocation -> { - URL url = invocation.getArgument(0); - if (startUrl.equals(url)) { - return publicAddress; - } - throw new IllegalArgumentException("private"); - }); - security.when(() -> LinkSecurityUtils.toConnectUrl(startUrl, publicAddress)) - .thenReturn("http://93.184.216.34/feed.xml"); - security.when(LinkSecurityUtils::getMaxRedirects).thenReturn(3); - jsoup.when(() -> Jsoup.connect("http://93.184.216.34/feed.xml")) - .thenReturn(mockConn); - - assertThatThrownBy(() -> SafeUrlFetcher.fetch(startUrl.toExternalForm(), - SafeUrlFetcher.FetchOptions.feed(startUrl.toExternalForm(), null, null))) - .isInstanceOf(ServerErrorException.class) - .hasMessageContaining("blocked"); - } + void shouldRejectNonHttpScheme() { + assertThatThrownBy(() -> SafeUrlFetcher.fetch("file:///etc/passwd", + SafeUrlFetcher.FetchOptions.html("file:///etc/passwd"))) + .isInstanceOf(ServerErrorException.class) + .hasMessageContaining("blocked"); } @Test - void shouldRejectResponseOverMaximumBodySize() throws Exception { - URL startUrl = new URL("http://example.com/feed.xml"); - InetAddress publicAddress = mock(InetAddress.class); - when(publicAddress.getHostAddress()).thenReturn("93.184.216.34"); - - Connection mockConn = mock(Connection.class); - Connection.Response oversizedResponse = mockResponse(200, null); - when(oversizedResponse.header("Content-Length")).thenReturn( - String.valueOf(SafeUrlFetcher.DEFAULT_MAX_BODY_SIZE + 1L)); - - when(mockConn.followRedirects(false)).thenReturn(mockConn); - when(mockConn.ignoreHttpErrors(true)).thenReturn(mockConn); - when(mockConn.ignoreContentType(true)).thenReturn(mockConn); - when(mockConn.maxBodySize(anyInt())).thenReturn(mockConn); - when(mockConn.timeout(anyInt())).thenReturn(mockConn); - when(mockConn.headers(anyMap())).thenReturn(mockConn); - when(mockConn.execute()).thenReturn(oversizedResponse); - - try (MockedStatic security = mockStatic(LinkSecurityUtils.class); - MockedStatic jsoup = mockStatic(Jsoup.class)) { - security.when(() -> LinkSecurityUtils.validateUrl(startUrl)).thenReturn(publicAddress); - security.when(() -> LinkSecurityUtils.toConnectUrl(startUrl, publicAddress)) - .thenReturn("http://93.184.216.34/feed.xml"); - security.when(LinkSecurityUtils::getMaxRedirects).thenReturn(3); - jsoup.when(() -> Jsoup.connect("http://93.184.216.34/feed.xml")) - .thenReturn(mockConn); - - assertThatThrownBy(() -> SafeUrlFetcher.fetch(startUrl.toExternalForm(), - SafeUrlFetcher.FetchOptions.feed(startUrl.toExternalForm(), null, null))) - .isInstanceOf(ServerErrorException.class) - .hasMessageContaining("maximum size"); - } + void shouldFollowRedirectAndPreserveHeaders() { + List requests = new ArrayList<>(); + String startUrl = "http://example.com/feed.xml"; + SafeUrlFetcher.setExchangeFunctionForTesting(request -> { + requests.add(request); + if (request.url().getPath().equals("/feed.xml")) { + return Mono.just(response(HttpStatus.FOUND, "", builder -> + builder.header(HttpHeaders.LOCATION, "/final.xml"))); + } + return Mono.just(response(HttpStatus.OK, "", builder -> builder + .header(HttpHeaders.ETAG, "\"v2\"") + .header(HttpHeaders.LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT"))); + }); + + SafeUrlFetcher.FetchResult result = SafeUrlFetcher.fetch(startUrl, + SafeUrlFetcher.FetchOptions.feed(startUrl, "\"v1\"", + "Tue, 20 Oct 2015 07:28:00 GMT")); + + assertThat(result.url().toExternalForm()).isEqualTo("http://example.com/final.xml"); + assertThat(result.statusCode()).isEqualTo(200); + assertThat(result.body()).isEqualTo(""); + assertThat(result.etag()).isEqualTo("\"v2\""); + assertThat(result.lastModified()).isEqualTo("Wed, 21 Oct 2015 07:28:00 GMT"); + assertThat(requests).hasSize(2); + ClientRequest firstRequest = requests.getFirst(); + assertThat(firstRequest.url()).isEqualTo(URI.create(startUrl)); + assertThat(firstRequest.headers().getFirst(HttpHeaders.HOST)).isEqualTo("example.com"); + assertThat(firstRequest.headers().getFirst(HttpHeaders.USER_AGENT)) + .isEqualTo(SafeUrlFetcher.USER_AGENT); + assertThat(firstRequest.headers().getFirst(HttpHeaders.ACCEPT)) + .isEqualTo("application/rss+xml,application/atom+xml,application/xml,text/xml"); + assertThat(firstRequest.headers().getFirst(HttpHeaders.REFERER)).isEqualTo(startUrl); + assertThat(firstRequest.headers().getFirst(HttpHeaders.IF_NONE_MATCH)).isEqualTo("\"v1\""); + assertThat(firstRequest.headers().getFirst(HttpHeaders.IF_MODIFIED_SINCE)) + .isEqualTo("Tue, 20 Oct 2015 07:28:00 GMT"); } @Test - void shouldKeepVerificationStatusWhenResponseOverMaximumBodySize() throws Exception { - URL startUrl = new URL("http://example.com"); - InetAddress publicAddress = mock(InetAddress.class); - when(publicAddress.getHostAddress()).thenReturn("93.184.216.34"); - - Connection mockConn = mock(Connection.class); - Connection.Response oversizedResponse = mockResponse(200, null); - when(oversizedResponse.header("Content-Length")).thenReturn("65537"); - - when(mockConn.followRedirects(false)).thenReturn(mockConn); - when(mockConn.ignoreHttpErrors(true)).thenReturn(mockConn); - when(mockConn.ignoreContentType(true)).thenReturn(mockConn); - when(mockConn.maxBodySize(anyInt())).thenReturn(mockConn); - when(mockConn.timeout(anyInt())).thenReturn(mockConn); - when(mockConn.headers(anyMap())).thenReturn(mockConn); - when(mockConn.execute()).thenReturn(oversizedResponse); - - try (MockedStatic security = mockStatic(LinkSecurityUtils.class); - MockedStatic jsoup = mockStatic(Jsoup.class)) { - security.when(() -> LinkSecurityUtils.validateUrl(startUrl)).thenReturn(publicAddress); - security.when(() -> LinkSecurityUtils.toConnectUrl(startUrl, publicAddress)) - .thenReturn("http://93.184.216.34"); - security.when(LinkSecurityUtils::getMaxRedirects).thenReturn(3); - jsoup.when(() -> Jsoup.connect("http://93.184.216.34")) - .thenReturn(mockConn); - - SafeUrlFetcher.FetchResult result = SafeUrlFetcher.fetch(startUrl.toExternalForm(), - SafeUrlFetcher.FetchOptions.verification(startUrl.toExternalForm(), 64 * 1024)); - - assertThat(result.statusCode()).isEqualTo(200); - assertThat(result.body()).isEmpty(); - } + void shouldBlockTooManyRedirects() { + SafeUrlFetcher.setExchangeFunctionForTesting(request -> Mono.just( + response(HttpStatus.FOUND, "", builder -> + builder.header(HttpHeaders.LOCATION, "/next")))); + + assertThatThrownBy(() -> SafeUrlFetcher.fetch("http://example.com/start", + SafeUrlFetcher.FetchOptions.feed("http://example.com/start", null, null))) + .isInstanceOf(ServerErrorException.class) + .hasMessageContaining("Too many redirects"); } @Test - void shouldKeepHttpsVerificationStatusWhenResponseOverMaximumBodySize() throws Exception { - URL startUrl = new URL("https://example.com"); - InetAddress validatedAddress = InetAddress.getByName("203.0.113.10"); - FakeSocket socket = new FakeSocket("HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"); - - SafeUrlFetcher.setPinnedHttpsConnectorForTesting((url, address, timeout) -> socket); - - try (MockedStatic security = mockStatic(LinkSecurityUtils.class)) { - security.when(() -> LinkSecurityUtils.validateUrl(startUrl)) - .thenReturn(validatedAddress); - security.when(LinkSecurityUtils::getMaxRedirects).thenReturn(3); - - SafeUrlFetcher.FetchResult result = SafeUrlFetcher.fetch(startUrl.toExternalForm(), - SafeUrlFetcher.FetchOptions.verification(startUrl.toExternalForm(), 1)); - - assertThat(result.statusCode()).isEqualTo(200); - assertThat(result.body()).isEmpty(); - } finally { - SafeUrlFetcher.setPinnedHttpsConnectorForTesting(null); - } + void shouldRejectResponseOverMaximumBodySize() { + SafeUrlFetcher.setExchangeFunctionForTesting(request -> Mono.just( + response(HttpStatus.OK, "abcdef", builder -> { + }))); + + assertThatThrownBy(() -> SafeUrlFetcher.fetch("http://example.com", + SafeUrlFetcher.FetchOptions.verificationHtml("http://example.com", 5, 1000))) + .isInstanceOf(ServerErrorException.class) + .hasMessageContaining("maximum size"); } @Test - void shouldConnectHttpsThroughValidatedAddress() throws Exception { - URL startUrl = new URL("https://example.com/feed.xml"); - InetAddress validatedAddress = InetAddress.getByName("203.0.113.10"); - AtomicReference requestedUrl = new AtomicReference<>(); - AtomicReference connectedAddress = new AtomicReference<>(); - FakeSocket socket = new FakeSocket(""" - HTTP/1.1 200 OK\r - ETag: "feed-v1"\r - \r - """); - - SafeUrlFetcher.setPinnedHttpsConnectorForTesting((url, address, timeout) -> { - requestedUrl.set(url); - connectedAddress.set(address); - return socket; - }); + void shouldKeepVerificationStatusWhenResponseOverMaximumBodySize() { + SafeUrlFetcher.setExchangeFunctionForTesting(request -> Mono.just( + response(HttpStatus.OK, "abcdef", builder -> { + }))); - try (MockedStatic security = mockStatic(LinkSecurityUtils.class)) { - security.when(() -> LinkSecurityUtils.validateUrl(startUrl)) - .thenReturn(validatedAddress); - security.when(LinkSecurityUtils::getMaxRedirects).thenReturn(3); - - SafeUrlFetcher.FetchResult result = SafeUrlFetcher.fetch(startUrl.toExternalForm(), - SafeUrlFetcher.FetchOptions.feed(startUrl.toExternalForm(), null, null)); - - assertThat(result.statusCode()).isEqualTo(200); - assertThat(result.body()).isEqualTo(""); - assertThat(requestedUrl).hasValue(startUrl); - assertThat(connectedAddress).hasValue(validatedAddress); - assertThat(socket.output()).contains("Host: example.com"); - } finally { - SafeUrlFetcher.setPinnedHttpsConnectorForTesting(null); - } - } + SafeUrlFetcher.FetchResult result = SafeUrlFetcher.fetch("http://example.com", + SafeUrlFetcher.FetchOptions.verification("http://example.com", 5, 1000)); - private static Connection.Response mockResponse(int statusCode, String location) - throws IOException { - Connection.Response response = mock(Response.class); - when(response.statusCode()).thenReturn(statusCode); - when(response.header("Location")).thenReturn(location); - when(response.header("Content-Length")).thenReturn(null); - when(response.body()).thenReturn(""); - when(response.url()).thenReturn(new URL("http://example.com/feed.xml")); - return response; + assertThat(result.statusCode()).isEqualTo(200); + assertThat(result.body()).isEmpty(); } - private interface Response extends Connection.Response { + @Test + void shouldPreserveNotModifiedStatusAndHeaders() { + SafeUrlFetcher.setExchangeFunctionForTesting(request -> Mono.just( + response(HttpStatus.NOT_MODIFIED, "", builder -> builder + .header(HttpHeaders.ETAG, "\"v1\"") + .header(HttpHeaders.LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT")))); + + SafeUrlFetcher.FetchResult result = SafeUrlFetcher.fetch("http://example.com/feed.xml", + SafeUrlFetcher.FetchOptions.feed("http://example.com/feed.xml", "\"v1\"", + "Wed, 21 Oct 2015 07:28:00 GMT")); + + assertThat(result.statusCode()).isEqualTo(304); + assertThat(result.body()).isEmpty(); + assertThat(result.etag()).isEqualTo("\"v1\""); + assertThat(result.lastModified()).isEqualTo("Wed, 21 Oct 2015 07:28:00 GMT"); } - private static final class FakeSocket extends Socket { - private final ByteArrayInputStream input; - private final ByteArrayOutputStream output = new ByteArrayOutputStream(); - - private FakeSocket(String response) { - this.input = new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8)); - } + @Test + void shouldParseHtmlDocumentWhenRequested() { + SafeUrlFetcher.setExchangeFunctionForTesting(request -> Mono.just( + response(HttpStatus.OK, "Example", + builder -> builder.header(HttpHeaders.CONTENT_TYPE, "text/html; charset=UTF-8")))); - @Override - public InputStream getInputStream() { - return input; - } + SafeUrlFetcher.FetchResult result = SafeUrlFetcher.fetch("http://example.com", + SafeUrlFetcher.FetchOptions.html("http://example.com")); - @Override - public OutputStream getOutputStream() { - return output; - } + assertThat(result.statusCode()).isEqualTo(200); + assertThat(result.document()).isNotNull(); + assertThat(result.document().title()).isEqualTo("Example"); + } - private String output() { - return output.toString(StandardCharsets.UTF_8); - } + private static ClientResponse response(HttpStatus status, String body, + Consumer customizer) { + ClientResponse.Builder builder = ClientResponse.create(status); + customizer.accept(builder); + return builder.body(body).build(); } }