diff --git a/.gitignore b/.gitignore index 6e328c1..80c8a22 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ # llm agents /.gemini/ /.claude/ + +# Local planning and sandbox configuration +/PROJECT_NGINX.md +/.nono.json diff --git a/.woodpecker/build-docker.yml b/.woodpecker/build-docker.yml index d7e04e3..d102b41 100644 --- a/.woodpecker/build-docker.yml +++ b/.woodpecker/build-docker.yml @@ -9,11 +9,17 @@ when: include: - "**/*.go" - "**/*.gohtml" + - "go.mod" + - "go.sum" + - ".golangci.yml" + - "scripts/check-coverage.sh" + - "bin/generate-encrypted-token.sh" - ".woodpecker/build-docker.yml" - "Dockerfile" exclude: - "**/*.md" on_empty: true + - event: pull_request - event: tag ref: "refs/tags/v*.*.*" @@ -50,6 +56,29 @@ steps: when: - event: tag + extract-pr-tag: + image: alpine:3.22 + commands: + - | + branch_slug=$(printf '%s' "$CI_COMMIT_SOURCE_BRANCH" | sed 's/[^A-Za-z0-9_.-]/-/g' | cut -c1-96) + printf 'pr-%s-%s\n' "$CI_COMMIT_PULL_REQUEST" "$branch_slug" > .tags + cat .tags + when: + - event: pull_request + + build-docker-pr: + image: woodpeckerci/plugin-docker-buildx:6.0.3 + settings: + repo: git.eleith.com/eleith.private/readeckobo + username: eleith + dockerfile: Dockerfile + registry: https://git.eleith.com + password: + from_secret: gitea_docker_registry + tags_file: .tags + when: + - event: pull_request + build-docker-main: image: woodpeckerci/plugin-docker-buildx:6.0.3 settings: @@ -79,8 +108,8 @@ steps: - event: tag ref: "refs/tags/v*.*.*" - notify-success: - image: deblan/woodpecker-email:3.2.0 + notify: + image: deblan/woodpecker-email:3.3.0 settings: dsn: from_secret: woodpecker_email_dsn @@ -91,23 +120,8 @@ steps: from_secret: woodpecker_email_from_name recipients: from_secret: woodpecker_email_recipient - level: success recipients_only: true when: - - status: [success] - notify-failure: - image: deblan/woodpecker-email:3.2.0 - settings: - dsn: - from_secret: woodpecker_email_dsn - from: - address: - from_secret: woodpecker_email_from_address - name: - from_secret: woodpecker_email_from_name - recipients: - from_secret: woodpecker_email_recipient - level: failure - recipients_only: true - when: - - status: [failure] + - event: push + branch: main + status: [failure, success] diff --git a/Agents.md b/Agents.md index e491fb7..dc1f03b 100644 --- a/Agents.md +++ b/Agents.md @@ -16,8 +16,7 @@ The project follows a standard Go project layout: - **`internal/models/`**: Shared data structures for Kobo and Readeck API payloads. ## Deployment & Infrastructure -- **Proxy Interception**: The system relies on a reverse proxy (like Nginx) to intercept requests from the Kobo device intended for `www.instapaper.com` and redirect them to this service. -- **Configuration**: See `nginx.conf.snippet` for the rewrite rules. +- **Deployment**: Kobo requests can reach this app through Cloudflare Tunnel or Nginx. See `docs/CLOUDFLARE.md` and `docs/NGINX.md` for the routing and initialization rewrite examples. - **Docker**: The application is containerized (`Dockerfile`, `docker-compose.yml`). ## Development Guidelines for AI Agents diff --git a/README.md b/README.md index 8928c53..48c37d7 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,34 @@ more. ## 🚀 Quick Start (for Users) -Getting up and running is a breeze with Docker. +### 1. Choose how to publish -### 1. Configure `readeckobo` +Choose a public HTTPS hostname (for example, `readeckobo.example.com`) and one +deployment method: -First, copy `config.yaml.example` to `config.yaml` and edit it to match your setup. +* [Cloudflare Tunnel](docs/CLOUDFLARE.md): forward requests to `readeckobo`. +* [Nginx](docs/NGINX.md): proxy Kobo Store and article requests. -For a detailed explanation of all options and how to get your Readeck API token, see [docs/CONFIG.md](docs/CONFIG.md). +Both use the same Kobo settings below. + +### 2. Generate a device token + +Copy `config.yaml.example` to `config.yaml`, then build the image. Find your +Kobo's serial number under **Settings -> Device Information**. Replace +`` below with that number (without angle brackets): + +```sh +docker-compose build +docker-compose run --rm --entrypoint /app/bin/generate-encrypted-token.sh readeckobo +``` + +This one-off container generates a **plain text token** for `config.yaml` and +an **encrypted token** for the Kobo. The app is not running yet. + +### 3. Configure `readeckobo` and Kobo + +Edit `config.yaml` with your Readeck host and API token and the **plain text** +device token from step 2: ```yaml server: @@ -32,97 +53,48 @@ log_level: info readeck: host: "https://your-readeck-instance.com" users: - - token: "a-random-uuid-token-for-a-kobo" + - token: "" readeck_access_token: "a-readeck-api-token" ``` -### 2. Run with Docker +For other options, see [docs/CONFIG.md](docs/CONFIG.md). To sync ebooks too, +see the [Kobo ebook sync guide](docs/KOBO_SYNC.md) for the per-device +`api_endpoint` instead of the article-only setting below. -Once your configuration is ready, fire it up! +Mount your Kobo and edit `.kobo/Kobo/Kobo eReader.conf`, using the **encrypted** +token from the script. Replace `readeckobo.example.com` with the hostname you +chose in step 1: + +```ini +[OneStoreServices] +api_endpoint=https://readeckobo.example.com/instapaper-proxy/storeapi +instapaper_env_url=https://readeckobo.example.com + +[Instapaper] +AccessToken=@ByteArray() +``` + +### 4. Run the app ```sh -docker-compose build docker-compose up -d ``` -The server will be available at `http://localhost:8080`. - -### 3. Generate a Device Token - -For each Kobo device, you will need a unique token. This process involves -generating a token and then encrypting it for the Kobo device. - -First, find your Kobo's serial number, which is available under -**Settings -> Device Information** on your e-reader. - -With `readeckobo` running, use this command to generate and encrypt the token, -replacing `` with your device's serial number: - -```sh -docker-compose exec readeckobo bin/generate-encrypted-token.sh -``` - -The script will output two important pieces of information: - -1. A **plain text UUID token** to be used in your `config.yaml`. -2. An **encrypted token** to be used in your Kobo's configuration file. - -### 4. Configure Your `readeckobo` and Kobo Device - -Follow the output from the script to configure your services. - -1. **Update `config.yaml`**: Add the plain text UUID token to the `users` - section of your `config.yaml`. - - ```yaml - users: - - token: "" - readeck_access_token: "a-readeck-api-token" - ``` - -2. **Update Your Kobo**: Mount your Kobo and find the - `.kobo/Kobo/Kobo eReader.conf` file. Add or update these settings using the - **encrypted** token from the script's output. - - ```ini - [OneStoreServices] - api_endpoint=https://readeckobo.example.com/instapaper-proxy/storeapi - instapaper_env_url=https://readeckobo.example.com - - [Instapaper] - AccessToken=@ByteArray() - ``` - -Replace `https://readeckobo.example.com` with the full path to where you are deploying -the readeckobo service to - -### 5. Set Up a Reverse Proxy - -`readeckobo` must be run behind a reverse proxy to handle HTTPS. It's crucial -to proxy three specific location blocks, as shown in our `nginx.conf.snippet` -example. - -Your Kobo device periodically re-syncs its configuration from Kobo's servers, -which can overwrite your custom Instapaper endpoint. The proxy rules below -ensure this connection is preserved. - - -| Location Block | Proxies To | Purpose | -| ----------------------------------------------- | ------------------------ | --------------------------------------------------------------------------------------------------- | -| `/instapaper-proxy/instapaper/` | `readeckobo` application | Handles the main Instapaper API requests (sync, download, etc.) to your `readeckobo` instance. | -| `/instapaper-proxy/storeapi/` | `storeapi.kobo.com` | Forwards general API requests to Kobo's servers. | -| `/instapaper-proxy/storeapi/v1/initialization` | `storeapi.kobo.com` | Intercepts the Kobo configuration response to rewrite the Instapaper URL back to your proxy endpoint. | - - -Without these rules, your Kobo will eventually lose its connection to `readeckobo`. +The app is available locally at `http://localhost:8080`. Set up your chosen +[Cloudflare Tunnel](docs/CLOUDFLARE.md) or [Nginx](docs/NGINX.md) deployment +before or after starting the app; the public HTTPS address must work before +Kobo syncs. If you later edit `config.yaml`, recreate the running Docker +container so it reads the new file; the deployment guides show the command. ## 🔒 A Quick Word on Security A little security goes a long way. -* **Use HTTPS:** deploy behind a reverse proxy that provides HTTPS -* **Stay Local:** Keep it on your local private network -* **Kobo Password:** prevent unauthorized mounting with a Kobo password +* **Use HTTPS:** follow either deployment guide above, and keep the app's HTTP + port private. +* **Protect tokens:** don't share config files or request URLs containing device + tokens. Browser login or bot challenges may prevent Kobo sync. +* **Kobo Password:** prevent unauthorized mounting with a Kobo password. ## 🧑‍💻 For Developers diff --git a/config.yaml.example b/config.yaml.example index c5fc544..62cc12a 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -2,7 +2,13 @@ server: port: 8080 log_level: info readeck: - host: "https://your-readeck-instance.com" + host: "https://readeck.example.com" +# public_url: "https://readeckobo.example.com" # optional: use if proxy headers cannot supply a trusted public origin users: - token: "a-very-secret-token-for-your-kobo" readeck_access_token: "your-plain-text-readeck-access-token" + # Optional: overrides the Kobo Store fallback for this device. + # book_sync_url: "https://books.example.com/api/kobo/device-token" + # Optional: send archived bookmarks with Kobo status "2" during incremental sync. + # Default false keeps status "1"; neither setting deletes Readeck bookmarks. + # remove_archived_from_kobo: false diff --git a/docs/CLOUDFLARE.md b/docs/CLOUDFLARE.md new file mode 100644 index 0000000..2f9ecfd --- /dev/null +++ b/docs/CLOUDFLARE.md @@ -0,0 +1,41 @@ +# Cloudflare Tunnel + +Use the [README quick start](../README.md) for `config.yaml` and the Kobo's Instapaper settings. This guide publishes that same setup over HTTPS without Nginx. + +## Publish the app + +For a simple Tunnel route, set `public_url` to your public HTTPS origin in +`config.yaml`. This pins the rewritten Instapaper URL without relying on a +forwarded host header the Tunnel may not overwrite: + +```yaml +public_url: "https://readeckobo.example.com" +``` + +If you are changing an existing deployment, recreate the Docker service to +reload the bind-mounted config file (`docker-compose up -d --force-recreate +readeckobo`), or restart your non-Docker process. + +Before making the hostname public, add a [Cloudflare WAF custom rule](https://developers.cloudflare.com/waf/custom-rules/) with **Block** action for `(http.host eq "readeckobo.example.com" and http.request.uri.path eq "/api/convert-image")`. This direct, unauthenticated endpoint fetches a caller-provided URL and is not needed by the prefixed Kobo article routes. The Tunnel can still forward all other paths without path-specific routing rules. + +Add a [published application route](https://developers.cloudflare.com/cloudflare-one/networks/routes/add-routes/) to a connected Cloudflare Tunnel: + +| Route setting | Example | +| :--- | :--- | +| Public hostname | `readeckobo.example.com` | +| Service URL | `http://localhost:8080` | + +Use the actual address `cloudflared` can reach: `localhost` works when it runs on the same host as the published app port, not when it runs in a separate container. Forward **all paths** to the app without a path rule. Go proxies Kobo Store traffic and rewrites initialization so Kobo continues sending Instapaper requests to `readeckobo`. + +## Verify and secure + +Check that initialization points back to your hostname: + +```sh +curl --compressed -fsS https://readeckobo.example.com/instapaper-proxy/storeapi/v1/initialization \ + | grep -F 'https://readeckobo.example.com/instapaper-proxy/instapaper' +``` + +Then check sync on a real Kobo. Browser login or bot challenges at the edge may block its requests. `docker-compose.yml` publishes port 8080 on all interfaces: restrict direct HTTP access with a firewall, or bind the port to `127.0.0.1:8080:8080` if `cloudflared` runs on the host. If it runs in another container or host, use a private network instead. Keep device tokens and request URLs containing them private. + +For Kobo ebook syncing with a compatible service, see [Kobo ebook sync](KOBO_SYNC.md). The same Tunnel route forwards those requests to readeckobo. diff --git a/docs/CONFIG.md b/docs/CONFIG.md index a12e47b..711e841 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1,6 +1,7 @@ # Configuration Reference -This document provides a detailed reference for all available configuration options in `config.yaml`. +Configuration options for `config.yaml`. For a starting file, see +[config.yaml.example](../config.yaml.example) or the [quick start](../README.md#3-configure-readeckobo-and-kobo). ## Top-Level Options @@ -10,6 +11,9 @@ This document provides a detailed reference for all available configuration opti | `log_level` | Output verbosity. Options: `debug`, `info`, `warn`, `error`. | String | `info` | No | | `readeck.host` | The full base URL of your Readeck instance. | URL String | - | **Yes** | | `users` | A list of Kobo-to-Readeck user mappings. | List of Objects | - | **Yes** | +| `public_url` | Public origin used for the rewritten Instapaper URL in Kobo initialization. | URL String | Request host/forwarding headers | No | + +Set `public_url` when your ingress cannot reliably supply the public HTTPS origin, cannot overwrite client-supplied forwarding headers, or you need one canonical hostname. Use the origin only (no path, query, or credentials). Otherwise, Go-proxied initialization uses `X-Forwarded-Host` (or the request `Host`) and `X-Forwarded-Proto` (or the connection scheme), **not** `readeck.host`. Keep the app behind a trusted ingress when relying on those headers. See [Cloudflare Tunnel](CLOUDFLARE.md) and [Nginx](NGINX.md). ## User Object @@ -19,6 +23,8 @@ Each entry in the `users` list maps a Kobo device identity to a Readeck account. | :--- | :--- | | `token` | A unique UUID used to identify the Kobo device. This is the **Plain Text** token generated by the `bin/generate-encrypted-token.sh` script. | | `readeck_access_token` | The API token for the Readeck account you want to sync. | +| `book_sync_url` | Optional per-user Kobo API endpoint for an ebook service such as Komga. When used through `/booksync//`, it must generate cover and download URLs the Kobo can reach directly; readeckobo does not rewrite them. See [Kobo ebook sync](KOBO_SYNC.md). | +| `remove_archived_from_kobo` | Optional, default `false`. In incremental sync, send archived items with status `"2"` (requesting removal) instead of `"1"`. Full sync excludes them; neither setting deletes Readeck data. Verify on a real Kobo. | ### How to get a Readeck API Token @@ -27,17 +33,4 @@ Each entry in the `users` list maps a Kobo device identity to a Readeck account. 3. Create a new token with at least **Bookmarks: Read** permissions (Write permissions are needed for archiving/favoriting). 4. Copy the token string into your `config.yaml`. -## Example `config.yaml` - -```yaml -server: - port: 8080 -log_level: debug - -readeck: - host: "https://my-readeck.example.com" - -users: - - token: "550e8400-e29b-41d4-a716-446655440000" # From generate-encrypted-token.sh - readeck_access_token: "rdk_..." # From Readeck Profile -``` +For Kobo-compatible ebook services, see [Kobo ebook sync](KOBO_SYNC.md). diff --git a/docs/KOBO_SYNC.md b/docs/KOBO_SYNC.md new file mode 100644 index 0000000..d5ab52b --- /dev/null +++ b/docs/KOBO_SYNC.md @@ -0,0 +1,79 @@ +# Kobo ebook sync alongside Readeck articles + +Use a per-device readeckobo API endpoint to sync Readeck articles alongside +books from a Kobo-compatible service. Readeckobo forwards Kobo API requests to +the selected user's ebook service and points the initialization response's +Instapaper URL at readeckobo. Book-service URLs in initialization and later +responses pass through unchanged. Covers and ebook downloads use the absolute +URLs generated by the ebook service, so the Kobo contacts that service directly. + +[Komga's Kobo sync](https://komga.org/docs/guides/kobo/) is one example. + +## Configure the ebook service + +For each Kobo device, set `book_sync_url` to that user's full **Kobo API +endpoint** on the ebook service, including any authentication path. For +example, with Komga: + +```yaml +users: + - token: "" + readeck_access_token: "" + book_sync_url: "https://books.example.com/kobo/" +``` + +Set `public_url` if your ingress cannot reliably provide readeckobo's +public HTTPS origin; see [Cloudflare Tunnel](CLOUDFLARE.md) and +[Nginx](NGINX.md). + +Use an ebook-service hostname that both readeckobo and the Kobo can reach. Its +initialization response should contain the literal `https://www.instapaper.com` +in its Instapaper URL or already contain the desired readeckobo URL. Go replaces +that literal in a successful initialization response with +`https://reader.example.com/instapaper-proxy/instapaper`. The service must generate +Kobo-reachable cover and download URLs because readeckobo leaves them unchanged. +Publish the ebook service separately from readeckobo if needed. + +Reload readeckobo after changing `config.yaml` (for Docker Compose: +`docker-compose up -d --force-recreate readeckobo`). + +## Publish readeckobo + +[Cloudflare Tunnel](CLOUDFLARE.md) forwards `/booksync/` to Go with its normal +app route. With [Nginx](NGINX.md#kobo-ebook-syncing), add the `/booksync/` +location to readeckobo's public HTTPS server. The plain device token in the +route selects the matching user's `book_sync_url`; readeckobo forwards the +remaining path and query to that service. The same public hostname also serves +article requests. + +Check the initialization response before changing the Kobo: + +```text +https://reader.example.com/booksync//v1/initialization +``` + +Its `instapaper_env_url` should point to readeckobo. For a service returning +`https://www.instapaper.com/api/kobo`, the resulting URL is +`https://reader.example.com/instapaper-proxy/instapaper/api/kobo`. Its +`image_host` and image templates should contain URLs reachable by the Kobo, +whether served by the ebook service or a CDN. Treat the request URL and the service API key as secrets +when sharing logs or screenshots. + +## Configure the Kobo + +In `.kobo/Kobo/Kobo eReader.conf`, set the per-device API endpoint: + +```ini +[OneStoreServices] +api_endpoint=https://reader.example.com/booksync/ +``` + +Keep the Kobo's existing `instapaper_env_url` and encrypted Instapaper +`AccessToken`. The token in `api_endpoint` is readeckobo's plain device token; +`book_sync_url` holds the ebook service's endpoint and its own credential. +Configure the ebook service to handle any Kobo Store requests you need; for +example, Komga offers Kobo Store proxying in its Server Settings. + +On a real Kobo, verify initialization, library sync, covers, ebook downloads, +and Readeck articles. Check the generated service URLs and forwarding headers +for your deployment. To roll back, restore the Kobo's previous `api_endpoint`. diff --git a/docs/NGINX.md b/docs/NGINX.md new file mode 100644 index 0000000..715a79f --- /dev/null +++ b/docs/NGINX.md @@ -0,0 +1,81 @@ +# Nginx + +Use the [README quick start](../README.md) for `config.yaml` and the Kobo's +Instapaper settings. + +## Kobo article syncing + +Add these locations to the HTTPS `server` block for **readeckobo's public +hostname** (the Instapaper article address, and the `api_endpoint` address when +using ebook sync). +The example assumes Nginx can reach `readeckobo` at `127.0.0.1:8080`; use your +app's address if different. + +```nginx +# The bare Kobo Store endpoint. +location = /instapaper-proxy/storeapi { + proxy_ssl_server_name on; + proxy_ssl_name storeapi.kobo.com; + proxy_set_header Host storeapi.kobo.com; + proxy_pass https://storeapi.kobo.com/; +} + +# Forward Store subpaths with a single leading slash upstream. +location /instapaper-proxy/storeapi/ { + proxy_ssl_server_name on; + proxy_ssl_name storeapi.kobo.com; + proxy_set_header Host storeapi.kobo.com; + proxy_pass https://storeapi.kobo.com/; +} + +# Fetch Kobo Store initialization for the Instapaper URL rewrite. +location = /instapaper-proxy/storeapi/v1/initialization { + proxy_ssl_server_name on; + proxy_ssl_name storeapi.kobo.com; + proxy_set_header Host storeapi.kobo.com; + proxy_set_header Accept-Encoding ""; + proxy_set_header If-None-Match ""; + proxy_set_header If-Modified-Since ""; + proxy_set_header Range ""; + proxy_set_header If-Range ""; + proxy_pass https://storeapi.kobo.com/v1/initialization; + + sub_filter https://www.instapaper.com "$scheme://$host/instapaper-proxy/instapaper"; + sub_filter_types application/json; +} + +# Instapaper article requests go to readeckobo. +location /instapaper-proxy/instapaper/ { + proxy_pass http://127.0.0.1:8080/; + proxy_set_header Host $host; + client_max_body_size 128M; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Prefix /instapaper-proxy/instapaper; +} +``` + +Keep port 8080 private: `docker-compose.yml` exposes it on all interfaces by +default. Bind it to `127.0.0.1:8080:8080` if Nginx runs on the host, or use a +private network if it runs elsewhere. + +## Kobo ebook syncing + +For [Kobo ebook sync](KOBO_SYNC.md), add this location to readeckobo's public +HTTPS `server` block. The `proxy_pass` forwards the full +`/booksync//...` path to Go, which selects the user's ebook +service. The ebook service publishes its own Kobo-reachable cover and download +URLs: + +```nginx +location /booksync/ { + access_log off; # Request paths contain a device token. + proxy_pass http://127.0.0.1:8080; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; +} +``` diff --git a/internal/app/app.go b/internal/app/app.go index 988b494..4a47857 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "image" "image/draw" @@ -11,8 +12,8 @@ import ( "image/jpeg" _ "image/png" "io" + "mime" "net/http" - "net/http/httputil" "net/url" "strconv" "strings" @@ -33,6 +34,7 @@ type App struct { Logger *logger.Logger ImageHTTPClient *http.Client ReadeckHTTPClient *http.Client + ProxyTransport http.RoundTripper } func WithImageHTTPClient(client *http.Client) Option { @@ -69,6 +71,12 @@ func WithReadeckHTTPClient(client *http.Client) Option { } } +func WithProxyTransport(transport http.RoundTripper) Option { + return func(a *App) { + a.ProxyTransport = transport + } +} + func (a *App) handleFullSync(ctx context.Context, readeckClient *readeck.Client, req *models.KoboGetRequest) (map[string]models.KoboArticleItem, int, error) { count, _ := strconv.Atoi(req.Count) offset, _ := strconv.Atoi(req.Offset) @@ -136,7 +144,7 @@ func (a *App) handleFullSync(ctx context.Context, readeckClient *readeck.Client, return resultList, totalNonArchivedBookmarks, nil } -func (a *App) handleIncrementalSync(ctx context.Context, readeckClient *readeck.Client, since *time.Time) (map[string]models.KoboArticleItem, int, error) { +func (a *App) handleIncrementalSync(ctx context.Context, readeckClient *readeck.Client, since *time.Time, user config.User) (map[string]models.KoboArticleItem, int, error) { resultList := make(map[string]models.KoboArticleItem) bsyncs, err := readeckClient.GetBookmarksSync(ctx, since) @@ -186,6 +194,9 @@ func (a *App) handleIncrementalSync(ctx context.Context, readeckClient *readeck. if bookmark.IsArchived { entry.Status = "1" + if user.RemoveArchivedFromKobo { + entry.Status = "2" + } } else { entry.Status = "0" totalNonArchivedBookmarks++ @@ -200,12 +211,12 @@ func (a *App) HandleKoboGet(w http.ResponseWriter, r *http.Request) { bodyBytes, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read request body", http.StatusInternalServerError) - a.Logger.Errorf("Error reading /api/kobo/get request body: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error reading /api/kobo/get request body: %v", err) return } r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) - a.Logger.Debugf("Incoming Kobo Request for /api/kobo/get:\nMethod: %s\nURL: %s\nHeaders: %v\nBody: %s", r.Method, r.URL, r.Header, string(bodyBytes)) + a.Logger.Debugf("Incoming Kobo Request for /api/kobo/get: Method=%s", r.Method) if r.Method != http.MethodPost { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) @@ -215,21 +226,21 @@ func (a *App) HandleKoboGet(w http.ResponseWriter, r *http.Request) { var req models.KoboGetRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { http.Error(w, "Invalid request body", http.StatusBadRequest) - a.Logger.Errorf("Error decoding /api/kobo/get request: %v, body: %s, URL: %s, Params: %v", err, string(bodyBytes), r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error decoding /api/kobo/get request: %v", err) return } - readeckToken, err := a.getReadeckToken(req.AccessToken) + user, err := a.getUser(req.AccessToken) if err != nil { http.Error(w, "Invalid access token", http.StatusUnauthorized) - a.Logger.Errorf("Error authenticating token for /api/kobo/get: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error authenticating token for /api/kobo/get: %v", err) return } - readeckClient, err := a.newReadeckClient(readeckToken) + readeckClient, err := a.newReadeckClient(user.ReadeckAccessToken) if err != nil { http.Error(w, "Failed to initialize Readeck client", http.StatusInternalServerError) - a.Logger.Errorf("Error initializing Readeck client for /api/kobo/get: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error initializing Readeck client for /api/kobo/get: %v", err) return } @@ -252,7 +263,7 @@ func (a *App) HandleKoboGet(w http.ResponseWriter, r *http.Request) { resultList, total, err = a.handleFullSync(r.Context(), readeckClient, &req) } else { a.Logger.Debugf("Handling incremental sync.") - resultList, total, err = a.handleIncrementalSync(r.Context(), readeckClient, since) + resultList, total, err = a.handleIncrementalSync(r.Context(), readeckClient, since, user) } if err != nil { @@ -325,12 +336,12 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) { bodyBytes, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read request body", http.StatusInternalServerError) - a.Logger.Errorf("Error reading /api/kobo/download request body: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error reading /api/kobo/download request body: %v", err) return } r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) - a.Logger.Debugf("Incoming Kobo Request for /api/kobo/download:\nMethod: %s\nURL: %s\nHeaders: %v\nBody: %s", r.Method, r.URL, r.Header, string(bodyBytes)) + a.Logger.Debugf("Incoming Kobo Request for /api/kobo/download: Method=%s", r.Method) if r.Method != http.MethodPost { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) @@ -339,44 +350,45 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) { var req models.KoboDownloadRequest if err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&req); err != nil { - if err := r.ParseForm(); err != nil { + form, err := downloadFormValues(r, bodyBytes) + if err != nil { http.Error(w, "Invalid request body or form data", http.StatusBadRequest) - a.Logger.Errorf("Error decoding /api/kobo/download request: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error decoding /api/kobo/download request: %v", err) return } - req.AccessToken = r.FormValue("access_token") - req.ConsumerKey = r.FormValue("consumer_key") - req.Images, _ = strconv.Atoi(r.FormValue("images")) - req.Refresh, _ = strconv.Atoi(r.FormValue("refresh")) - req.Output = r.FormValue("output") - req.URL = r.FormValue("url") + req.AccessToken = form.Get("access_token") + req.ConsumerKey = form.Get("consumer_key") + req.Images, _ = strconv.Atoi(form.Get("images")) + req.Refresh, _ = strconv.Atoi(form.Get("refresh")) + req.Output = form.Get("output") + req.URL = form.Get("url") } readeckToken, err := a.getReadeckToken(req.AccessToken) if err != nil { http.Error(w, "Invalid access token", http.StatusUnauthorized) - a.Logger.Errorf("Error authenticating token for /api/kobo/download: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error authenticating token for /api/kobo/download: %v", err) return } readeckClient, err := a.newReadeckClient(readeckToken) if err != nil { http.Error(w, "Failed to initialize Readeck client", http.StatusInternalServerError) - a.Logger.Errorf("Error initializing Readeck client for /api/kobo/download: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error initializing Readeck client for /api/kobo/download: %v", err) return } reqURLStr := req.URL if reqURLStr == "" { http.Error(w, "Missing 'url' parameter", http.StatusBadRequest) - a.Logger.Errorf("Error: Missing 'url' parameter in /api/kobo/download request, URL: %s, Params: %v", r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Missing 'url' parameter in /api/kobo/download request") return } parsedURL, err := url.Parse(reqURLStr) if err != nil { http.Error(w, "Invalid 'url' parameter", http.StatusBadRequest) - a.Logger.Errorf("Error: Invalid 'url' parameter in /api/kobo/download request: %v, url: %s, URL: %s, Params: %v", err, reqURLStr, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Invalid 'url' parameter in /api/kobo/download request: %T", err) return } @@ -392,7 +404,7 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) { isArchived := false bookmarks, tp, err := readeckClient.GetBookmarks(ctx, site, currentPage, &isArchived) if err != nil { - a.Logger.Warnf("Error searching Readeck bookmarks for site %s, page %d in /api/kobo/download: %v, URL: %s, Params: %v", site, currentPage, err, r.URL.Path, r.URL.Query()) + a.Logger.Warnf("Error searching Readeck bookmarks for site %s, page %d in /api/kobo/download: %v", site, currentPage, err) break } totalPages = tp @@ -401,7 +413,7 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) { if bookmarks[i].URL != "" { match, err := compareURLs(bookmarks[i].URL, reqURLStr) if err != nil { - a.Logger.Warnf("Error comparing URLs for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarks[i].ID, err, r.URL.Path, r.URL.Query()) + a.Logger.Warnf("Error comparing URLs for bookmark %s in /api/kobo/download: %T", bookmarks[i].ID, err) continue } if match { @@ -428,14 +440,14 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) { articleHTML, err := readeckClient.GetBookmarkArticle(ctx, bookmarkFound.ID) if err != nil { http.Error(w, "Failed to fetch article content", http.StatusInternalServerError) - a.Logger.Errorf("Error fetching article content for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarkFound.ID, err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error fetching article content for bookmark %s in /api/kobo/download: %v", bookmarkFound.ID, err) return } doc, err := html.Parse(strings.NewReader(articleHTML)) if err != nil { http.Error(w, "Failed to parse article HTML", http.StatusInternalServerError) - a.Logger.Errorf("Error parsing article HTML for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarkFound.ID, err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error parsing article HTML for bookmark %s in /api/kobo/download: %v", bookmarkFound.ID, err) return } @@ -474,7 +486,7 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) { var buf bytes.Buffer if err := html.Render(&buf, doc); err != nil { http.Error(w, "Failed to render modified HTML", http.StatusInternalServerError) - a.Logger.Errorf("Error rendering modified HTML for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarkFound.ID, err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error rendering modified HTML for bookmark %s in /api/kobo/download: %v", bookmarkFound.ID, err) return } @@ -485,10 +497,47 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(response); err != nil { - a.Logger.Errorf("Error encoding response for /api/kobo/download: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error encoding response for /api/kobo/download: %v", err) } } +func downloadFormValues(r *http.Request, body []byte) (url.Values, error) { + if err := r.ParseForm(); err != nil { + mediaType, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")) + if mediaType != "application/x-www-form-urlencoded" || len(body) > 10<<20 { + return nil, err + } + + // Kobo sends a raw ; inside the URL value. Escape only that form field, + // then let Go validate and decode the complete form as usual. + fields := strings.Split(string(body), "&") + urlFields := 0 + repaired := false + for i, field := range fields { + if !strings.HasPrefix(field, "url=") { + continue + } + urlFields++ + if strings.Contains(field, ";") { + fields[i] = strings.ReplaceAll(field, ";", "%3B") + repaired = true + } + } + if urlFields != 1 || !repaired { + return nil, err + } + form, err := url.ParseQuery(strings.Join(fields, "&")) + if err != nil { + return nil, err + } + if len(form["url"]) != 1 { + return nil, errors.New("duplicate URL form field") + } + return form, nil + } + return r.Form, nil +} + func getSitesToTry(host string) []string { var sites []string parts := strings.Split(host, ".") @@ -518,12 +567,12 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) { bodyBytes, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read request body", http.StatusInternalServerError) - a.Logger.Errorf("Error reading /api/kobo/send request body: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error reading /api/kobo/send request body: %v", err) return } r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) - a.Logger.Debugf("Incoming Kobo Request for /api/kobo/send:\nMethod: %s\nURL: %s\nHeaders: %v\nBody: %s", r.Method, r.URL, r.Header, string(bodyBytes)) + a.Logger.Debugf("Incoming Kobo Request for /api/kobo/send: Method=%s", r.Method) if r.Method != http.MethodPost { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) @@ -533,21 +582,21 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) { var req models.KoboSendRequest if err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&req); err != nil { http.Error(w, "Invalid request body", http.StatusBadRequest) - a.Logger.Errorf("Error decoding /api/kobo/send request: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error decoding /api/kobo/send request: %v", err) return } readeckToken, err := a.getReadeckToken(req.AccessToken) if err != nil { http.Error(w, "Invalid access token", http.StatusUnauthorized) - a.Logger.Errorf("Error authenticating token for /api/kobo/send: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error authenticating token for /api/kobo/send: %v", err) return } readeckClient, err := a.newReadeckClient(readeckToken) if err != nil { http.Error(w, "Failed to initialize Readeck client", http.StatusInternalServerError) - a.Logger.Errorf("Error initializing Readeck client for /api/kobo/send: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error initializing Readeck client for /api/kobo/send: %v", err) return } @@ -592,7 +641,7 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) { } if err != nil { - a.Logger.Warnf("Error processing action '%s' in /api/kobo/send: %v, URL: %s, Params: %v", action, err, r.URL.Path, r.URL.Query()) + a.Logger.Warnf("Error processing action '%s' in /api/kobo/send: %v", action, err) actionResults[i] = false allSucceeded = false } else { @@ -607,7 +656,7 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(response); err != nil { - a.Logger.Errorf("Error encoding response for /api/kobo/send: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error encoding response for /api/kobo/send: %v", err) } } @@ -629,25 +678,25 @@ func (a *App) HandleConvertImage(w http.ResponseWriter, r *http.Request) { } resp, err := client.Get(imageURL) if err != nil { - a.Logger.Errorf("Failed to fetch image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Failed to fetch image in /api/convert-image: %T", err) a.returnPlaceholderImage(w, r, "Image fetch failed") return } defer func() { if err := resp.Body.Close(); err != nil { - a.Logger.Warnf("Error closing response body for image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query()) + a.Logger.Warnf("Error closing response body in /api/convert-image: %v", err) } }() if resp.StatusCode != http.StatusOK { - a.Logger.Warnf("Failed to fetch image %s in /api/convert-image: status %d, URL: %s, Params: %v", imageURL, resp.StatusCode, r.URL.Path, r.URL.Query()) + a.Logger.Warnf("Failed to fetch image in /api/convert-image: status %d", resp.StatusCode) a.returnPlaceholderImage(w, r, "Image not found") return } img, _, err := image.Decode(resp.Body) if err != nil { - a.Logger.Warnf("Failed to decode image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query()) + a.Logger.Warnf("Failed to decode image in /api/convert-image: %v", err) a.returnPlaceholderImage(w, r, "Image decoding failed") return } @@ -659,7 +708,7 @@ func (a *App) HandleConvertImage(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "image/jpeg") w.Header().Set("Cache-Control", "public, max-age=3600") if err := jpeg.Encode(w, rgbImg, &jpeg.Options{Quality: 85}); err != nil { - a.Logger.Errorf("Failed to encode JPEG for image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Failed to encode JPEG in /api/convert-image: %v", err) } } @@ -681,7 +730,7 @@ func (a *App) returnPlaceholderImage(w http.ResponseWriter, r *http.Request, mes w.Header().Set("Content-Type", "image/jpeg") w.Header().Set("Cache-control", "public, max-age=300") if err := jpeg.Encode(w, img, &jpeg.Options{Quality: 85}); err != nil { - a.Logger.Errorf("Error encoding placeholder image: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query()) + a.Logger.Errorf("Error encoding placeholder image: %v", err) } } @@ -701,41 +750,23 @@ func compareURLs(url1, url2 string) (bool, error) { return u1.Scheme == u2.Scheme && u1.Host == u2.Host && u1.Path == u2.Path, nil } -func (a *App) getReadeckToken(deviceToken string) (string, error) { +func (a *App) getUser(deviceToken string) (config.User, error) { for _, user := range a.Config.Users { if user.Token == deviceToken { - return user.ReadeckAccessToken, nil + return user, nil } } - return "", fmt.Errorf("unauthorized device token") + return config.User{}, fmt.Errorf("unauthorized device token") +} + +func (a *App) getReadeckToken(deviceToken string) (string, error) { + user, err := a.getUser(deviceToken) + if err != nil { + return "", err + } + return user.ReadeckAccessToken, nil } func (a *App) newReadeckClient(readeckToken string) (*readeck.Client, error) { return readeck.NewClient(a.Config.Readeck.Host, readeckToken, a.Logger, a.ReadeckHTTPClient) } - -func (a *App) HandleDumpAndForward(w http.ResponseWriter, r *http.Request) { - a.Logger.Debugf("Dumping request from %s", r.RemoteAddr) - a.Logger.Debugf("Method: %s", r.Method) - a.Logger.Debugf("URL: %s", r.URL.String()) - a.Logger.Debugf("Headers: %v", r.Header) - - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "Failed to read request body", http.StatusInternalServerError) - a.Logger.Debugf("Error reading request body: %v", err) - return - } - r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) - - a.Logger.Debugf("Body: %s", string(bodyBytes)) - - target, err := url.Parse("https://storeapi.kobo.com") - if err != nil { - a.Logger.Errorf("Error parsing target URL: %v", err) - return - } - proxy := httputil.NewSingleHostReverseProxy(target) - proxy.ServeHTTP(w, r) -} - diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 4465e34..f1ccf51 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -173,6 +173,7 @@ func TestHandleKoboGet(t *testing.T) { expectedStatus int expectedListSize int expectedTotal int + expectedStatuses map[string]string } testCases := []koboGetTestCase{ @@ -191,6 +192,22 @@ func TestHandleKoboGet(t *testing.T) { expectedListSize: 1, // Only the unread item expectedTotal: 1, }, + { + name: "full sync with opted-in user still excludes archived", + reqBody: &models.KoboGetRequest{Count: "10", AccessToken: "remove-archived-device"}, + mockBookmarksSync: []readeck.BookmarkSync{ + {ID: "1", Type: "update"}, + {ID: "2", Type: "update"}, + }, + mockBookmarkDetails: map[string]*readeck.Bookmark{ + "1": {ID: "1", Title: "Unread"}, + "2": {ID: "2", Title: "Archived", IsArchived: true}, + }, + expectedStatus: http.StatusOK, + expectedListSize: 1, + expectedTotal: 1, + expectedStatuses: map[string]string{"1": "0"}, + }, { name: "full sync with favorited item", reqBody: &models.KoboGetRequest{Count: "10", AccessToken: mockDeviceToken}, // No 'Since' @@ -236,6 +253,7 @@ func TestHandleKoboGet(t *testing.T) { expectedStatus: http.StatusOK, expectedListSize: 1, // The deleted status update expectedTotal: 0, + expectedStatuses: map[string]string{"1": "2"}, }, { name: "incremental sync with newly archived", @@ -249,6 +267,24 @@ func TestHandleKoboGet(t *testing.T) { expectedStatus: http.StatusOK, expectedListSize: 1, // The full archived item expectedTotal: 0, + expectedStatuses: map[string]string{"1": "1"}, + }, + { + name: "incremental sync removes archived for opted-in user", + reqBody: &models.KoboGetRequest{Since: sinceValue, AccessToken: "remove-archived-device"}, + mockBookmarksSync: []readeck.BookmarkSync{ + {ID: "1", Type: "update"}, + {ID: "2", Type: "update"}, + {ID: "3", Type: "delete"}, + }, + mockBookmarkDetails: map[string]*readeck.Bookmark{ + "1": {ID: "1", Title: "Archived", IsArchived: true}, + "2": {ID: "2", Title: "Unread", IsArchived: false}, + }, + expectedStatus: http.StatusOK, + expectedListSize: 3, + expectedTotal: 1, + expectedStatuses: map[string]string{"1": "2", "2": "0", "3": "2"}, }, { name: "incremental sync with GetBookmarksSync error", @@ -327,7 +363,10 @@ func TestHandleKoboGet(t *testing.T) { app := NewApp( WithConfig(&config.Config{ - Users: []config.User{{Token: mockDeviceToken, ReadeckAccessToken: mockPlaintextReadeckToken}}, + Users: []config.User{ + {Token: mockDeviceToken, ReadeckAccessToken: mockPlaintextReadeckToken}, + {Token: "remove-archived-device", ReadeckAccessToken: mockPlaintextReadeckToken, RemoveArchivedFromKobo: true}, + }, Readeck: config.ConfigReadeck{Host: mockServer.URL}, }), WithLogger(testLogger), @@ -359,6 +398,13 @@ func TestHandleKoboGet(t *testing.T) { t.Errorf("expected total to be %d, got %d", tc.expectedTotal, resp.Total) } + for id, wantStatus := range tc.expectedStatuses { + item, ok := resp.List[id] + if !ok || item.Status != wantStatus { + t.Errorf("item %s status = %q (found: %v), want %q", id, item.Status, ok, wantStatus) + } + } + // Specific checks for each test case switch tc.name { case "full sync with favorited item": @@ -366,16 +412,6 @@ func TestHandleKoboGet(t *testing.T) { if item.Favorite != "1" { t.Errorf("expected favorited item 'favorite' status to be '1', got '%s'", item.Favorite) } - case "incremental sync with deleted": - item := resp.List["1"] - if item.Status != "2" { - t.Errorf("expected deleted item status to be '2', got '%s'", item.Status) - } - case "incremental sync with newly archived": - item := resp.List["1"] - if item.Status != "1" { - t.Errorf("expected archived item status to be '1', got '%s'", item.Status) - } case "full sync with image item": item := resp.List["1"] if item.HasImage != "1" { @@ -389,6 +425,7 @@ func TestHandleKoboGet(t *testing.T) { }) } } + // koboDownloadTestCase defines the structure for test cases in TestHandleKoboDownload. type koboDownloadTestCase struct { name string @@ -414,6 +451,19 @@ func TestHandleKoboDownload(t *testing.T) { }, mockArticle: `

Test Article

`, }, + { + name: "successful download (JSON URL with semicolon)", + reqBody: models.KoboDownloadRequest{ + AccessToken: mockDeviceToken, + URL: "http://example.com/article;edition=2?page=1", + }, + contentType: "application/json", + expectedStatus: http.StatusOK, + mockBookmarks: []readeck.Bookmark{ + {ID: "1", Title: "Test Article", URL: "http://example.com/article;edition=2?page=1"}, + }, + mockArticle: `

Test Article

`, + }, { name: "successful download (Form)", reqBody: url.Values{ @@ -427,6 +477,49 @@ func TestHandleKoboDownload(t *testing.T) { }, mockArticle: `

Test Article

`, }, + { + name: "successful download (encoded semicolon in form URL path)", + reqBody: url.Values{ + "access_token": {mockDeviceToken}, + "url": {"http://example.com/article;edition=2?page=1"}, + }, + contentType: "application/x-www-form-urlencoded", + expectedStatus: http.StatusOK, + mockBookmarks: []readeck.Bookmark{ + {ID: "1", Title: "Test Article", URL: "http://example.com/article;edition=2?page=1"}, + }, + mockArticle: `

Test Article

`, + }, + { + name: "successful download (raw semicolon in form URL path)", + reqBody: "access_token=" + mockDeviceToken + "&url=http%3A%2F%2Fexample.com%2Farticle;edition=2%3Fpage%3D1", + contentType: "application/x-www-form-urlencoded", + expectedStatus: http.StatusOK, + mockBookmarks: []readeck.Bookmark{ + {ID: "1", Title: "Test Article", URL: "http://example.com/article;edition=2?page=1"}, + }, + mockArticle: `

Test Article

`, + }, + { + name: "successful download (raw semicolon in form URL query)", + reqBody: "access_token=" + mockDeviceToken + "&url=http%3A%2F%2Fexample.com%2Farticle1%3Futm_source%3Dmail;utm_medium%3Dreader", + contentType: "application/x-www-form-urlencoded", + expectedStatus: http.StatusOK, + mockBookmarks: []readeck.Bookmark{ + {ID: "1", Title: "Test Article", URL: "http://example.com/article1?utm_source=mail;utm_medium=reader"}, + }, + mockArticle: `

Test Article

`, + }, + { + name: "successful download (captured Kobo form shape)", + reqBody: "images=0&consumer_key=fake&access_token=" + mockDeviceToken + "&refresh=0&output=json&url=https://example.com/article?kobo_probe%3D1;part%3D2", + contentType: "application/x-www-form-urlencoded", + expectedStatus: http.StatusOK, + mockBookmarks: []readeck.Bookmark{ + {ID: "1", Title: "Test Article", URL: "https://example.com/article?kobo_probe=1;part=2"}, + }, + mockArticle: `

Test Article

`, + }, { name: "missing url", reqBody: models.KoboDownloadRequest{ @@ -436,6 +529,12 @@ func TestHandleKoboDownload(t *testing.T) { contentType: "application/json", expectedStatus: http.StatusBadRequest, }, + { + name: "missing access token in raw-semicolon form", + reqBody: "url=https://example.com/article;kobo_probe%3D1", + contentType: "application/x-www-form-urlencoded", + expectedStatus: http.StatusUnauthorized, + }, { name: "invalid access token", reqBody: models.KoboDownloadRequest{ @@ -490,8 +589,14 @@ func TestHandleKoboDownload(t *testing.T) { } body = bytes.NewReader(jsonBody) case "application/x-www-form-urlencoded": - formValues := tc.reqBody.(url.Values) - body = strings.NewReader(formValues.Encode()) + switch form := tc.reqBody.(type) { + case url.Values: + body = strings.NewReader(form.Encode()) + case string: + body = strings.NewReader(form) + default: + t.Fatalf("unexpected form body type %T", tc.reqBody) + } } req := httptest.NewRequest(http.MethodPost, "/api/kobo/download", body) @@ -525,6 +630,69 @@ func TestHandleKoboDownload(t *testing.T) { } } +func TestDownloadFormValuesWithSemicolons(t *testing.T) { + for _, tt := range []struct { + name, body, wantURL string + wantError bool + }{ + { + name: "normally encoded URL with semicolon", + body: url.Values{"access_token": {"device"}, "url": {"https://example.com/article;edition=2?utm=a;b=c"}}.Encode(), + wantURL: "https://example.com/article;edition=2?utm=a;b=c", + }, + { + name: "raw semicolon in URL path", + body: "access_token=device&url=https%3A%2F%2Fexample.com%2Farticle;edition=2%3Fpage%3D1&output=epub", + wantURL: "https://example.com/article;edition=2?page=1", + }, + { + name: "Kobo form encodes equals but leaves semicolon raw", + body: "images=0&consumer_key=fake&access_token=device&refresh=0&output=json&url=https://example.com/article?kobo_probe%3D1;part%3D2", + wantURL: "https://example.com/article?kobo_probe=1;part=2", + }, + { + name: "raw semicolon in URL query", + body: "url=https%3A%2F%2Fexample.com%2Farticle%3Futm_source%3Dmail;utm_medium%3Dreader%26page%3D2&access_token=device", + wantURL: "https://example.com/article?utm_source=mail;utm_medium=reader&page=2", + }, + { + name: "semicolon outside URL field", + body: "access_token=device;consumer_key=other&url=https%3A%2F%2Fexample.com%2Farticle;edition=2", + wantError: true, + }, + { + name: "duplicate URL field", + body: "access_token=device&url=https%3A%2F%2Fexample.com%2Fa;edition=2&url=https%3A%2F%2Fexample.com%2Fb", + wantError: true, + }, + { + name: "invalid form without raw semicolon stays invalid", + body: "access_token=device&url=https%3A%2F%2Fexample.com%2Fa&output=%zz", + wantError: true, + }, + { + name: "invalid percent escape", + body: "access_token=device&url=https%3A%2F%2Fexample.com%2Fa;edition=2&output=%zz", + wantError: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/api/kobo/download", strings.NewReader(tt.body)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + form, err := downloadFormValues(req, []byte(tt.body)) + if tt.wantError { + if err == nil { + t.Fatalf("expected a form parsing error, got %v", form) + } + return + } + if err != nil || form.Get("url") != tt.wantURL || form.Get("access_token") != "device" { + t.Errorf("URL = %q, token = %q, err = %v; want URL %q and token device", form.Get("url"), form.Get("access_token"), err, tt.wantURL) + } + }) + } +} + // koboSendTestCase defines the structure for test cases in TestHandleKoboSend. type koboSendTestCase struct { name string @@ -842,5 +1010,3 @@ func TestHandleConvertImage(t *testing.T) { } }) } - - diff --git a/internal/app/booksync.go b/internal/app/booksync.go new file mode 100644 index 0000000..389c527 --- /dev/null +++ b/internal/app/booksync.go @@ -0,0 +1,38 @@ +package app + +import ( + "net/http" + "net/url" +) + +func (a *App) HandleBookSync(w http.ResponseWriter, r *http.Request) { + token := r.PathValue("deviceToken") + upstream, known := a.bookSyncUpstream(token) + if !known { + http.NotFound(w, r) + return + } + + target, err := url.Parse(upstream) + if err != nil { + a.Logger.Errorf("Invalid configured book sync upstream: %T", err) + http.Error(w, "Bad gateway", http.StatusBadGateway) + return + } + + http.StripPrefix("/booksync/"+token, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + a.serveKoboProxy(w, r, target, token) + })).ServeHTTP(w, r) +} + +func (a *App) bookSyncUpstream(token string) (string, bool) { + for _, user := range a.Config.Users { + if user.Token == token { + if user.BookSyncURL != "" { + return user.BookSyncURL, true + } + return "https://" + storeAPIHost, true + } + } + return "", false +} diff --git a/internal/app/proxy.go b/internal/app/proxy.go new file mode 100644 index 0000000..ba21459 --- /dev/null +++ b/internal/app/proxy.go @@ -0,0 +1,183 @@ +package app + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httputil" + "net/url" + "strconv" + "strings" + "time" +) + +const ( + storeAPIHost = "storeapi.kobo.com" + maxInitializationBody = 8 << 20 +) + +var koboTransport = func() *http.Transport { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.ResponseHeaderTimeout = 20 * time.Second + return transport +}() + +func (a *App) HandleStoreProxy(w http.ResponseWriter, r *http.Request) { + a.serveKoboProxy(w, r, &url.URL{Scheme: "https", Host: storeAPIHost}, "") +} + +// serveKoboProxy expects the public route prefix to have been removed from r.URL. +func (a *App) serveKoboProxy(w http.ResponseWriter, r *http.Request, target *url.URL, deviceToken string) { + path := r.URL.Path + if path == "" { + path = "/" + } + proxy := a.newKoboProxy(target, path) + if deviceToken != "" { + originalDirector := proxy.Director + proxy.Director = func(req *http.Request) { + originalDirector(req) + // Book services can build absolute URLs from forwarded headers. + // Use the upstream origin, not the public ingress host. + req.Header.Del("Forwarded") + req.Header.Del("X-Forwarded-Port") + req.Header.Del("X-Forwarded-Prefix") + req.Header.Set("X-Forwarded-Host", target.Host) + req.Header.Set("X-Forwarded-Proto", target.Scheme) + } + } + if path == "/v1/initialization" { + origin, err := a.publicOrigin(r) + if err != nil { + http.Error(w, "Invalid public origin", http.StatusBadGateway) + return + } + configureInitializationProxy(proxy, origin+"/instapaper-proxy/instapaper") + } + proxy.ServeHTTP(w, r) +} + +func (a *App) newKoboProxy(target *url.URL, path string) *httputil.ReverseProxy { + proxy := httputil.NewSingleHostReverseProxy(target) + originalDirector := proxy.Director + proxy.Director = func(req *http.Request) { + req.URL.Path = path + originalDirector(req) + req.Host = target.Host + } + if a.ProxyTransport != nil { + proxy.Transport = a.ProxyTransport + } else { + proxy.Transport = koboTransport + } + proxy.ErrorHandler = func(w http.ResponseWriter, _ *http.Request, err error) { + a.Logger.Errorf("Kobo upstream proxy failed: %T", err) + http.Error(w, "Bad gateway", http.StatusBadGateway) + } + return proxy +} + +func configureInitializationProxy(proxy *httputil.ReverseProxy, instapaperURL string) { + originalDirector := proxy.Director + proxy.Director = func(req *http.Request) { + originalDirector(req) + // Request a complete document so its body can be rewritten. + req.Header.Set("Accept-Encoding", "identity") + req.Header.Del("If-None-Match") + req.Header.Del("If-Modified-Since") + req.Header.Del("Range") + req.Header.Del("If-Range") + } + proxy.ModifyResponse = func(resp *http.Response) error { + if resp.StatusCode == http.StatusPartialContent { + return errors.New("partial initialization response") + } + if resp.StatusCode != http.StatusOK { + return nil + } + return rewriteInitialization(resp, instapaperURL) + } +} + +func (a *App) publicOrigin(r *http.Request) (string, error) { + if a.Config.PublicURL != "" { + return strings.TrimRight(a.Config.PublicURL, "/"), nil + } + + scheme := r.Header.Get("X-Forwarded-Proto") + if scheme == "" { + scheme = "http" + if r.TLS != nil { + scheme = "https" + } + } + host := r.Header.Get("X-Forwarded-Host") + if host == "" { + host = r.Host + } + if scheme != "http" && scheme != "https" || host == "" || strings.ContainsAny(host, "/@?#, \t\r\n\"\\") { + return "", errors.New("invalid forwarded origin") + } + parsed, err := url.Parse("//" + host) + if err != nil || parsed.Hostname() == "" || parsed.Host != host { + return "", errors.New("invalid forwarded host") + } + return scheme + "://" + host, nil +} + +func rewriteInitialization(resp *http.Response, instapaperURL string) error { + originalBody := resp.Body + defer func() { _ = originalBody.Close() }() + + var reader io.Reader = originalBody + compressed := resp.Header.Get("Content-Encoding") == "gzip" + if compressed { + gz, err := gzip.NewReader(originalBody) + if err != nil { + return err + } + defer func() { _ = gz.Close() }() + reader = gz + } else if encoding := resp.Header.Get("Content-Encoding"); encoding != "" && encoding != "identity" { + return fmt.Errorf("unsupported initialization encoding: %s", encoding) + } + + body, err := io.ReadAll(io.LimitReader(reader, maxInitializationBody+1)) + if err != nil { + return err + } + if len(body) > maxInitializationBody { + return errors.New("initialization response too large") + } + encodedURL, err := json.Marshal(instapaperURL) + if err != nil { + return err + } + body = bytes.ReplaceAll(body, []byte("https://www.instapaper.com"), encodedURL[1:len(encodedURL)-1]) + if compressed { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + if _, err := gw.Write(body); err != nil { + return err + } + if err := gw.Close(); err != nil { + return err + } + body = buf.Bytes() + } else { + resp.Header.Del("Content-Encoding") + } + + resp.Body = io.NopCloser(bytes.NewReader(body)) + resp.ContentLength = int64(len(body)) + resp.Header.Set("Content-Length", strconv.Itoa(len(body))) + for _, key := range []string{"ETag", "Last-Modified", "Content-MD5", "Digest", "Content-Range", "Accept-Ranges"} { + resp.Header.Del(key) + } + resp.Header.Set("Cache-Control", "private, no-store") + return nil +} diff --git a/internal/app/proxy_test.go b/internal/app/proxy_test.go new file mode 100644 index 0000000..4129c27 --- /dev/null +++ b/internal/app/proxy_test.go @@ -0,0 +1,313 @@ +package app + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "readeckobo/internal/config" +) + +func storeHandler(a *App) http.Handler { + return http.StripPrefix("/instapaper-proxy/storeapi", http.HandlerFunc(a.HandleStoreProxy)) +} + +func TestHandleStoreProxy(t *testing.T) { + t.Run("forwards requests to the fixed store origin", func(t *testing.T) { + body := "book sync response" + a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{ + RoundTripFunc: func(req *http.Request) (*http.Response, error) { + if req.URL.Scheme != "https" || req.URL.Host != storeAPIHost || req.Host != storeAPIHost || req.URL.Path != "/v1/library/sync" || req.URL.RawQuery != "page=2" { + t.Errorf("unexpected upstream request: URL=%s Host=%q", req.URL, req.Host) + } + data, err := io.ReadAll(req.Body) + if err != nil || string(data) != "payload" { + t.Errorf("upstream body = %q, err = %v", data, err) + } + if req.Header.Get("X-Test") != "forwarded" { + t.Errorf("X-Test = %q", req.Header.Get("X-Test")) + } + return &http.Response{StatusCode: http.StatusAccepted, Header: http.Header{"Content-Type": {"application/octet-stream"}}, Body: io.NopCloser(strings.NewReader(body))}, nil + }, + })) + req := httptest.NewRequest(http.MethodPost, "/instapaper-proxy/storeapi/v1/library/sync?page=2", strings.NewReader("payload")) + req.Header.Set("X-Test", "forwarded") + rr := httptest.NewRecorder() + storeHandler(a).ServeHTTP(rr, req) + if rr.Code != http.StatusAccepted || rr.Body.String() != body { + t.Errorf("status = %d, body length = %d; want 202 and %d", rr.Code, rr.Body.Len(), len(body)) + } + }) + + t.Run("streams an ordinary response before upstream completion", func(t *testing.T) { + reader, writer := io.Pipe() + a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{ + RoundTripFunc: func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": {"application/octet-stream"}}, + Body: reader, + ContentLength: -1, + }, nil + }, + })) + server := httptest.NewServer(storeHandler(a)) + defer server.Close() + defer func() { _ = writer.Close() }() + + firstChunk := bytes.Repeat([]byte("x"), 32*1024) + result := make(chan error, 1) + go func() { + resp, err := server.Client().Get(server.URL + "/instapaper-proxy/storeapi/v1/library/sync") + if err != nil { + result <- err + return + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + result <- errors.New("unexpected status") + return + } + data := make([]byte, len(firstChunk)) + _, err = io.ReadFull(resp.Body, data) + if err == nil && !bytes.Equal(data, firstChunk) { + err = errors.New("unexpected bytes in first chunk") + } + result <- err + }() + go func() { _, _ = writer.Write(firstChunk) }() + + select { + case err := <-result: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("proxy did not deliver bytes before upstream finished") + } + }) + + for _, tt := range []struct { + name string + gzip bool + publicURL string + wantOrigin string + }{ + {name: "plaintext uses configured public origin", publicURL: "https://reader.example.com/", wantOrigin: "https://reader.example.com"}, + {name: "gzipped response uses trusted forwarded origin", gzip: true, wantOrigin: "https://reader.example.com"}, + } { + t.Run(tt.name, func(t *testing.T) { + upstreamBody := []byte(`{"instapaper":"https://www.instapaper.com"}`) + responseHeader := http.Header{ + "Content-Type": {"application/json"}, + "Etag": {`"old-tag"`}, + "Last-Modified": {"Tue, 01 Jan 2019 00:00:00 GMT"}, + } + if tt.gzip { + var buf bytes.Buffer + writer := gzip.NewWriter(&buf) + if _, err := writer.Write(upstreamBody); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + upstreamBody = buf.Bytes() + responseHeader.Set("Content-Encoding", "gzip") + } + responseHeader.Set("Content-Length", strconv.Itoa(len(upstreamBody))) + a := NewApp(WithConfig(&config.Config{PublicURL: tt.publicURL}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{ + RoundTripFunc: func(req *http.Request) (*http.Response, error) { + if req.URL.Path != "/v1/initialization" || req.URL.RawQuery != "locale=en" || req.Header.Get("Accept-Encoding") != "identity" || req.Header.Get("If-None-Match") != "" || req.Header.Get("Range") != "" || req.Header.Get("If-Range") != "" { + t.Errorf("unexpected initialization request: %s, headers: %v", req.URL, req.Header) + } + return &http.Response{StatusCode: http.StatusOK, Header: responseHeader, Body: io.NopCloser(bytes.NewReader(upstreamBody))}, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization?locale=en", nil) + req.Host = "untrusted.example.com" + forwardedHost := "reader.example.com" + if tt.publicURL != "" { + forwardedHost = "ignored.example.com" + } + req.Header.Set("X-Forwarded-Host", forwardedHost) + req.Header.Set("X-Forwarded-Proto", "https") + req.Header.Set("If-None-Match", `"old-tag"`) + req.Header.Set("Range", "bytes=0-50") + req.Header.Set("If-Range", `"old-tag"`) + rr := httptest.NewRecorder() + storeHandler(a).ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + if got := rr.Header().Get("Content-Length"); got != strconv.Itoa(rr.Body.Len()) { + t.Errorf("Content-Length = %q, body size = %d", got, rr.Body.Len()) + } + if rr.Header().Get("ETag") != "" || rr.Header().Get("Last-Modified") != "" || rr.Header().Get("Cache-Control") != "private, no-store" { + t.Errorf("stale response headers: %v", rr.Header()) + } + var result []byte + if tt.gzip { + if rr.Header().Get("Content-Encoding") != "gzip" { + t.Errorf("Content-Encoding = %q", rr.Header().Get("Content-Encoding")) + } + reader, err := gzip.NewReader(rr.Body) + if err != nil { + t.Fatal(err) + } + result, err = io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + _ = reader.Close() + } else { + if rr.Header().Get("Content-Encoding") != "" { + t.Errorf("Content-Encoding = %q", rr.Header().Get("Content-Encoding")) + } + result = rr.Body.Bytes() + } + want := `{"instapaper":"` + tt.wantOrigin + `/instapaper-proxy/instapaper"}` + if string(result) != want { + t.Errorf("body = %q, want %q", result, want) + } + }) + } + + t.Run("preserves escaped path and query", func(t *testing.T) { + a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{ + RoundTripFunc: func(req *http.Request) (*http.Response, error) { + if req.URL.Path != "/v1//books" || req.URL.EscapedPath() != "/v1/%2Fbooks" || req.URL.RawQuery != "a=1%3B2" { + t.Errorf("upstream URL = %q, RawPath = %q", req.URL, req.URL.RawPath) + } + return &http.Response{StatusCode: http.StatusNoContent, Header: http.Header{}, Body: http.NoBody}, nil + }, + })) + rr := httptest.NewRecorder() + storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/%2Fbooks?a=1%3B2", nil)) + if rr.Code != http.StatusNoContent { + t.Errorf("status = %d, want 204", rr.Code) + } + }) + + for _, tt := range []struct { + name, encoding, body string + }{ + {name: "rejects invalid gzip initialization", encoding: "gzip", body: "not gzip"}, + {name: "rejects unsupported initialization encoding", encoding: "br", body: "compressed"}, + {name: "limits initialization size", body: strings.Repeat("x", maxInitializationBody+1)}, + } { + t.Run(tt.name, func(t *testing.T) { + a := NewApp(WithConfig(&config.Config{PublicURL: "https://reader.example.com"}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{ + RoundTripFunc: func(*http.Request) (*http.Response, error) { + header := http.Header{} + header.Set("Content-Encoding", tt.encoding) + return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(strings.NewReader(tt.body))}, nil + }, + })) + rr := httptest.NewRecorder() + storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil)) + if rr.Code != http.StatusBadGateway { + t.Errorf("status = %d, want 502", rr.Code) + } + }) + } + + t.Run("partial initialization is rejected", func(t *testing.T) { + a := NewApp(WithConfig(&config.Config{PublicURL: "https://reader.example.com"}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{ + RoundTripFunc: func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusPartialContent, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("partial"))}, nil + }, + })) + rr := httptest.NewRecorder() + storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil)) + if rr.Code != http.StatusBadGateway { + t.Errorf("status = %d, want 502", rr.Code) + } + }) + + t.Run("non-OK initialization passes through without rewriting", func(t *testing.T) { + body := `{"url":"https://www.instapaper.com"}` + a := NewApp(WithConfig(&config.Config{PublicURL: "https://reader.example.com"}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{ + RoundTripFunc: func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusNotFound, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body))}, nil + }, + })) + rr := httptest.NewRecorder() + storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil)) + if rr.Code != http.StatusNotFound || rr.Body.String() != body { + t.Errorf("status = %d, body = %q; want 404 and unchanged body", rr.Code, rr.Body.String()) + } + }) + + t.Run("encodes rewritten URLs as valid JSON strings", func(t *testing.T) { + urlWithQuote := `https://reader.example.com/path?value="quoted"` + resp := &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"instapaper":"https://www.instapaper.com"}`))} + if err := rewriteInitialization(resp, urlWithQuote); err != nil { + t.Fatal(err) + } + var decoded struct { + Instapaper string `json:"instapaper"` + } + if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil { + t.Fatalf("rewritten response is invalid JSON: %v", err) + } + if decoded.Instapaper != urlWithQuote { + t.Errorf("instapaper URL = %q, want %q", decoded.Instapaper, urlWithQuote) + } + }) + + t.Run("rejects malformed forwarded origin", func(t *testing.T) { + a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger)) + req := httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil) + req.Header.Set("X-Forwarded-Proto", "javascript") + rr := httptest.NewRecorder() + storeHandler(a).ServeHTTP(rr, req) + if rr.Code != http.StatusBadGateway { + t.Errorf("status = %d, want 502", rr.Code) + } + }) + + t.Run("rejects quoted forwarded host", func(t *testing.T) { + a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger)) + req := httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil) + req.Header.Set("X-Forwarded-Host", `reader.example.com"injected`) + rr := httptest.NewRecorder() + storeHandler(a).ServeHTTP(rr, req) + if rr.Code != http.StatusBadGateway { + t.Errorf("status = %d, want 502", rr.Code) + } + }) + + t.Run("returns 502 on upstream failure", func(t *testing.T) { + a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{ + RoundTripFunc: func(*http.Request) (*http.Response, error) { return nil, errors.New("upstream unavailable") }, + })) + rr := httptest.NewRecorder() + storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/library/sync", nil)) + if rr.Code != http.StatusBadGateway { + t.Errorf("status = %d, want 502", rr.Code) + } + }) + + t.Run("does not rewrite other initialization-like paths", func(t *testing.T) { + body := `{"url":"https://www.instapaper.com"}` + a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{ + RoundTripFunc: func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body))}, nil + }, + })) + rr := httptest.NewRecorder() + storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v2/initialization", nil)) + if rr.Body.String() != body { + t.Errorf("body = %q, want %q", rr.Body.String(), body) + } + }) +} diff --git a/internal/config/booksync_test.go b/internal/config/booksync_test.go new file mode 100644 index 0000000..c0df22c --- /dev/null +++ b/internal/config/booksync_test.go @@ -0,0 +1,61 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestBookSyncConfiguration(t *testing.T) { + tests := []struct { + name string + extra string + userExtra string + wantErr string + wantBookSyncURL string + wantPublicURL string + wantRemove bool + }{ + {name: "no optional settings"}, + {name: "old disabled switch is ignored", extra: "book_sync: false\n"}, + {name: "old enabled switch is ignored", extra: "book_sync: true\n"}, + {name: "custom sync endpoint with path", userExtra: " book_sync_url: https://books.example.com/api/kobo/secret\n", wantBookSyncURL: "https://books.example.com/api/kobo/secret"}, + {name: "http sync endpoint is allowed", userExtra: " book_sync_url: http://localhost:8081/kobo\n", wantBookSyncURL: "http://localhost:8081/kobo"}, + {name: "public URL and archived removal", extra: "public_url: https://reader.example.com/\n", userExtra: " remove_archived_from_kobo: true\n", wantPublicURL: "https://reader.example.com/", wantRemove: true}, + {name: "invalid sync scheme", userExtra: " book_sync_url: ftp://books.example.com\n", wantErr: "BookSyncURL"}, + {name: "relative sync URL", userExtra: " book_sync_url: /api/kobo/device\n", wantErr: "BookSyncURL"}, + {name: "sync URL missing host", userExtra: " book_sync_url: https:/kobo\n", wantErr: "BookSyncURL"}, + {name: "invalid sync port", userExtra: " book_sync_url: https://books.example.com:bad/kobo\n", wantErr: "BookSyncURL"}, + {name: "invalid public scheme", extra: "public_url: ftp://reader.example.com\n", wantErr: "PublicURL"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data := "readeck:\n host: https://readeck.example.com\n" + tt.extra + "users:\n - token: device\n readeck_access_token: readeck\n" + tt.userExtra + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(data), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("Load() error = %v, want error mentioning %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if got := cfg.Users[0].BookSyncURL; got != tt.wantBookSyncURL { + t.Errorf("BookSyncURL = %q, want %q", got, tt.wantBookSyncURL) + } + if cfg.PublicURL != tt.wantPublicURL { + t.Errorf("PublicURL = %q, want %q", cfg.PublicURL, tt.wantPublicURL) + } + if cfg.Users[0].RemoveArchivedFromKobo != tt.wantRemove { + t.Errorf("RemoveArchivedFromKobo = %v, want %v", cfg.Users[0].RemoveArchivedFromKobo, tt.wantRemove) + } + }) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 5c3e6a7..bff1822 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,8 +12,10 @@ import ( ) type User struct { - Token string `koanf:"token" validate:"required"` - ReadeckAccessToken string `koanf:"readeck_access_token" validate:"required"` + Token string `koanf:"token" validate:"required"` + ReadeckAccessToken string `koanf:"readeck_access_token" validate:"required"` + BookSyncURL string `koanf:"book_sync_url" validate:"omitempty,http_url"` + RemoveArchivedFromKobo bool `koanf:"remove_archived_from_kobo"` } type ConfigReadeck struct { @@ -21,12 +23,13 @@ type ConfigReadeck struct { } type Config struct { - Readeck ConfigReadeck `koanf:"readeck"` - Server struct { + Readeck ConfigReadeck `koanf:"readeck"` + Server struct { Port int `koanf:"port" validate:"min=1,max=65535"` } `koanf:"server"` - Users []User `koanf:"users" validate:"required,min=1,dive"` - LogLevel string `koanf:"log_level" validate:"oneof=error warn info debug"` + Users []User `koanf:"users" validate:"required,min=1,dive"` + LogLevel string `koanf:"log_level" validate:"oneof=error warn info debug"` + PublicURL string `koanf:"public_url" validate:"omitempty,http_url"` } func (c *Config) Validate() error { diff --git a/internal/readeck/client.go b/internal/readeck/client.go index 5dd1de4..e346c7a 100644 --- a/internal/readeck/client.go +++ b/internal/readeck/client.go @@ -9,7 +9,6 @@ import ( "mime" "mime/multipart" "net/http" - "net/http/httputil" // Added import "net/url" "strconv" "strings" @@ -129,26 +128,12 @@ func (c *Client) doRequestRaw(ctx context.Context, method, path string, queryPar req.Header.Set("Content-Type", "application/json") // Ensure Content-Type is set for requests with a body } - // Log the outgoing request for debugging - dump, err := httputil.DumpRequestOut(req, true) - if err != nil { - c.Logger.Errorf("Failed to dump outgoing request: %v", err) - } else { - c.Logger.Debugf("Outgoing Readeck API Request:\n%s", dump) - } - + start := time.Now() resp, err := c.HTTPClient.Do(req) if err != nil { return nil, fmt.Errorf("failed to execute request: %w", err) } - - // Log the incoming response for debugging - // dumpResp, err := httputil.DumpResponse(resp, true) - // if err != nil { - // c.Logger.Errorf("Failed to dump incoming response: %v", err) - // } else { - // c.Logger.Debugf("Incoming Readeck API Response:\n%s", dumpResp) - // } + c.Logger.Debugf("Readeck %s %s: HTTP %d in %s", req.Method, req.URL.Path, resp.StatusCode, time.Since(start)) if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { defer func() { _ = resp.Body.Close() }() diff --git a/internal/readeck/client_test.go b/internal/readeck/client_test.go index 83c13f0..c6a4b2d 100644 --- a/internal/readeck/client_test.go +++ b/internal/readeck/client_test.go @@ -1,10 +1,14 @@ package readeck import ( + "bytes" "context" "encoding/json" + "log" "net/http" "net/http/httptest" + "net/url" + "strings" "testing" "time" @@ -224,6 +228,54 @@ func TestCreateBookmark(t *testing.T) { } } +func TestRawRequestDebugLogOmitsSecrets(t *testing.T) { + const token = "private-readeck-token" + const query = "private-query-value" + const body = "private-body-value" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer "+token { + t.Errorf("Authorization header = %q, want bearer token", got) + } + if got := r.URL.Query().Get("q"); got != query { + t.Errorf("query value = %q, want %q", got, query) + } + var data map[string]string + if err := json.NewDecoder(r.Body).Decode(&data); err != nil { + t.Errorf("decode request body: %v", err) + } else if data["data"] != body { + t.Errorf("body data = %q, want %q", data["data"], body) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + var output bytes.Buffer + previousOutput := log.Writer() + log.SetOutput(&output) + defer log.SetOutput(previousOutput) + + client, err := NewClient(server.URL, token, logger.New(logger.DEBUG), nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.doRequestRaw(context.Background(), http.MethodPost, "/api/bookmarks/sync", url.Values{"q": {query}}, map[string]string{"data": body}) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + + got := output.String() + if !strings.Contains(got, "Readeck POST /api/bookmarks/sync: HTTP 200 in ") { + t.Errorf("missing request summary in debug log: %q", got) + } + for _, secret := range []string{token, query, body, "Authorization"} { + if strings.Contains(got, secret) { + t.Errorf("debug log contains private request data: %q", secret) + } + } +} + func TestGetBookmarksWithIsArchived(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/api/bookmarks" { diff --git a/internal/webserver/booksync_test.go b/internal/webserver/booksync_test.go new file mode 100644 index 0000000..f394025 --- /dev/null +++ b/internal/webserver/booksync_test.go @@ -0,0 +1,316 @@ +package webserver + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "readeckobo/internal/app" + "readeckobo/internal/config" + "readeckobo/internal/logger" +) + +func TestBookSyncRoutes(t *testing.T) { + for _, tt := range []struct { + name, path, scheme, host, upstreamPath, escapedPath, query string + wantStatus int + }{ + {"custom endpoint with path prefix", "/booksync/books/v1/library/sync?page=2", "https", "books.example.com", "/api/kobo/device/v1/library/sync", "/api/kobo/device/v1/library/sync", "page=2", http.StatusAccepted}, + {"custom HTTP endpoint", "/booksync/httpbook/v1/library/sync", "http", "localhost:8081", "/kobo/v1/library/sync", "/kobo/v1/library/sync", "", http.StatusAccepted}, + {"escaped suffix", "/booksync/books/v1/%2Fbooks?a=1%3B2", "https", "books.example.com", "/api/kobo/device/v1//books", "/api/kobo/device/v1/%2Fbooks", "a=1%3B2", http.StatusAccepted}, + {"custom endpoint root", "/booksync/books/", "https", "books.example.com", "/api/kobo/device/", "/api/kobo/device/", "", http.StatusAccepted}, + {"known token uses Kobo Store", "/booksync/store/v1/library/sync?page=3", "https", "storeapi.kobo.com", "/v1/library/sync", "/v1/library/sync", "page=3", http.StatusAccepted}, + {"unknown token", "/booksync/unknown/v1/library/sync", "", "", "", "", "", http.StatusNotFound}, + {"partial token does not match", "/booksync/book/v1/library/sync", "", "", "", "", "", http.StatusNotFound}, + } { + t.Run(tt.name, func(t *testing.T) { + var calls int + application := app.NewApp( + app.WithConfig(&config.Config{Users: []config.User{ + {Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"}, + {Token: "httpbook", BookSyncURL: "http://localhost:8081/kobo"}, + {Token: "store"}, + }}), + app.WithLogger(logger.New(logger.ERROR)), + app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) { + calls++ + if req.URL.Scheme != tt.scheme || req.URL.Host != tt.host || req.Host != tt.host || req.URL.Path != tt.upstreamPath || req.URL.EscapedPath() != tt.escapedPath || req.URL.RawQuery != tt.query { + t.Errorf("upstream URL = %s, Host = %q; want %s://%s, path %q, escaped %q, query %q", req.URL, req.Host, tt.scheme, tt.host, tt.upstreamPath, tt.escapedPath, tt.query) + } + if req.Method != http.MethodPost || req.Header.Get("X-Test") != "forwarded" { + t.Errorf("upstream method = %s, X-Test = %q", req.Method, req.Header.Get("X-Test")) + } + body, err := io.ReadAll(req.Body) + if err != nil || string(body) != "payload" { + t.Errorf("upstream body = %q, err = %v", body, err) + } + return &http.Response{StatusCode: http.StatusAccepted, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("book response"))}, nil + })), + ) + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, tt.path, strings.NewReader("payload")) + req.Header.Set("X-Test", "forwarded") + NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, req) + if rr.Code != tt.wantStatus { + t.Errorf("status = %d, want %d", rr.Code, tt.wantStatus) + } + if tt.host == "" { + if calls != 0 { + t.Errorf("unknown token reached upstream (%d calls)", calls) + } + } else if calls != 1 || rr.Body.String() != "book response" { + t.Errorf("upstream calls = %d, body = %q; want one call and book response", calls, rr.Body.String()) + } + }) + } +} + +func TestBookSyncDoesNotRetryConfiguredService(t *testing.T) { + var calls int + application := app.NewApp( + app.WithConfig(&config.Config{Users: []config.User{{Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"}}}), + app.WithLogger(logger.New(logger.ERROR)), + app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) { + calls++ + if req.URL.Host != "books.example.com" { + t.Errorf("upstream host = %q, want books.example.com", req.URL.Host) + } + return nil, errors.New("book service unavailable") + })), + ) + rr := httptest.NewRecorder() + NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/booksync/books/v1/library/sync", nil)) + if rr.Code != http.StatusBadGateway || calls != 1 { + t.Errorf("status = %d, upstream calls = %d; want 502 and one call", rr.Code, calls) + } +} + +func TestBookSyncInitialization(t *testing.T) { + for _, tt := range []struct { + name, token, upstreamHost, upstreamPath string + gzip bool + }{ + {"custom plaintext", "books", "books.example.com", "/api/kobo/device/v1/initialization", false}, + {"custom gzip", "books", "books.example.com", "/api/kobo/device/v1/initialization", true}, + {"store fallback", "store", "storeapi.kobo.com", "/v1/initialization", false}, + } { + t.Run(tt.name, func(t *testing.T) { + body := []byte(`{"instapaper":"https://www.instapaper.com"}`) + header := http.Header{"Content-Type": {"application/json"}, "Etag": {`"old"`}} + if tt.gzip { + var b bytes.Buffer + writer := gzip.NewWriter(&b) + if _, err := writer.Write(body); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + body = b.Bytes() + header.Set("Content-Encoding", "gzip") + } + application := app.NewApp( + app.WithConfig(&config.Config{PublicURL: "https://reader.example.com", Users: []config.User{ + {Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"}, {Token: "store"}, + }}), + app.WithLogger(logger.New(logger.ERROR)), + app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Host != tt.upstreamHost || req.URL.Path != tt.upstreamPath || req.URL.RawQuery != "locale=en" || req.Header.Get("Accept-Encoding") != "identity" || req.Header.Get("Range") != "" || req.Header.Get("If-None-Match") != "" { + t.Errorf("unexpected initialization upstream: URL=%s, headers=%v", req.URL, req.Header) + } + return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(bytes.NewReader(body))}, nil + })), + ) + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/booksync/"+tt.token+"/v1/initialization?locale=en", nil) + req.Header.Set("If-None-Match", `"old"`) + req.Header.Set("Range", "bytes=0-10") + NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, req) + if rr.Code != http.StatusOK || rr.Header().Get("ETag") != "" { + t.Fatalf("status = %d, headers = %v", rr.Code, rr.Header()) + } + if got := rr.Header().Get("Content-Length"); got != strconv.Itoa(rr.Body.Len()) { + t.Errorf("Content-Length = %q, body size = %d", got, rr.Body.Len()) + } + var result []byte + var err error + if tt.gzip { + if rr.Header().Get("Content-Encoding") != "gzip" { + t.Errorf("Content-Encoding = %q, want gzip", rr.Header().Get("Content-Encoding")) + } + reader, gzipErr := gzip.NewReader(rr.Body) + if gzipErr != nil { + t.Fatal(gzipErr) + } + result, err = io.ReadAll(reader) + _ = reader.Close() + } else { + result = rr.Body.Bytes() + } + if err != nil || string(result) != `{"instapaper":"https://reader.example.com/instapaper-proxy/instapaper"}` { + t.Errorf("rewritten body = %q, err = %v", result, err) + } + }) + } +} + +func TestBookSyncInitializationPreservesBookResourcesPerUser(t *testing.T) { + for _, tt := range []struct{ token, host, key string }{ + {"alice", "books-alice.example.com", "alice-key"}, + {"bob", "books-bob.example.com", "bob-key"}, + } { + t.Run(tt.token, func(t *testing.T) { + upstream := "https://" + tt.host + "/kobo/" + tt.key + body := `{"Resources":{"instapaper_env_url":"https://www.instapaper.com/api/kobo",` + + `"image_host":"https://` + tt.host + `",` + + `"image_url_template":"` + upstream + `/v1/books/{ImageId}/thumbnail/{Width}/{Height}/false/image.jpg",` + + `"image_url_quality_template":"` + upstream + `/v1/books/{ImageId}/thumbnail/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg",` + + `"other_resource":"https://cdn.kobo.com/book.jpg"}}` + application := app.NewApp( + app.WithConfig(&config.Config{PublicURL: "https://reader.example.com", Users: []config.User{ + {Token: "alice", BookSyncURL: "https://books-alice.example.com/kobo/alice-key"}, + {Token: "bob", BookSyncURL: "https://books-bob.example.com/kobo/bob-key"}, + }}), + app.WithLogger(logger.New(logger.ERROR)), + app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Host != tt.host || req.URL.Path != "/kobo/"+tt.key+"/v1/initialization" { + t.Errorf("upstream URL = %s, want %s/v1/initialization", req.URL, upstream) + } + if req.Header.Get("X-Forwarded-Host") != tt.host || req.Header.Get("X-Forwarded-Proto") != "https" { + t.Errorf("forwarded host/proto = %q/%q; want %s/https", req.Header.Get("X-Forwarded-Host"), req.Header.Get("X-Forwarded-Proto"), tt.host) + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}}, Body: io.NopCloser(strings.NewReader(body))}, nil + })), + ) + req := httptest.NewRequest(http.MethodGet, "/booksync/"+tt.token+"/v1/initialization", nil) + req.Header.Set("X-Forwarded-Host", "untrusted.example.com") + req.Header.Set("X-Forwarded-Proto", "https") + req.Header.Set("X-Forwarded-Prefix", "/wrong") + rr := httptest.NewRecorder() + NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + var result struct { + Resources map[string]string + } + if err := json.Unmarshal(rr.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + for key, want := range map[string]string{ + "instapaper_env_url": "https://reader.example.com/instapaper-proxy/instapaper/api/kobo", + "image_host": "https://" + tt.host, + "image_url_template": upstream + "/v1/books/{ImageId}/thumbnail/{Width}/{Height}/false/image.jpg", + "image_url_quality_template": upstream + "/v1/books/{ImageId}/thumbnail/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg", + "other_resource": "https://cdn.kobo.com/book.jpg", + } { + if got := result.Resources[key]; got != want { + t.Errorf("Resources[%s] = %q, want %q", key, got, want) + } + } + if !strings.Contains(rr.Body.String(), "/kobo/"+tt.key+"/v1/books/") { + t.Error("book service cover URL was not preserved") + } + }) + } +} + +func TestBookSyncLeavesOtherResponsesUntouched(t *testing.T) { + for _, tt := range []struct{ token, host, prefix string }{ + {"alice", "books-alice.example.com", "/kobo/alice-key"}, + {"bob", "books-bob.example.com", "/kobo/bob-key"}, + {"store", "storeapi.kobo.com", ""}, + } { + t.Run(tt.token, func(t *testing.T) { + link := "https://" + tt.host + tt.prefix + "/v1/books/book-1/file/epub?convert_kepub=true" + metadata := `{"EntitlementId":"book-1","CoverImageId":"cover-1","DownloadUrls":[{"Format":"KEPUB","Url":"` + link + `"}]}` + application := app.NewApp( + app.WithConfig(&config.Config{PublicURL: "https://reader.example.com", Users: []config.User{ + {Token: "alice", BookSyncURL: "https://books-alice.example.com/kobo/alice-key"}, + {Token: "bob", BookSyncURL: "https://books-bob.example.com/kobo/bob-key"}, + {Token: "store"}, + }}), + app.WithLogger(logger.New(logger.ERROR)), + app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Host != tt.host || req.Header.Get("If-None-Match") != `"old"` || req.Header.Get("Range") != "bytes=0-10" { + t.Errorf("upstream URL = %s, headers = %v", req.URL, req.Header) + } + if !strings.HasPrefix(req.URL.Path, tt.prefix+"/v1/library/") { + t.Errorf("upstream path = %q, want prefix %s/v1/library/", req.URL.Path, tt.prefix) + } + body := "[" + metadata + "]" + if strings.HasSuffix(req.URL.Path, "/sync") { + body = `[{"NewEntitlement":{"BookMetadata":` + metadata + `}}]` + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}, "Etag": {`"old"`}}, Body: io.NopCloser(strings.NewReader(body))}, nil + })), + ) + handler := NewHandler(application, logger.New(logger.ERROR)) + for _, path := range []string{"/v1/library/sync", "/v1/library/book-1/metadata"} { + req := httptest.NewRequest(http.MethodGet, "/booksync/"+tt.token+path, nil) + req.Header.Set("If-None-Match", `"old"`) + req.Header.Set("Range", "bytes=0-10") + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + wantBody := "[" + metadata + "]" + if path == "/v1/library/sync" { + wantBody = `[{"NewEntitlement":{"BookMetadata":` + metadata + `}}]` + } + if rr.Code != http.StatusOK || rr.Header().Get("Etag") != `"old"` || rr.Body.String() != wantBody { + t.Errorf("%s: status = %d, headers = %v, body = %s; want %s", path, rr.Code, rr.Header(), rr.Body.String(), wantBody) + } + } + }) + } +} + +func TestBookSyncStreamsDownload(t *testing.T) { + reader, writer := io.Pipe() + application := app.NewApp( + app.WithConfig(&config.Config{Users: []config.User{{Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"}}}), + app.WithLogger(logger.New(logger.ERROR)), + app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Path != "/api/kobo/device/v1/books/1/download" { + t.Errorf("upstream path = %q", req.URL.Path) + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/epub+zip"}}, Body: reader, ContentLength: -1}, nil + })), + ) + server := httptest.NewServer(NewHandler(application, logger.New(logger.ERROR))) + defer server.Close() + defer func() { _ = writer.Close() }() + + firstChunk := bytes.Repeat([]byte("x"), 32*1024) + result := make(chan error, 1) + go func() { + resp, err := server.Client().Get(server.URL + "/booksync/books/v1/books/1/download") + if err != nil { + result <- err + return + } + defer func() { _ = resp.Body.Close() }() + data := make([]byte, len(firstChunk)) + _, err = io.ReadFull(resp.Body, data) + if err == nil && !bytes.Equal(data, firstChunk) { + err = io.ErrUnexpectedEOF + } + result <- err + }() + go func() { _, _ = writer.Write(firstChunk) }() + select { + case err := <-result: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("book download did not stream before upstream completion") + } +} diff --git a/internal/webserver/middleware.go b/internal/webserver/middleware.go index d71ab86..0cd9c64 100644 --- a/internal/webserver/middleware.go +++ b/internal/webserver/middleware.go @@ -21,18 +21,16 @@ func (rw *responseWriter) WriteHeader(code int) { rw.ResponseWriter.WriteHeader(code) } +func (rw *responseWriter) Unwrap() http.ResponseWriter { + return rw.ResponseWriter +} + // LoggingMiddleware logs all incoming requests func LoggingMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { start := time.Now() rw := newResponseWriter(w) next.ServeHTTP(rw, r) - log.Printf( - "%-7s %s %d %s", - r.Method, - r.RequestURI, - rw.statusCode, - time.Since(start), - ) + log.Printf("%-7s %d %s", r.Method, rw.statusCode, time.Since(start)) }) } diff --git a/internal/webserver/webserver.go b/internal/webserver/webserver.go index 97a8b42..028f6ee 100644 --- a/internal/webserver/webserver.go +++ b/internal/webserver/webserver.go @@ -12,7 +12,12 @@ import ( func ListenAndServe(port int, application *app.App, logger *logger.Logger) { addr := fmt.Sprintf(":%d", port) logger.Infof("Web server starting on port %s", addr) + if err := http.ListenAndServe(addr, NewHandler(application, logger)); err != nil { + logger.Errorf("Web server failed to start: %v", err) + } +} +func NewHandler(application *app.App, logger *logger.Logger) http.Handler { mux := http.NewServeMux() // Register handlers @@ -20,18 +25,19 @@ func ListenAndServe(port int, application *app.App, logger *logger.Logger) { mux.HandleFunc("/api/kobo/download", application.HandleKoboDownload) mux.HandleFunc("/api/kobo/send", application.HandleKoboSend) mux.HandleFunc("/api/convert-image", application.HandleConvertImage) - mux.HandleFunc("/instapaper-proxy/storeapi/v1/initialization", application.HandleDumpAndForward) + mux.HandleFunc("/instapaper-proxy/instapaper/api/kobo/get", application.HandleKoboGet) + mux.HandleFunc("/instapaper-proxy/instapaper/api/kobo/download", application.HandleKoboDownload) + mux.HandleFunc("/instapaper-proxy/instapaper/api/kobo/send", application.HandleKoboSend) + storeProxy := http.StripPrefix("/instapaper-proxy/storeapi", http.HandlerFunc(application.HandleStoreProxy)) + mux.Handle("/instapaper-proxy/storeapi/", storeProxy) + mux.Handle("/instapaper-proxy/storeapi", storeProxy) + mux.HandleFunc("/booksync/{deviceToken}/", application.HandleBookSync) // Catch-all for unimplemented routes mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - logger.Warnf("404 Not Found: URL=%s, Method=%s, Params=%v", r.URL.Path, r.Method, r.URL.Query()) + logger.Warnf("404 Not Found: Method=%s", r.Method) http.Error(w, "404 Not Found", http.StatusNotFound) }) - // Apply logging middleware - loggedMux := LoggingMiddleware(mux) - - if err := http.ListenAndServe(addr, loggedMux); err != nil { - logger.Errorf("Web server failed to start: %v", err) - } -} \ No newline at end of file + return LoggingMiddleware(mux) +} diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go new file mode 100644 index 0000000..aad83f5 --- /dev/null +++ b/internal/webserver/webserver_test.go @@ -0,0 +1,121 @@ +package webserver + +import ( + "bytes" + "io" + "log" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "readeckobo/internal/app" + "readeckobo/internal/config" + "readeckobo/internal/logger" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + +func TestStoreRoutes(t *testing.T) { + application := app.NewApp( + app.WithConfig(&config.Config{PublicURL: "https://reader.example.com"}), + app.WithLogger(logger.New(logger.ERROR)), + app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Host != "storeapi.kobo.com" || req.Host != "storeapi.kobo.com" { + t.Errorf("upstream host = %q, Host = %q", req.URL.Host, req.Host) + } + switch req.URL.Path { + case "/", "/v1/initialization", "/v1/library/sync", "/v1//books": + default: + t.Errorf("unexpected upstream path = %q", req.URL.Path) + } + if req.URL.Path == "/v1//books" && req.URL.EscapedPath() != "/v1/%2Fbooks" { + t.Errorf("escaped upstream path = %q, want /v1/%%2Fbooks", req.URL.EscapedPath()) + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"url":"https://www.instapaper.com"}`))}, nil + })), + ) + handler := NewHandler(application, logger.New(logger.ERROR)) + for _, path := range []string{"/instapaper-proxy/storeapi/v1/initialization", "/instapaper-proxy/storeapi/v1/library/sync", "/instapaper-proxy/storeapi/v1/%2Fbooks?a=1%3B2", "/instapaper-proxy/storeapi"} { + t.Run(path, func(t *testing.T) { + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, path, nil)) + if rr.Code != http.StatusOK { + t.Errorf("status = %d, want 200", rr.Code) + } + }) + } + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi-nope", nil)) + if rr.Code != http.StatusNotFound { + t.Errorf("unknown store route status = %d, want 404", rr.Code) + } +} + +func TestArticleRoutes(t *testing.T) { + readeckServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/bookmarks/sync" { + t.Errorf("unexpected Readeck request: %s", r.URL.Path) + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, "[]") + })) + defer readeckServer.Close() + + application := app.NewApp( + app.WithConfig(&config.Config{ + Readeck: config.ConfigReadeck{Host: readeckServer.URL}, + Users: []config.User{{Token: "device", ReadeckAccessToken: "readeck"}}, + }), + app.WithLogger(logger.New(logger.ERROR)), + app.WithReadeckHTTPClient(readeckServer.Client()), + ) + handler := NewHandler(application, logger.New(logger.ERROR)) + for _, tt := range []struct { + name, path, body, wantBody string + wantStatus int + }{ + {"get", "/instapaper-proxy/instapaper/api/kobo/get", `{"access_token":"device"}`, `"status":1`, http.StatusOK}, + {"download", "/instapaper-proxy/instapaper/api/kobo/download", `{"access_token":"device"}`, "Missing 'url' parameter", http.StatusBadRequest}, + {"send", "/instapaper-proxy/instapaper/api/kobo/send", `{"access_token":"device","actions":[]}`, `"status":true`, http.StatusOK}, + } { + t.Run(tt.name, func(t *testing.T) { + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, tt.path, strings.NewReader(tt.body)) + req.Header.Set("Content-Type", "application/json") + handler.ServeHTTP(rr, req) + if rr.Code != tt.wantStatus || !strings.Contains(rr.Body.String(), tt.wantBody) { + t.Errorf("status = %d, body = %q; want %d and %q", rr.Code, rr.Body.String(), tt.wantStatus, tt.wantBody) + } + }) + } +} + +func TestRequestLogsDoNotExposeTokens(t *testing.T) { + var logs bytes.Buffer + old := log.Writer() + log.SetOutput(&logs) + defer log.SetOutput(old) + + application := app.NewApp( + app.WithConfig(&config.Config{}), + app.WithLogger(logger.New(logger.DEBUG)), + app.WithImageHTTPClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, io.ErrUnexpectedEOF + })}), + ) + handler := NewHandler(application, logger.New(logger.DEBUG)) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/booksync/private-token/?access_token=query-secret", nil)) + req := httptest.NewRequest(http.MethodPost, "/instapaper-proxy/instapaper/api/kobo/get", strings.NewReader(`{"access_token":"body-secret"}`)) + handler.ServeHTTP(httptest.NewRecorder(), req) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/api/convert-image?url=https%3A%2F%2Fexample.com%2Fimg%3Ftoken%3Dimage-secret", nil)) + for _, secret := range []string{"private-token", "query-secret", "body-secret", "image-secret"} { + if strings.Contains(logs.String(), secret) { + t.Errorf("request logs contain secret %q", secret) + } + } +} diff --git a/nginx.conf.snippet b/nginx.conf.snippet deleted file mode 100644 index 3cbbd39..0000000 --- a/nginx.conf.snippet +++ /dev/null @@ -1,30 +0,0 @@ - # Proxy Kobo Store API so that we can modify the returned Instapaper URL. - location /instapaper-proxy/storeapi { - proxy_ssl_server_name on; - proxy_ssl_name storeapi.kobo.com; - proxy_set_header Host storeapi.kobo.com; - proxy_pass https://storeapi.kobo.com/; - } - - location = /instapaper-proxy/storeapi/v1/initialization { - proxy_ssl_server_name on; - proxy_ssl_name storeapi.kobo.com; - proxy_set_header Host storeapi.kobo.com; - proxy_set_header Accept-Encoding ""; - proxy_pass https://storeapi.kobo.com/; - - sub_filter https://www.instapaper.com "$scheme://$host/instapaper-proxy/instapaper"; - sub_filter_types application/json; - } - - # Proxy to local Kobeck application - location /instapaper-proxy/instapaper/ { - proxy_pass http://readeckobo-upstream/; - proxy_set_header Host $host; - client_max_body_size 128M; - - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Prefix /instapaper-proxy/instapaper; - }