Support Kobo sync without custom Nginx routing #2

Merged
eleith merged 12 commits from eleith/feat/zero-nginx into main 2026-10-01 15:38:09 +00:00
23 changed files with 1707 additions and 275 deletions

No files matched your search

+4
View File
@@ -15,3 +15,7 @@
# llm agents
/.gemini/
/.claude/
# Local planning and sandbox configuration
/PROJECT_NGINX.md
/.nono.json
+34 -20
View File
@@ -9,11 +9,17 @@ when:
include:
- "**/*.go"
- "**/*.gohtml"
- "go.mod"
- "go.sum"
- ".golangci.yml"
- "scripts/check-coverage.sh"
- "bin/generate-encrypted-token.sh"
- ".woodpecker/build-docker.yml"
- "Dockerfile"
exclude:
- "**/*.md"
on_empty: true
- event: pull_request
- event: tag
ref: "refs/tags/v*.*.*"
@@ -50,6 +56,29 @@ steps:
when:
- event: tag
extract-pr-tag:
image: alpine:3.22
commands:
- |
branch_slug=$(printf '%s' "$CI_COMMIT_SOURCE_BRANCH" | sed 's/[^A-Za-z0-9_.-]/-/g' | cut -c1-96)
printf 'pr-%s-%s\n' "$CI_COMMIT_PULL_REQUEST" "$branch_slug" > .tags
cat .tags
when:
- event: pull_request
build-docker-pr:
image: woodpeckerci/plugin-docker-buildx:6.0.3
settings:
repo: git.eleith.com/eleith.private/readeckobo
username: eleith
dockerfile: Dockerfile
registry: https://git.eleith.com
password:
from_secret: gitea_docker_registry
tags_file: .tags
when:
- event: pull_request
build-docker-main:
image: woodpeckerci/plugin-docker-buildx:6.0.3
settings:
@@ -79,8 +108,8 @@ steps:
- event: tag
ref: "refs/tags/v*.*.*"
notify-success:
image: deblan/woodpecker-email:3.2.0
notify:
image: deblan/woodpecker-email:3.3.0
settings:
dsn:
from_secret: woodpecker_email_dsn
@@ -91,23 +120,8 @@ steps:
from_secret: woodpecker_email_from_name
recipients:
from_secret: woodpecker_email_recipient
level: success
recipients_only: true
when:
- status: [success]
notify-failure:
image: deblan/woodpecker-email:3.2.0
settings:
dsn:
from_secret: woodpecker_email_dsn
from:
address:
from_secret: woodpecker_email_from_address
name:
from_secret: woodpecker_email_from_name
recipients:
from_secret: woodpecker_email_recipient
level: failure
recipients_only: true
when:
- status: [failure]
- event: push
branch: main
status: [failure, success]
+1 -2
View File
@@ -16,8 +16,7 @@ The project follows a standard Go project layout:
- **`internal/models/`**: Shared data structures for Kobo and Readeck API payloads.
## Deployment & Infrastructure
- **Proxy Interception**: The system relies on a reverse proxy (like Nginx) to intercept requests from the Kobo device intended for `www.instapaper.com` and redirect them to this service.
- **Configuration**: See `nginx.conf.snippet` for the rewrite rules.
- **Deployment**: Kobo requests can reach this app through Cloudflare Tunnel or Nginx. See `docs/CLOUDFLARE.md` and `docs/NGINX.md` for the routing and initialization rewrite examples.
- **Docker**: The application is containerized (`Dockerfile`, `docker-compose.yml`).
## Development Guidelines for AI Agents
+53 -81
View File
@@ -17,13 +17,34 @@ more.
## 🚀 Quick Start (for Users)
Getting up and running is a breeze with Docker.
### 1. Choose how to publish
### 1. Configure `readeckobo`
Choose a public HTTPS hostname (for example, `readeckobo.example.com`) and one
deployment method:
First, copy `config.yaml.example` to `config.yaml` and edit it to match your setup.
* [Cloudflare Tunnel](docs/CLOUDFLARE.md): forward requests to `readeckobo`.
* [Nginx](docs/NGINX.md): proxy Kobo Store and article requests.
For a detailed explanation of all options and how to get your Readeck API token, see [docs/CONFIG.md](docs/CONFIG.md).
Both use the same Kobo settings below.
### 2. Generate a device token
Copy `config.yaml.example` to `config.yaml`, then build the image. Find your
Kobo's serial number under **Settings -> Device Information**. Replace
`<YOUR_KOBO_SERIAL>` below with that number (without angle brackets):
```sh
docker-compose build
docker-compose run --rm --entrypoint /app/bin/generate-encrypted-token.sh readeckobo <YOUR_KOBO_SERIAL>
```
This one-off container generates a **plain text token** for `config.yaml` and
an **encrypted token** for the Kobo. The app is not running yet.
### 3. Configure `readeckobo` and Kobo
Edit `config.yaml` with your Readeck host and API token and the **plain text**
device token from step 2:
```yaml
server:
@@ -32,97 +53,48 @@ log_level: info
readeck:
host: "https://your-readeck-instance.com"
users:
- token: "a-random-uuid-token-for-a-kobo"
- token: "<THE-PLAIN-TEXT-TOKEN-FROM-THE-SCRIPT>"
readeck_access_token: "a-readeck-api-token"
```
### 2. Run with Docker
For other options, see [docs/CONFIG.md](docs/CONFIG.md). To sync ebooks too,
see the [Kobo ebook sync guide](docs/KOBO_SYNC.md) for the per-device
`api_endpoint` instead of the article-only setting below.
Once your configuration is ready, fire it up!
Mount your Kobo and edit `.kobo/Kobo/Kobo eReader.conf`, using the **encrypted**
token from the script. Replace `readeckobo.example.com` with the hostname you
chose in step 1:
```ini
[OneStoreServices]
api_endpoint=https://readeckobo.example.com/instapaper-proxy/storeapi
instapaper_env_url=https://readeckobo.example.com
[Instapaper]
AccessToken=@ByteArray(<THE-ENCRYPTED-TOKEN-FROM-THE-SCRIPT>)
```
### 4. Run the app
```sh
docker-compose build
docker-compose up -d
```
The server will be available at `http://localhost:8080`.
### 3. Generate a Device Token
For each Kobo device, you will need a unique token. This process involves
generating a token and then encrypting it for the Kobo device.
First, find your Kobo's serial number, which is available under
**Settings -> Device Information** on your e-reader.
With `readeckobo` running, use this command to generate and encrypt the token,
replacing `<YOUR_KOBO_SERIAL>` with your device's serial number:
```sh
docker-compose exec readeckobo bin/generate-encrypted-token.sh <YOUR_KOBO_SERIAL>
```
The script will output two important pieces of information:
1. A **plain text UUID token** to be used in your `config.yaml`.
2. An **encrypted token** to be used in your Kobo's configuration file.
### 4. Configure Your `readeckobo` and Kobo Device
Follow the output from the script to configure your services.
1. **Update `config.yaml`**: Add the plain text UUID token to the `users`
section of your `config.yaml`.
```yaml
users:
- token: "<THE-PLAIN-TEXT-UUID-FROM-THE-SCRIPT>"
readeck_access_token: "a-readeck-api-token"
```
2. **Update Your Kobo**: Mount your Kobo and find the
`.kobo/Kobo/Kobo eReader.conf` file. Add or update these settings using the
**encrypted** token from the script's output.
```ini
[OneStoreServices]
api_endpoint=https://readeckobo.example.com/instapaper-proxy/storeapi
instapaper_env_url=https://readeckobo.example.com
[Instapaper]
AccessToken=@ByteArray(<THE-ENCRYPTED-TOKEN-FROM-THE-SCRIPT>)
```
Replace `https://readeckobo.example.com` with the full path to where you are deploying
the readeckobo service to
### 5. Set Up a Reverse Proxy
`readeckobo` must be run behind a reverse proxy to handle HTTPS. It's crucial
to proxy three specific location blocks, as shown in our `nginx.conf.snippet`
example.
Your Kobo device periodically re-syncs its configuration from Kobo's servers,
which can overwrite your custom Instapaper endpoint. The proxy rules below
ensure this connection is preserved.
<!-- markdownlint-disable MD013 -->
| Location Block | Proxies To | Purpose |
| ----------------------------------------------- | ------------------------ | --------------------------------------------------------------------------------------------------- |
| `/instapaper-proxy/instapaper/` | `readeckobo` application | Handles the main Instapaper API requests (sync, download, etc.) to your `readeckobo` instance. |
| `/instapaper-proxy/storeapi/` | `storeapi.kobo.com` | Forwards general API requests to Kobo's servers. |
| `/instapaper-proxy/storeapi/v1/initialization` | `storeapi.kobo.com` | Intercepts the Kobo configuration response to rewrite the Instapaper URL back to your proxy endpoint. |
<!-- markdownlint-enable MD013 -->
Without these rules, your Kobo will eventually lose its connection to `readeckobo`.
The app is available locally at `http://localhost:8080`. Set up your chosen
[Cloudflare Tunnel](docs/CLOUDFLARE.md) or [Nginx](docs/NGINX.md) deployment
before or after starting the app; the public HTTPS address must work before
Kobo syncs. If you later edit `config.yaml`, recreate the running Docker
container so it reads the new file; the deployment guides show the command.
## 🔒 A Quick Word on Security
A little security goes a long way.
* **Use HTTPS:** deploy behind a reverse proxy that provides HTTPS
* **Stay Local:** Keep it on your local private network
* **Kobo Password:** prevent unauthorized mounting with a Kobo password
* **Use HTTPS:** follow either deployment guide above, and keep the app's HTTP
port private.
* **Protect tokens:** don't share config files or request URLs containing device
tokens. Browser login or bot challenges may prevent Kobo sync.
* **Kobo Password:** prevent unauthorized mounting with a Kobo password.
## 🧑‍💻 For Developers
+7 -1
View File
@@ -2,7 +2,13 @@ server:
port: 8080
log_level: info
readeck:
host: "https://your-readeck-instance.com"
host: "https://readeck.example.com"
# public_url: "https://readeckobo.example.com" # optional: use if proxy headers cannot supply a trusted public origin
users:
- token: "a-very-secret-token-for-your-kobo"
readeck_access_token: "your-plain-text-readeck-access-token"
# Optional: overrides the Kobo Store fallback for this device.
# book_sync_url: "https://books.example.com/api/kobo/device-token"
# Optional: send archived bookmarks with Kobo status "2" during incremental sync.
# Default false keeps status "1"; neither setting deletes Readeck bookmarks.
# remove_archived_from_kobo: false
+41
View File
@@ -0,0 +1,41 @@
# Cloudflare Tunnel
Use the [README quick start](../README.md) for `config.yaml` and the Kobo's Instapaper settings. This guide publishes that same setup over HTTPS without Nginx.
## Publish the app
For a simple Tunnel route, set `public_url` to your public HTTPS origin in
`config.yaml`. This pins the rewritten Instapaper URL without relying on a
forwarded host header the Tunnel may not overwrite:
```yaml
public_url: "https://readeckobo.example.com"
```
If you are changing an existing deployment, recreate the Docker service to
reload the bind-mounted config file (`docker-compose up -d --force-recreate
readeckobo`), or restart your non-Docker process.
Before making the hostname public, add a [Cloudflare WAF custom rule](https://developers.cloudflare.com/waf/custom-rules/) with **Block** action for `(http.host eq "readeckobo.example.com" and http.request.uri.path eq "/api/convert-image")`. This direct, unauthenticated endpoint fetches a caller-provided URL and is not needed by the prefixed Kobo article routes. The Tunnel can still forward all other paths without path-specific routing rules.
Add a [published application route](https://developers.cloudflare.com/cloudflare-one/networks/routes/add-routes/) to a connected Cloudflare Tunnel:
| Route setting | Example |
| :--- | :--- |
| Public hostname | `readeckobo.example.com` |
| Service URL | `http://localhost:8080` |
Use the actual address `cloudflared` can reach: `localhost` works when it runs on the same host as the published app port, not when it runs in a separate container. Forward **all paths** to the app without a path rule. Go proxies Kobo Store traffic and rewrites initialization so Kobo continues sending Instapaper requests to `readeckobo`.
## Verify and secure
Check that initialization points back to your hostname:
```sh
curl --compressed -fsS https://readeckobo.example.com/instapaper-proxy/storeapi/v1/initialization \
| grep -F 'https://readeckobo.example.com/instapaper-proxy/instapaper'
```
Then check sync on a real Kobo. Browser login or bot challenges at the edge may block its requests. `docker-compose.yml` publishes port 8080 on all interfaces: restrict direct HTTP access with a firewall, or bind the port to `127.0.0.1:8080:8080` if `cloudflared` runs on the host. If it runs in another container or host, use a private network instead. Keep device tokens and request URLs containing them private.
For Kobo ebook syncing with a compatible service, see [Kobo ebook sync](KOBO_SYNC.md). The same Tunnel route forwards those requests to readeckobo.
+8 -15
View File
@@ -1,6 +1,7 @@
# Configuration Reference
This document provides a detailed reference for all available configuration options in `config.yaml`.
Configuration options for `config.yaml`. For a starting file, see
[config.yaml.example](../config.yaml.example) or the [quick start](../README.md#3-configure-readeckobo-and-kobo).
## Top-Level Options
@@ -10,6 +11,9 @@ This document provides a detailed reference for all available configuration opti
| `log_level` | Output verbosity. Options: `debug`, `info`, `warn`, `error`. | String | `info` | No |
| `readeck.host` | The full base URL of your Readeck instance. | URL String | - | **Yes** |
| `users` | A list of Kobo-to-Readeck user mappings. | List of Objects | - | **Yes** |
| `public_url` | Public origin used for the rewritten Instapaper URL in Kobo initialization. | URL String | Request host/forwarding headers | No |
Set `public_url` when your ingress cannot reliably supply the public HTTPS origin, cannot overwrite client-supplied forwarding headers, or you need one canonical hostname. Use the origin only (no path, query, or credentials). Otherwise, Go-proxied initialization uses `X-Forwarded-Host` (or the request `Host`) and `X-Forwarded-Proto` (or the connection scheme), **not** `readeck.host`. Keep the app behind a trusted ingress when relying on those headers. See [Cloudflare Tunnel](CLOUDFLARE.md) and [Nginx](NGINX.md).
## User Object
@@ -19,6 +23,8 @@ Each entry in the `users` list maps a Kobo device identity to a Readeck account.
| :--- | :--- |
| `token` | A unique UUID used to identify the Kobo device. This is the **Plain Text** token generated by the `bin/generate-encrypted-token.sh` script. |
| `readeck_access_token` | The API token for the Readeck account you want to sync. |
| `book_sync_url` | Optional per-user Kobo API endpoint for an ebook service such as Komga. When used through `/booksync/<device-token>/`, it must generate cover and download URLs the Kobo can reach directly; readeckobo does not rewrite them. See [Kobo ebook sync](KOBO_SYNC.md). |
| `remove_archived_from_kobo` | Optional, default `false`. In incremental sync, send archived items with status `"2"` (requesting removal) instead of `"1"`. Full sync excludes them; neither setting deletes Readeck data. Verify on a real Kobo. |
### How to get a Readeck API Token
@@ -27,17 +33,4 @@ Each entry in the `users` list maps a Kobo device identity to a Readeck account.
3. Create a new token with at least **Bookmarks: Read** permissions (Write permissions are needed for archiving/favoriting).
4. Copy the token string into your `config.yaml`.
## Example `config.yaml`
```yaml
server:
port: 8080
log_level: debug
readeck:
host: "https://my-readeck.example.com"
users:
- token: "550e8400-e29b-41d4-a716-446655440000" # From generate-encrypted-token.sh
readeck_access_token: "rdk_..." # From Readeck Profile
```
For Kobo-compatible ebook services, see [Kobo ebook sync](KOBO_SYNC.md).
+79
View File
@@ -0,0 +1,79 @@
# Kobo ebook sync alongside Readeck articles
Use a per-device readeckobo API endpoint to sync Readeck articles alongside
books from a Kobo-compatible service. Readeckobo forwards Kobo API requests to
the selected user's ebook service and points the initialization response's
Instapaper URL at readeckobo. Book-service URLs in initialization and later
responses pass through unchanged. Covers and ebook downloads use the absolute
URLs generated by the ebook service, so the Kobo contacts that service directly.
[Komga's Kobo sync](https://komga.org/docs/guides/kobo/) is one example.
## Configure the ebook service
For each Kobo device, set `book_sync_url` to that user's full **Kobo API
endpoint** on the ebook service, including any authentication path. For
example, with Komga:
```yaml
users:
- token: "<READECKOBO-PLAIN-DEVICE-TOKEN>"
readeck_access_token: "<READECK-API-TOKEN>"
book_sync_url: "https://books.example.com/kobo/<KOMGA_API_KEY>"
```
Set `public_url` if your ingress cannot reliably provide readeckobo's
public HTTPS origin; see [Cloudflare Tunnel](CLOUDFLARE.md) and
[Nginx](NGINX.md).
Use an ebook-service hostname that both readeckobo and the Kobo can reach. Its
initialization response should contain the literal `https://www.instapaper.com`
in its Instapaper URL or already contain the desired readeckobo URL. Go replaces
that literal in a successful initialization response with
`https://reader.example.com/instapaper-proxy/instapaper`. The service must generate
Kobo-reachable cover and download URLs because readeckobo leaves them unchanged.
Publish the ebook service separately from readeckobo if needed.
Reload readeckobo after changing `config.yaml` (for Docker Compose:
`docker-compose up -d --force-recreate readeckobo`).
## Publish readeckobo
[Cloudflare Tunnel](CLOUDFLARE.md) forwards `/booksync/` to Go with its normal
app route. With [Nginx](NGINX.md#kobo-ebook-syncing), add the `/booksync/`
location to readeckobo's public HTTPS server. The plain device token in the
route selects the matching user's `book_sync_url`; readeckobo forwards the
remaining path and query to that service. The same public hostname also serves
article requests.
Check the initialization response before changing the Kobo:
```text
https://reader.example.com/booksync/<READECKOBO-PLAIN-DEVICE-TOKEN>/v1/initialization
```
Its `instapaper_env_url` should point to readeckobo. For a service returning
`https://www.instapaper.com/api/kobo`, the resulting URL is
`https://reader.example.com/instapaper-proxy/instapaper/api/kobo`. Its
`image_host` and image templates should contain URLs reachable by the Kobo,
whether served by the ebook service or a CDN. Treat the request URL and the service API key as secrets
when sharing logs or screenshots.
## Configure the Kobo
In `.kobo/Kobo/Kobo eReader.conf`, set the per-device API endpoint:
```ini
[OneStoreServices]
api_endpoint=https://reader.example.com/booksync/<READECKOBO-PLAIN-DEVICE-TOKEN>
```
Keep the Kobo's existing `instapaper_env_url` and encrypted Instapaper
`AccessToken`. The token in `api_endpoint` is readeckobo's plain device token;
`book_sync_url` holds the ebook service's endpoint and its own credential.
Configure the ebook service to handle any Kobo Store requests you need; for
example, Komga offers Kobo Store proxying in its Server Settings.
On a real Kobo, verify initialization, library sync, covers, ebook downloads,
and Readeck articles. Check the generated service URLs and forwarding headers
for your deployment. To roll back, restore the Kobo's previous `api_endpoint`.
+81
View File
@@ -0,0 +1,81 @@
# Nginx
Use the [README quick start](../README.md) for `config.yaml` and the Kobo's
Instapaper settings.
## Kobo article syncing
Add these locations to the HTTPS `server` block for **readeckobo's public
hostname** (the Instapaper article address, and the `api_endpoint` address when
using ebook sync).
The example assumes Nginx can reach `readeckobo` at `127.0.0.1:8080`; use your
app's address if different.
```nginx
# The bare Kobo Store endpoint.
location = /instapaper-proxy/storeapi {
proxy_ssl_server_name on;
proxy_ssl_name storeapi.kobo.com;
proxy_set_header Host storeapi.kobo.com;
proxy_pass https://storeapi.kobo.com/;
}
# Forward Store subpaths with a single leading slash upstream.
location /instapaper-proxy/storeapi/ {
proxy_ssl_server_name on;
proxy_ssl_name storeapi.kobo.com;
proxy_set_header Host storeapi.kobo.com;
proxy_pass https://storeapi.kobo.com/;
}
# Fetch Kobo Store initialization for the Instapaper URL rewrite.
location = /instapaper-proxy/storeapi/v1/initialization {
proxy_ssl_server_name on;
proxy_ssl_name storeapi.kobo.com;
proxy_set_header Host storeapi.kobo.com;
proxy_set_header Accept-Encoding "";
proxy_set_header If-None-Match "";
proxy_set_header If-Modified-Since "";
proxy_set_header Range "";
proxy_set_header If-Range "";
proxy_pass https://storeapi.kobo.com/v1/initialization;
sub_filter https://www.instapaper.com "$scheme://$host/instapaper-proxy/instapaper";
sub_filter_types application/json;
}
# Instapaper article requests go to readeckobo.
location /instapaper-proxy/instapaper/ {
proxy_pass http://127.0.0.1:8080/;
proxy_set_header Host $host;
client_max_body_size 128M;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Prefix /instapaper-proxy/instapaper;
}
```
Keep port 8080 private: `docker-compose.yml` exposes it on all interfaces by
default. Bind it to `127.0.0.1:8080:8080` if Nginx runs on the host, or use a
private network if it runs elsewhere.
## Kobo ebook syncing
For [Kobo ebook sync](KOBO_SYNC.md), add this location to readeckobo's public
HTTPS `server` block. The `proxy_pass` forwards the full
`/booksync/<device-token>/...` path to Go, which selects the user's ebook
service. The ebook service publishes its own Kobo-reachable cover and download
URLs:
```nginx
location /booksync/ {
access_log off; # Request paths contain a device token.
proxy_pass http://127.0.0.1:8080;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Proto $scheme;
}
```
+103 -72
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"image"
"image/draw"
@@ -11,8 +12,8 @@ import (
"image/jpeg"
_ "image/png"
"io"
"mime"
"net/http"
"net/http/httputil"
"net/url"
"strconv"
"strings"
@@ -33,6 +34,7 @@ type App struct {
Logger *logger.Logger
ImageHTTPClient *http.Client
ReadeckHTTPClient *http.Client
ProxyTransport http.RoundTripper
}
func WithImageHTTPClient(client *http.Client) Option {
@@ -69,6 +71,12 @@ func WithReadeckHTTPClient(client *http.Client) Option {
}
}
func WithProxyTransport(transport http.RoundTripper) Option {
return func(a *App) {
a.ProxyTransport = transport
}
}
func (a *App) handleFullSync(ctx context.Context, readeckClient *readeck.Client, req *models.KoboGetRequest) (map[string]models.KoboArticleItem, int, error) {
count, _ := strconv.Atoi(req.Count)
offset, _ := strconv.Atoi(req.Offset)
@@ -136,7 +144,7 @@ func (a *App) handleFullSync(ctx context.Context, readeckClient *readeck.Client,
return resultList, totalNonArchivedBookmarks, nil
}
func (a *App) handleIncrementalSync(ctx context.Context, readeckClient *readeck.Client, since *time.Time) (map[string]models.KoboArticleItem, int, error) {
func (a *App) handleIncrementalSync(ctx context.Context, readeckClient *readeck.Client, since *time.Time, user config.User) (map[string]models.KoboArticleItem, int, error) {
resultList := make(map[string]models.KoboArticleItem)
bsyncs, err := readeckClient.GetBookmarksSync(ctx, since)
@@ -186,6 +194,9 @@ func (a *App) handleIncrementalSync(ctx context.Context, readeckClient *readeck.
if bookmark.IsArchived {
entry.Status = "1"
if user.RemoveArchivedFromKobo {
entry.Status = "2"
}
} else {
entry.Status = "0"
totalNonArchivedBookmarks++
@@ -200,12 +211,12 @@ func (a *App) HandleKoboGet(w http.ResponseWriter, r *http.Request) {
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, "Failed to read request body", http.StatusInternalServerError)
a.Logger.Errorf("Error reading /api/kobo/get request body: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error reading /api/kobo/get request body: %v", err)
return
}
r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/get:\nMethod: %s\nURL: %s\nHeaders: %v\nBody: %s", r.Method, r.URL, r.Header, string(bodyBytes))
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/get: Method=%s", r.Method)
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
@@ -215,21 +226,21 @@ func (a *App) HandleKoboGet(w http.ResponseWriter, r *http.Request) {
var req models.KoboGetRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "Invalid request body", http.StatusBadRequest)
a.Logger.Errorf("Error decoding /api/kobo/get request: %v, body: %s, URL: %s, Params: %v", err, string(bodyBytes), r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error decoding /api/kobo/get request: %v", err)
return
}
readeckToken, err := a.getReadeckToken(req.AccessToken)
user, err := a.getUser(req.AccessToken)
if err != nil {
http.Error(w, "Invalid access token", http.StatusUnauthorized)
a.Logger.Errorf("Error authenticating token for /api/kobo/get: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error authenticating token for /api/kobo/get: %v", err)
return
}
readeckClient, err := a.newReadeckClient(readeckToken)
readeckClient, err := a.newReadeckClient(user.ReadeckAccessToken)
if err != nil {
http.Error(w, "Failed to initialize Readeck client", http.StatusInternalServerError)
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/get: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/get: %v", err)
return
}
@@ -252,7 +263,7 @@ func (a *App) HandleKoboGet(w http.ResponseWriter, r *http.Request) {
resultList, total, err = a.handleFullSync(r.Context(), readeckClient, &req)
} else {
a.Logger.Debugf("Handling incremental sync.")
resultList, total, err = a.handleIncrementalSync(r.Context(), readeckClient, since)
resultList, total, err = a.handleIncrementalSync(r.Context(), readeckClient, since, user)
}
if err != nil {
@@ -325,12 +336,12 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, "Failed to read request body", http.StatusInternalServerError)
a.Logger.Errorf("Error reading /api/kobo/download request body: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error reading /api/kobo/download request body: %v", err)
return
}
r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/download:\nMethod: %s\nURL: %s\nHeaders: %v\nBody: %s", r.Method, r.URL, r.Header, string(bodyBytes))
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/download: Method=%s", r.Method)
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
@@ -339,44 +350,45 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
var req models.KoboDownloadRequest
if err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&req); err != nil {
if err := r.ParseForm(); err != nil {
form, err := downloadFormValues(r, bodyBytes)
if err != nil {
http.Error(w, "Invalid request body or form data", http.StatusBadRequest)
a.Logger.Errorf("Error decoding /api/kobo/download request: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error decoding /api/kobo/download request: %v", err)
return
}
req.AccessToken = r.FormValue("access_token")
req.ConsumerKey = r.FormValue("consumer_key")
req.Images, _ = strconv.Atoi(r.FormValue("images"))
req.Refresh, _ = strconv.Atoi(r.FormValue("refresh"))
req.Output = r.FormValue("output")
req.URL = r.FormValue("url")
req.AccessToken = form.Get("access_token")
req.ConsumerKey = form.Get("consumer_key")
req.Images, _ = strconv.Atoi(form.Get("images"))
req.Refresh, _ = strconv.Atoi(form.Get("refresh"))
req.Output = form.Get("output")
req.URL = form.Get("url")
}
readeckToken, err := a.getReadeckToken(req.AccessToken)
if err != nil {
http.Error(w, "Invalid access token", http.StatusUnauthorized)
a.Logger.Errorf("Error authenticating token for /api/kobo/download: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error authenticating token for /api/kobo/download: %v", err)
return
}
readeckClient, err := a.newReadeckClient(readeckToken)
if err != nil {
http.Error(w, "Failed to initialize Readeck client", http.StatusInternalServerError)
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/download: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/download: %v", err)
return
}
reqURLStr := req.URL
if reqURLStr == "" {
http.Error(w, "Missing 'url' parameter", http.StatusBadRequest)
a.Logger.Errorf("Error: Missing 'url' parameter in /api/kobo/download request, URL: %s, Params: %v", r.URL.Path, r.URL.Query())
a.Logger.Errorf("Missing 'url' parameter in /api/kobo/download request")
return
}
parsedURL, err := url.Parse(reqURLStr)
if err != nil {
http.Error(w, "Invalid 'url' parameter", http.StatusBadRequest)
a.Logger.Errorf("Error: Invalid 'url' parameter in /api/kobo/download request: %v, url: %s, URL: %s, Params: %v", err, reqURLStr, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Invalid 'url' parameter in /api/kobo/download request: %T", err)
return
}
@@ -392,7 +404,7 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
isArchived := false
bookmarks, tp, err := readeckClient.GetBookmarks(ctx, site, currentPage, &isArchived)
if err != nil {
a.Logger.Warnf("Error searching Readeck bookmarks for site %s, page %d in /api/kobo/download: %v, URL: %s, Params: %v", site, currentPage, err, r.URL.Path, r.URL.Query())
a.Logger.Warnf("Error searching Readeck bookmarks for site %s, page %d in /api/kobo/download: %v", site, currentPage, err)
break
}
totalPages = tp
@@ -401,7 +413,7 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
if bookmarks[i].URL != "" {
match, err := compareURLs(bookmarks[i].URL, reqURLStr)
if err != nil {
a.Logger.Warnf("Error comparing URLs for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarks[i].ID, err, r.URL.Path, r.URL.Query())
a.Logger.Warnf("Error comparing URLs for bookmark %s in /api/kobo/download: %T", bookmarks[i].ID, err)
continue
}
if match {
@@ -428,14 +440,14 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
articleHTML, err := readeckClient.GetBookmarkArticle(ctx, bookmarkFound.ID)
if err != nil {
http.Error(w, "Failed to fetch article content", http.StatusInternalServerError)
a.Logger.Errorf("Error fetching article content for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarkFound.ID, err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error fetching article content for bookmark %s in /api/kobo/download: %v", bookmarkFound.ID, err)
return
}
doc, err := html.Parse(strings.NewReader(articleHTML))
if err != nil {
http.Error(w, "Failed to parse article HTML", http.StatusInternalServerError)
a.Logger.Errorf("Error parsing article HTML for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarkFound.ID, err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error parsing article HTML for bookmark %s in /api/kobo/download: %v", bookmarkFound.ID, err)
return
}
@@ -474,7 +486,7 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
var buf bytes.Buffer
if err := html.Render(&buf, doc); err != nil {
http.Error(w, "Failed to render modified HTML", http.StatusInternalServerError)
a.Logger.Errorf("Error rendering modified HTML for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarkFound.ID, err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error rendering modified HTML for bookmark %s in /api/kobo/download: %v", bookmarkFound.ID, err)
return
}
@@ -485,10 +497,47 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(response); err != nil {
a.Logger.Errorf("Error encoding response for /api/kobo/download: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error encoding response for /api/kobo/download: %v", err)
}
}
func downloadFormValues(r *http.Request, body []byte) (url.Values, error) {
if err := r.ParseForm(); err != nil {
mediaType, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
if mediaType != "application/x-www-form-urlencoded" || len(body) > 10<<20 {
return nil, err
}
// Kobo sends a raw ; inside the URL value. Escape only that form field,
// then let Go validate and decode the complete form as usual.
fields := strings.Split(string(body), "&")
urlFields := 0
repaired := false
for i, field := range fields {
if !strings.HasPrefix(field, "url=") {
continue
}
urlFields++
if strings.Contains(field, ";") {
fields[i] = strings.ReplaceAll(field, ";", "%3B")
repaired = true
}
}
if urlFields != 1 || !repaired {
return nil, err
}
form, err := url.ParseQuery(strings.Join(fields, "&"))
if err != nil {
return nil, err
}
if len(form["url"]) != 1 {
return nil, errors.New("duplicate URL form field")
}
return form, nil
}
return r.Form, nil
}
func getSitesToTry(host string) []string {
var sites []string
parts := strings.Split(host, ".")
@@ -518,12 +567,12 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) {
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, "Failed to read request body", http.StatusInternalServerError)
a.Logger.Errorf("Error reading /api/kobo/send request body: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error reading /api/kobo/send request body: %v", err)
return
}
r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/send:\nMethod: %s\nURL: %s\nHeaders: %v\nBody: %s", r.Method, r.URL, r.Header, string(bodyBytes))
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/send: Method=%s", r.Method)
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
@@ -533,21 +582,21 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) {
var req models.KoboSendRequest
if err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&req); err != nil {
http.Error(w, "Invalid request body", http.StatusBadRequest)
a.Logger.Errorf("Error decoding /api/kobo/send request: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error decoding /api/kobo/send request: %v", err)
return
}
readeckToken, err := a.getReadeckToken(req.AccessToken)
if err != nil {
http.Error(w, "Invalid access token", http.StatusUnauthorized)
a.Logger.Errorf("Error authenticating token for /api/kobo/send: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error authenticating token for /api/kobo/send: %v", err)
return
}
readeckClient, err := a.newReadeckClient(readeckToken)
if err != nil {
http.Error(w, "Failed to initialize Readeck client", http.StatusInternalServerError)
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/send: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/send: %v", err)
return
}
@@ -592,7 +641,7 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) {
}
if err != nil {
a.Logger.Warnf("Error processing action '%s' in /api/kobo/send: %v, URL: %s, Params: %v", action, err, r.URL.Path, r.URL.Query())
a.Logger.Warnf("Error processing action '%s' in /api/kobo/send: %v", action, err)
actionResults[i] = false
allSucceeded = false
} else {
@@ -607,7 +656,7 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(response); err != nil {
a.Logger.Errorf("Error encoding response for /api/kobo/send: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error encoding response for /api/kobo/send: %v", err)
}
}
@@ -629,25 +678,25 @@ func (a *App) HandleConvertImage(w http.ResponseWriter, r *http.Request) {
}
resp, err := client.Get(imageURL)
if err != nil {
a.Logger.Errorf("Failed to fetch image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Failed to fetch image in /api/convert-image: %T", err)
a.returnPlaceholderImage(w, r, "Image fetch failed")
return
}
defer func() {
if err := resp.Body.Close(); err != nil {
a.Logger.Warnf("Error closing response body for image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query())
a.Logger.Warnf("Error closing response body in /api/convert-image: %v", err)
}
}()
if resp.StatusCode != http.StatusOK {
a.Logger.Warnf("Failed to fetch image %s in /api/convert-image: status %d, URL: %s, Params: %v", imageURL, resp.StatusCode, r.URL.Path, r.URL.Query())
a.Logger.Warnf("Failed to fetch image in /api/convert-image: status %d", resp.StatusCode)
a.returnPlaceholderImage(w, r, "Image not found")
return
}
img, _, err := image.Decode(resp.Body)
if err != nil {
a.Logger.Warnf("Failed to decode image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query())
a.Logger.Warnf("Failed to decode image in /api/convert-image: %v", err)
a.returnPlaceholderImage(w, r, "Image decoding failed")
return
}
@@ -659,7 +708,7 @@ func (a *App) HandleConvertImage(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/jpeg")
w.Header().Set("Cache-Control", "public, max-age=3600")
if err := jpeg.Encode(w, rgbImg, &jpeg.Options{Quality: 85}); err != nil {
a.Logger.Errorf("Failed to encode JPEG for image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Failed to encode JPEG in /api/convert-image: %v", err)
}
}
@@ -681,7 +730,7 @@ func (a *App) returnPlaceholderImage(w http.ResponseWriter, r *http.Request, mes
w.Header().Set("Content-Type", "image/jpeg")
w.Header().Set("Cache-control", "public, max-age=300")
if err := jpeg.Encode(w, img, &jpeg.Options{Quality: 85}); err != nil {
a.Logger.Errorf("Error encoding placeholder image: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
a.Logger.Errorf("Error encoding placeholder image: %v", err)
}
}
@@ -701,41 +750,23 @@ func compareURLs(url1, url2 string) (bool, error) {
return u1.Scheme == u2.Scheme && u1.Host == u2.Host && u1.Path == u2.Path, nil
}
func (a *App) getReadeckToken(deviceToken string) (string, error) {
func (a *App) getUser(deviceToken string) (config.User, error) {
for _, user := range a.Config.Users {
if user.Token == deviceToken {
return user.ReadeckAccessToken, nil
return user, nil
}
}
return "", fmt.Errorf("unauthorized device token")
return config.User{}, fmt.Errorf("unauthorized device token")
}
func (a *App) getReadeckToken(deviceToken string) (string, error) {
user, err := a.getUser(deviceToken)
if err != nil {
return "", err
}
return user.ReadeckAccessToken, nil
}
func (a *App) newReadeckClient(readeckToken string) (*readeck.Client, error) {
return readeck.NewClient(a.Config.Readeck.Host, readeckToken, a.Logger, a.ReadeckHTTPClient)
}
func (a *App) HandleDumpAndForward(w http.ResponseWriter, r *http.Request) {
a.Logger.Debugf("Dumping request from %s", r.RemoteAddr)
a.Logger.Debugf("Method: %s", r.Method)
a.Logger.Debugf("URL: %s", r.URL.String())
a.Logger.Debugf("Headers: %v", r.Header)
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, "Failed to read request body", http.StatusInternalServerError)
a.Logger.Debugf("Error reading request body: %v", err)
return
}
r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
a.Logger.Debugf("Body: %s", string(bodyBytes))
target, err := url.Parse("https://storeapi.kobo.com")
if err != nil {
a.Logger.Errorf("Error parsing target URL: %v", err)
return
}
proxy := httputil.NewSingleHostReverseProxy(target)
proxy.ServeHTTP(w, r)
}
+181 -15
View File
@@ -173,6 +173,7 @@ func TestHandleKoboGet(t *testing.T) {
expectedStatus int
expectedListSize int
expectedTotal int
expectedStatuses map[string]string
}
testCases := []koboGetTestCase{
@@ -191,6 +192,22 @@ func TestHandleKoboGet(t *testing.T) {
expectedListSize: 1, // Only the unread item
expectedTotal: 1,
},
{
name: "full sync with opted-in user still excludes archived",
reqBody: &models.KoboGetRequest{Count: "10", AccessToken: "remove-archived-device"},
mockBookmarksSync: []readeck.BookmarkSync{
{ID: "1", Type: "update"},
{ID: "2", Type: "update"},
},
mockBookmarkDetails: map[string]*readeck.Bookmark{
"1": {ID: "1", Title: "Unread"},
"2": {ID: "2", Title: "Archived", IsArchived: true},
},
expectedStatus: http.StatusOK,
expectedListSize: 1,
expectedTotal: 1,
expectedStatuses: map[string]string{"1": "0"},
},
{
name: "full sync with favorited item",
reqBody: &models.KoboGetRequest{Count: "10", AccessToken: mockDeviceToken}, // No 'Since'
@@ -236,6 +253,7 @@ func TestHandleKoboGet(t *testing.T) {
expectedStatus: http.StatusOK,
expectedListSize: 1, // The deleted status update
expectedTotal: 0,
expectedStatuses: map[string]string{"1": "2"},
},
{
name: "incremental sync with newly archived",
@@ -249,6 +267,24 @@ func TestHandleKoboGet(t *testing.T) {
expectedStatus: http.StatusOK,
expectedListSize: 1, // The full archived item
expectedTotal: 0,
expectedStatuses: map[string]string{"1": "1"},
},
{
name: "incremental sync removes archived for opted-in user",
reqBody: &models.KoboGetRequest{Since: sinceValue, AccessToken: "remove-archived-device"},
mockBookmarksSync: []readeck.BookmarkSync{
{ID: "1", Type: "update"},
{ID: "2", Type: "update"},
{ID: "3", Type: "delete"},
},
mockBookmarkDetails: map[string]*readeck.Bookmark{
"1": {ID: "1", Title: "Archived", IsArchived: true},
"2": {ID: "2", Title: "Unread", IsArchived: false},
},
expectedStatus: http.StatusOK,
expectedListSize: 3,
expectedTotal: 1,
expectedStatuses: map[string]string{"1": "2", "2": "0", "3": "2"},
},
{
name: "incremental sync with GetBookmarksSync error",
@@ -327,7 +363,10 @@ func TestHandleKoboGet(t *testing.T) {
app := NewApp(
WithConfig(&config.Config{
Users: []config.User{{Token: mockDeviceToken, ReadeckAccessToken: mockPlaintextReadeckToken}},
Users: []config.User{
{Token: mockDeviceToken, ReadeckAccessToken: mockPlaintextReadeckToken},
{Token: "remove-archived-device", ReadeckAccessToken: mockPlaintextReadeckToken, RemoveArchivedFromKobo: true},
},
Readeck: config.ConfigReadeck{Host: mockServer.URL},
}),
WithLogger(testLogger),
@@ -359,6 +398,13 @@ func TestHandleKoboGet(t *testing.T) {
t.Errorf("expected total to be %d, got %d", tc.expectedTotal, resp.Total)
}
for id, wantStatus := range tc.expectedStatuses {
item, ok := resp.List[id]
if !ok || item.Status != wantStatus {
t.Errorf("item %s status = %q (found: %v), want %q", id, item.Status, ok, wantStatus)
}
}
// Specific checks for each test case
switch tc.name {
case "full sync with favorited item":
@@ -366,16 +412,6 @@ func TestHandleKoboGet(t *testing.T) {
if item.Favorite != "1" {
t.Errorf("expected favorited item 'favorite' status to be '1', got '%s'", item.Favorite)
}
case "incremental sync with deleted":
item := resp.List["1"]
if item.Status != "2" {
t.Errorf("expected deleted item status to be '2', got '%s'", item.Status)
}
case "incremental sync with newly archived":
item := resp.List["1"]
if item.Status != "1" {
t.Errorf("expected archived item status to be '1', got '%s'", item.Status)
}
case "full sync with image item":
item := resp.List["1"]
if item.HasImage != "1" {
@@ -389,6 +425,7 @@ func TestHandleKoboGet(t *testing.T) {
})
}
}
// koboDownloadTestCase defines the structure for test cases in TestHandleKoboDownload.
type koboDownloadTestCase struct {
name string
@@ -414,6 +451,19 @@ func TestHandleKoboDownload(t *testing.T) {
},
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
},
{
name: "successful download (JSON URL with semicolon)",
reqBody: models.KoboDownloadRequest{
AccessToken: mockDeviceToken,
URL: "http://example.com/article;edition=2?page=1",
},
contentType: "application/json",
expectedStatus: http.StatusOK,
mockBookmarks: []readeck.Bookmark{
{ID: "1", Title: "Test Article", URL: "http://example.com/article;edition=2?page=1"},
},
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
},
{
name: "successful download (Form)",
reqBody: url.Values{
@@ -427,6 +477,49 @@ func TestHandleKoboDownload(t *testing.T) {
},
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
},
{
name: "successful download (encoded semicolon in form URL path)",
reqBody: url.Values{
"access_token": {mockDeviceToken},
"url": {"http://example.com/article;edition=2?page=1"},
},
contentType: "application/x-www-form-urlencoded",
expectedStatus: http.StatusOK,
mockBookmarks: []readeck.Bookmark{
{ID: "1", Title: "Test Article", URL: "http://example.com/article;edition=2?page=1"},
},
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
},
{
name: "successful download (raw semicolon in form URL path)",
reqBody: "access_token=" + mockDeviceToken + "&url=http%3A%2F%2Fexample.com%2Farticle;edition=2%3Fpage%3D1",
contentType: "application/x-www-form-urlencoded",
expectedStatus: http.StatusOK,
mockBookmarks: []readeck.Bookmark{
{ID: "1", Title: "Test Article", URL: "http://example.com/article;edition=2?page=1"},
},
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
},
{
name: "successful download (raw semicolon in form URL query)",
reqBody: "access_token=" + mockDeviceToken + "&url=http%3A%2F%2Fexample.com%2Farticle1%3Futm_source%3Dmail;utm_medium%3Dreader",
contentType: "application/x-www-form-urlencoded",
expectedStatus: http.StatusOK,
mockBookmarks: []readeck.Bookmark{
{ID: "1", Title: "Test Article", URL: "http://example.com/article1?utm_source=mail;utm_medium=reader"},
},
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
},
{
name: "successful download (captured Kobo form shape)",
reqBody: "images=0&consumer_key=fake&access_token=" + mockDeviceToken + "&refresh=0&output=json&url=https://example.com/article?kobo_probe%3D1;part%3D2",
contentType: "application/x-www-form-urlencoded",
expectedStatus: http.StatusOK,
mockBookmarks: []readeck.Bookmark{
{ID: "1", Title: "Test Article", URL: "https://example.com/article?kobo_probe=1;part=2"},
},
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
},
{
name: "missing url",
reqBody: models.KoboDownloadRequest{
@@ -436,6 +529,12 @@ func TestHandleKoboDownload(t *testing.T) {
contentType: "application/json",
expectedStatus: http.StatusBadRequest,
},
{
name: "missing access token in raw-semicolon form",
reqBody: "url=https://example.com/article;kobo_probe%3D1",
contentType: "application/x-www-form-urlencoded",
expectedStatus: http.StatusUnauthorized,
},
{
name: "invalid access token",
reqBody: models.KoboDownloadRequest{
@@ -490,8 +589,14 @@ func TestHandleKoboDownload(t *testing.T) {
}
body = bytes.NewReader(jsonBody)
case "application/x-www-form-urlencoded":
formValues := tc.reqBody.(url.Values)
body = strings.NewReader(formValues.Encode())
switch form := tc.reqBody.(type) {
case url.Values:
body = strings.NewReader(form.Encode())
case string:
body = strings.NewReader(form)
default:
t.Fatalf("unexpected form body type %T", tc.reqBody)
}
}
req := httptest.NewRequest(http.MethodPost, "/api/kobo/download", body)
@@ -525,6 +630,69 @@ func TestHandleKoboDownload(t *testing.T) {
}
}
func TestDownloadFormValuesWithSemicolons(t *testing.T) {
for _, tt := range []struct {
name, body, wantURL string
wantError bool
}{
{
name: "normally encoded URL with semicolon",
body: url.Values{"access_token": {"device"}, "url": {"https://example.com/article;edition=2?utm=a;b=c"}}.Encode(),
wantURL: "https://example.com/article;edition=2?utm=a;b=c",
},
{
name: "raw semicolon in URL path",
body: "access_token=device&url=https%3A%2F%2Fexample.com%2Farticle;edition=2%3Fpage%3D1&output=epub",
wantURL: "https://example.com/article;edition=2?page=1",
},
{
name: "Kobo form encodes equals but leaves semicolon raw",
body: "images=0&consumer_key=fake&access_token=device&refresh=0&output=json&url=https://example.com/article?kobo_probe%3D1;part%3D2",
wantURL: "https://example.com/article?kobo_probe=1;part=2",
},
{
name: "raw semicolon in URL query",
body: "url=https%3A%2F%2Fexample.com%2Farticle%3Futm_source%3Dmail;utm_medium%3Dreader%26page%3D2&access_token=device",
wantURL: "https://example.com/article?utm_source=mail;utm_medium=reader&page=2",
},
{
name: "semicolon outside URL field",
body: "access_token=device;consumer_key=other&url=https%3A%2F%2Fexample.com%2Farticle;edition=2",
wantError: true,
},
{
name: "duplicate URL field",
body: "access_token=device&url=https%3A%2F%2Fexample.com%2Fa;edition=2&url=https%3A%2F%2Fexample.com%2Fb",
wantError: true,
},
{
name: "invalid form without raw semicolon stays invalid",
body: "access_token=device&url=https%3A%2F%2Fexample.com%2Fa&output=%zz",
wantError: true,
},
{
name: "invalid percent escape",
body: "access_token=device&url=https%3A%2F%2Fexample.com%2Fa;edition=2&output=%zz",
wantError: true,
},
} {
t.Run(tt.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/api/kobo/download", strings.NewReader(tt.body))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
form, err := downloadFormValues(req, []byte(tt.body))
if tt.wantError {
if err == nil {
t.Fatalf("expected a form parsing error, got %v", form)
}
return
}
if err != nil || form.Get("url") != tt.wantURL || form.Get("access_token") != "device" {
t.Errorf("URL = %q, token = %q, err = %v; want URL %q and token device", form.Get("url"), form.Get("access_token"), err, tt.wantURL)
}
})
}
}
// koboSendTestCase defines the structure for test cases in TestHandleKoboSend.
type koboSendTestCase struct {
name string
@@ -842,5 +1010,3 @@ func TestHandleConvertImage(t *testing.T) {
}
})
}
+38
View File
@@ -0,0 +1,38 @@
package app
import (
"net/http"
"net/url"
)
func (a *App) HandleBookSync(w http.ResponseWriter, r *http.Request) {
token := r.PathValue("deviceToken")
upstream, known := a.bookSyncUpstream(token)
if !known {
http.NotFound(w, r)
return
}
target, err := url.Parse(upstream)
if err != nil {
a.Logger.Errorf("Invalid configured book sync upstream: %T", err)
http.Error(w, "Bad gateway", http.StatusBadGateway)
return
}
http.StripPrefix("/booksync/"+token, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
a.serveKoboProxy(w, r, target, token)
})).ServeHTTP(w, r)
}
func (a *App) bookSyncUpstream(token string) (string, bool) {
for _, user := range a.Config.Users {
if user.Token == token {
if user.BookSyncURL != "" {
return user.BookSyncURL, true
}
return "https://" + storeAPIHost, true
}
}
return "", false
}
+183
View File
@@ -0,0 +1,183 @@
package app
import (
"bytes"
"compress/gzip"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httputil"
"net/url"
"strconv"
"strings"
"time"
)
const (
storeAPIHost = "storeapi.kobo.com"
maxInitializationBody = 8 << 20
)
var koboTransport = func() *http.Transport {
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.ResponseHeaderTimeout = 20 * time.Second
return transport
}()
func (a *App) HandleStoreProxy(w http.ResponseWriter, r *http.Request) {
a.serveKoboProxy(w, r, &url.URL{Scheme: "https", Host: storeAPIHost}, "")
}
// serveKoboProxy expects the public route prefix to have been removed from r.URL.
func (a *App) serveKoboProxy(w http.ResponseWriter, r *http.Request, target *url.URL, deviceToken string) {
path := r.URL.Path
if path == "" {
path = "/"
}
proxy := a.newKoboProxy(target, path)
if deviceToken != "" {
originalDirector := proxy.Director
proxy.Director = func(req *http.Request) {
originalDirector(req)
// Book services can build absolute URLs from forwarded headers.
// Use the upstream origin, not the public ingress host.
req.Header.Del("Forwarded")
req.Header.Del("X-Forwarded-Port")
req.Header.Del("X-Forwarded-Prefix")
req.Header.Set("X-Forwarded-Host", target.Host)
req.Header.Set("X-Forwarded-Proto", target.Scheme)
}
}
if path == "/v1/initialization" {
origin, err := a.publicOrigin(r)
if err != nil {
http.Error(w, "Invalid public origin", http.StatusBadGateway)
return
}
configureInitializationProxy(proxy, origin+"/instapaper-proxy/instapaper")
}
proxy.ServeHTTP(w, r)
}
func (a *App) newKoboProxy(target *url.URL, path string) *httputil.ReverseProxy {
proxy := httputil.NewSingleHostReverseProxy(target)
originalDirector := proxy.Director
proxy.Director = func(req *http.Request) {
req.URL.Path = path
originalDirector(req)
req.Host = target.Host
}
if a.ProxyTransport != nil {
proxy.Transport = a.ProxyTransport
} else {
proxy.Transport = koboTransport
}
proxy.ErrorHandler = func(w http.ResponseWriter, _ *http.Request, err error) {
a.Logger.Errorf("Kobo upstream proxy failed: %T", err)
http.Error(w, "Bad gateway", http.StatusBadGateway)
}
return proxy
}
func configureInitializationProxy(proxy *httputil.ReverseProxy, instapaperURL string) {
originalDirector := proxy.Director
proxy.Director = func(req *http.Request) {
originalDirector(req)
// Request a complete document so its body can be rewritten.
req.Header.Set("Accept-Encoding", "identity")
req.Header.Del("If-None-Match")
req.Header.Del("If-Modified-Since")
req.Header.Del("Range")
req.Header.Del("If-Range")
}
proxy.ModifyResponse = func(resp *http.Response) error {
if resp.StatusCode == http.StatusPartialContent {
return errors.New("partial initialization response")
}
if resp.StatusCode != http.StatusOK {
return nil
}
return rewriteInitialization(resp, instapaperURL)
}
}
func (a *App) publicOrigin(r *http.Request) (string, error) {
if a.Config.PublicURL != "" {
return strings.TrimRight(a.Config.PublicURL, "/"), nil
}
scheme := r.Header.Get("X-Forwarded-Proto")
if scheme == "" {
scheme = "http"
if r.TLS != nil {
scheme = "https"
}
}
host := r.Header.Get("X-Forwarded-Host")
if host == "" {
host = r.Host
}
if scheme != "http" && scheme != "https" || host == "" || strings.ContainsAny(host, "/@?#, \t\r\n\"\\") {
return "", errors.New("invalid forwarded origin")
}
parsed, err := url.Parse("//" + host)
if err != nil || parsed.Hostname() == "" || parsed.Host != host {
return "", errors.New("invalid forwarded host")
}
return scheme + "://" + host, nil
}
func rewriteInitialization(resp *http.Response, instapaperURL string) error {
originalBody := resp.Body
defer func() { _ = originalBody.Close() }()
var reader io.Reader = originalBody
compressed := resp.Header.Get("Content-Encoding") == "gzip"
if compressed {
gz, err := gzip.NewReader(originalBody)
if err != nil {
return err
}
defer func() { _ = gz.Close() }()
reader = gz
} else if encoding := resp.Header.Get("Content-Encoding"); encoding != "" && encoding != "identity" {
return fmt.Errorf("unsupported initialization encoding: %s", encoding)
}
body, err := io.ReadAll(io.LimitReader(reader, maxInitializationBody+1))
if err != nil {
return err
}
if len(body) > maxInitializationBody {
return errors.New("initialization response too large")
}
encodedURL, err := json.Marshal(instapaperURL)
if err != nil {
return err
}
body = bytes.ReplaceAll(body, []byte("https://www.instapaper.com"), encodedURL[1:len(encodedURL)-1])
if compressed {
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
if _, err := gw.Write(body); err != nil {
return err
}
if err := gw.Close(); err != nil {
return err
}
body = buf.Bytes()
} else {
resp.Header.Del("Content-Encoding")
}
resp.Body = io.NopCloser(bytes.NewReader(body))
resp.ContentLength = int64(len(body))
resp.Header.Set("Content-Length", strconv.Itoa(len(body)))
for _, key := range []string{"ETag", "Last-Modified", "Content-MD5", "Digest", "Content-Range", "Accept-Ranges"} {
resp.Header.Del(key)
}
resp.Header.Set("Cache-Control", "private, no-store")
return nil
}
+313
View File
@@ -0,0 +1,313 @@
package app
import (
"bytes"
"compress/gzip"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
"readeckobo/internal/config"
)
func storeHandler(a *App) http.Handler {
return http.StripPrefix("/instapaper-proxy/storeapi", http.HandlerFunc(a.HandleStoreProxy))
}
func TestHandleStoreProxy(t *testing.T) {
t.Run("forwards requests to the fixed store origin", func(t *testing.T) {
body := "book sync response"
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
RoundTripFunc: func(req *http.Request) (*http.Response, error) {
if req.URL.Scheme != "https" || req.URL.Host != storeAPIHost || req.Host != storeAPIHost || req.URL.Path != "/v1/library/sync" || req.URL.RawQuery != "page=2" {
t.Errorf("unexpected upstream request: URL=%s Host=%q", req.URL, req.Host)
}
data, err := io.ReadAll(req.Body)
if err != nil || string(data) != "payload" {
t.Errorf("upstream body = %q, err = %v", data, err)
}
if req.Header.Get("X-Test") != "forwarded" {
t.Errorf("X-Test = %q", req.Header.Get("X-Test"))
}
return &http.Response{StatusCode: http.StatusAccepted, Header: http.Header{"Content-Type": {"application/octet-stream"}}, Body: io.NopCloser(strings.NewReader(body))}, nil
},
}))
req := httptest.NewRequest(http.MethodPost, "/instapaper-proxy/storeapi/v1/library/sync?page=2", strings.NewReader("payload"))
req.Header.Set("X-Test", "forwarded")
rr := httptest.NewRecorder()
storeHandler(a).ServeHTTP(rr, req)
if rr.Code != http.StatusAccepted || rr.Body.String() != body {
t.Errorf("status = %d, body length = %d; want 202 and %d", rr.Code, rr.Body.Len(), len(body))
}
})
t.Run("streams an ordinary response before upstream completion", func(t *testing.T) {
reader, writer := io.Pipe()
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
RoundTripFunc: func(*http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"application/octet-stream"}},
Body: reader,
ContentLength: -1,
}, nil
},
}))
server := httptest.NewServer(storeHandler(a))
defer server.Close()
defer func() { _ = writer.Close() }()
firstChunk := bytes.Repeat([]byte("x"), 32*1024)
result := make(chan error, 1)
go func() {
resp, err := server.Client().Get(server.URL + "/instapaper-proxy/storeapi/v1/library/sync")
if err != nil {
result <- err
return
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
result <- errors.New("unexpected status")
return
}
data := make([]byte, len(firstChunk))
_, err = io.ReadFull(resp.Body, data)
if err == nil && !bytes.Equal(data, firstChunk) {
err = errors.New("unexpected bytes in first chunk")
}
result <- err
}()
go func() { _, _ = writer.Write(firstChunk) }()
select {
case err := <-result:
if err != nil {
t.Fatal(err)
}
case <-time.After(3 * time.Second):
t.Fatal("proxy did not deliver bytes before upstream finished")
}
})
for _, tt := range []struct {
name string
gzip bool
publicURL string
wantOrigin string
}{
{name: "plaintext uses configured public origin", publicURL: "https://reader.example.com/", wantOrigin: "https://reader.example.com"},
{name: "gzipped response uses trusted forwarded origin", gzip: true, wantOrigin: "https://reader.example.com"},
} {
t.Run(tt.name, func(t *testing.T) {
upstreamBody := []byte(`{"instapaper":"https://www.instapaper.com"}`)
responseHeader := http.Header{
"Content-Type": {"application/json"},
"Etag": {`"old-tag"`},
"Last-Modified": {"Tue, 01 Jan 2019 00:00:00 GMT"},
}
if tt.gzip {
var buf bytes.Buffer
writer := gzip.NewWriter(&buf)
if _, err := writer.Write(upstreamBody); err != nil {
t.Fatal(err)
}
if err := writer.Close(); err != nil {
t.Fatal(err)
}
upstreamBody = buf.Bytes()
responseHeader.Set("Content-Encoding", "gzip")
}
responseHeader.Set("Content-Length", strconv.Itoa(len(upstreamBody)))
a := NewApp(WithConfig(&config.Config{PublicURL: tt.publicURL}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
RoundTripFunc: func(req *http.Request) (*http.Response, error) {
if req.URL.Path != "/v1/initialization" || req.URL.RawQuery != "locale=en" || req.Header.Get("Accept-Encoding") != "identity" || req.Header.Get("If-None-Match") != "" || req.Header.Get("Range") != "" || req.Header.Get("If-Range") != "" {
t.Errorf("unexpected initialization request: %s, headers: %v", req.URL, req.Header)
}
return &http.Response{StatusCode: http.StatusOK, Header: responseHeader, Body: io.NopCloser(bytes.NewReader(upstreamBody))}, nil
},
}))
req := httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization?locale=en", nil)
req.Host = "untrusted.example.com"
forwardedHost := "reader.example.com"
if tt.publicURL != "" {
forwardedHost = "ignored.example.com"
}
req.Header.Set("X-Forwarded-Host", forwardedHost)
req.Header.Set("X-Forwarded-Proto", "https")
req.Header.Set("If-None-Match", `"old-tag"`)
req.Header.Set("Range", "bytes=0-50")
req.Header.Set("If-Range", `"old-tag"`)
rr := httptest.NewRecorder()
storeHandler(a).ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
if got := rr.Header().Get("Content-Length"); got != strconv.Itoa(rr.Body.Len()) {
t.Errorf("Content-Length = %q, body size = %d", got, rr.Body.Len())
}
if rr.Header().Get("ETag") != "" || rr.Header().Get("Last-Modified") != "" || rr.Header().Get("Cache-Control") != "private, no-store" {
t.Errorf("stale response headers: %v", rr.Header())
}
var result []byte
if tt.gzip {
if rr.Header().Get("Content-Encoding") != "gzip" {
t.Errorf("Content-Encoding = %q", rr.Header().Get("Content-Encoding"))
}
reader, err := gzip.NewReader(rr.Body)
if err != nil {
t.Fatal(err)
}
result, err = io.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
_ = reader.Close()
} else {
if rr.Header().Get("Content-Encoding") != "" {
t.Errorf("Content-Encoding = %q", rr.Header().Get("Content-Encoding"))
}
result = rr.Body.Bytes()
}
want := `{"instapaper":"` + tt.wantOrigin + `/instapaper-proxy/instapaper"}`
if string(result) != want {
t.Errorf("body = %q, want %q", result, want)
}
})
}
t.Run("preserves escaped path and query", func(t *testing.T) {
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
RoundTripFunc: func(req *http.Request) (*http.Response, error) {
if req.URL.Path != "/v1//books" || req.URL.EscapedPath() != "/v1/%2Fbooks" || req.URL.RawQuery != "a=1%3B2" {
t.Errorf("upstream URL = %q, RawPath = %q", req.URL, req.URL.RawPath)
}
return &http.Response{StatusCode: http.StatusNoContent, Header: http.Header{}, Body: http.NoBody}, nil
},
}))
rr := httptest.NewRecorder()
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/%2Fbooks?a=1%3B2", nil))
if rr.Code != http.StatusNoContent {
t.Errorf("status = %d, want 204", rr.Code)
}
})
for _, tt := range []struct {
name, encoding, body string
}{
{name: "rejects invalid gzip initialization", encoding: "gzip", body: "not gzip"},
{name: "rejects unsupported initialization encoding", encoding: "br", body: "compressed"},
{name: "limits initialization size", body: strings.Repeat("x", maxInitializationBody+1)},
} {
t.Run(tt.name, func(t *testing.T) {
a := NewApp(WithConfig(&config.Config{PublicURL: "https://reader.example.com"}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
RoundTripFunc: func(*http.Request) (*http.Response, error) {
header := http.Header{}
header.Set("Content-Encoding", tt.encoding)
return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(strings.NewReader(tt.body))}, nil
},
}))
rr := httptest.NewRecorder()
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil))
if rr.Code != http.StatusBadGateway {
t.Errorf("status = %d, want 502", rr.Code)
}
})
}
t.Run("partial initialization is rejected", func(t *testing.T) {
a := NewApp(WithConfig(&config.Config{PublicURL: "https://reader.example.com"}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
RoundTripFunc: func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusPartialContent, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("partial"))}, nil
},
}))
rr := httptest.NewRecorder()
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil))
if rr.Code != http.StatusBadGateway {
t.Errorf("status = %d, want 502", rr.Code)
}
})
t.Run("non-OK initialization passes through without rewriting", func(t *testing.T) {
body := `{"url":"https://www.instapaper.com"}`
a := NewApp(WithConfig(&config.Config{PublicURL: "https://reader.example.com"}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
RoundTripFunc: func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusNotFound, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body))}, nil
},
}))
rr := httptest.NewRecorder()
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil))
if rr.Code != http.StatusNotFound || rr.Body.String() != body {
t.Errorf("status = %d, body = %q; want 404 and unchanged body", rr.Code, rr.Body.String())
}
})
t.Run("encodes rewritten URLs as valid JSON strings", func(t *testing.T) {
urlWithQuote := `https://reader.example.com/path?value="quoted"`
resp := &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"instapaper":"https://www.instapaper.com"}`))}
if err := rewriteInitialization(resp, urlWithQuote); err != nil {
t.Fatal(err)
}
var decoded struct {
Instapaper string `json:"instapaper"`
}
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
t.Fatalf("rewritten response is invalid JSON: %v", err)
}
if decoded.Instapaper != urlWithQuote {
t.Errorf("instapaper URL = %q, want %q", decoded.Instapaper, urlWithQuote)
}
})
t.Run("rejects malformed forwarded origin", func(t *testing.T) {
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger))
req := httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil)
req.Header.Set("X-Forwarded-Proto", "javascript")
rr := httptest.NewRecorder()
storeHandler(a).ServeHTTP(rr, req)
if rr.Code != http.StatusBadGateway {
t.Errorf("status = %d, want 502", rr.Code)
}
})
t.Run("rejects quoted forwarded host", func(t *testing.T) {
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger))
req := httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil)
req.Header.Set("X-Forwarded-Host", `reader.example.com"injected`)
rr := httptest.NewRecorder()
storeHandler(a).ServeHTTP(rr, req)
if rr.Code != http.StatusBadGateway {
t.Errorf("status = %d, want 502", rr.Code)
}
})
t.Run("returns 502 on upstream failure", func(t *testing.T) {
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
RoundTripFunc: func(*http.Request) (*http.Response, error) { return nil, errors.New("upstream unavailable") },
}))
rr := httptest.NewRecorder()
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/library/sync", nil))
if rr.Code != http.StatusBadGateway {
t.Errorf("status = %d, want 502", rr.Code)
}
})
t.Run("does not rewrite other initialization-like paths", func(t *testing.T) {
body := `{"url":"https://www.instapaper.com"}`
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
RoundTripFunc: func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body))}, nil
},
}))
rr := httptest.NewRecorder()
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v2/initialization", nil))
if rr.Body.String() != body {
t.Errorf("body = %q, want %q", rr.Body.String(), body)
}
})
}
+61
View File
@@ -0,0 +1,61 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestBookSyncConfiguration(t *testing.T) {
tests := []struct {
name string
extra string
userExtra string
wantErr string
wantBookSyncURL string
wantPublicURL string
wantRemove bool
}{
{name: "no optional settings"},
{name: "old disabled switch is ignored", extra: "book_sync: false\n"},
{name: "old enabled switch is ignored", extra: "book_sync: true\n"},
{name: "custom sync endpoint with path", userExtra: " book_sync_url: https://books.example.com/api/kobo/secret\n", wantBookSyncURL: "https://books.example.com/api/kobo/secret"},
{name: "http sync endpoint is allowed", userExtra: " book_sync_url: http://localhost:8081/kobo\n", wantBookSyncURL: "http://localhost:8081/kobo"},
{name: "public URL and archived removal", extra: "public_url: https://reader.example.com/\n", userExtra: " remove_archived_from_kobo: true\n", wantPublicURL: "https://reader.example.com/", wantRemove: true},
{name: "invalid sync scheme", userExtra: " book_sync_url: ftp://books.example.com\n", wantErr: "BookSyncURL"},
{name: "relative sync URL", userExtra: " book_sync_url: /api/kobo/device\n", wantErr: "BookSyncURL"},
{name: "sync URL missing host", userExtra: " book_sync_url: https:/kobo\n", wantErr: "BookSyncURL"},
{name: "invalid sync port", userExtra: " book_sync_url: https://books.example.com:bad/kobo\n", wantErr: "BookSyncURL"},
{name: "invalid public scheme", extra: "public_url: ftp://reader.example.com\n", wantErr: "PublicURL"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
data := "readeck:\n host: https://readeck.example.com\n" + tt.extra + "users:\n - token: device\n readeck_access_token: readeck\n" + tt.userExtra
path := filepath.Join(t.TempDir(), "config.yaml")
if err := os.WriteFile(path, []byte(data), 0600); err != nil {
t.Fatal(err)
}
cfg, err := Load(path)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Load() error = %v, want error mentioning %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatal(err)
}
if got := cfg.Users[0].BookSyncURL; got != tt.wantBookSyncURL {
t.Errorf("BookSyncURL = %q, want %q", got, tt.wantBookSyncURL)
}
if cfg.PublicURL != tt.wantPublicURL {
t.Errorf("PublicURL = %q, want %q", cfg.PublicURL, tt.wantPublicURL)
}
if cfg.Users[0].RemoveArchivedFromKobo != tt.wantRemove {
t.Errorf("RemoveArchivedFromKobo = %v, want %v", cfg.Users[0].RemoveArchivedFromKobo, tt.wantRemove)
}
})
}
}
+9 -6
View File
@@ -12,8 +12,10 @@ import (
)
type User struct {
Token string `koanf:"token" validate:"required"`
ReadeckAccessToken string `koanf:"readeck_access_token" validate:"required"`
Token string `koanf:"token" validate:"required"`
ReadeckAccessToken string `koanf:"readeck_access_token" validate:"required"`
BookSyncURL string `koanf:"book_sync_url" validate:"omitempty,http_url"`
RemoveArchivedFromKobo bool `koanf:"remove_archived_from_kobo"`
}
type ConfigReadeck struct {
@@ -21,12 +23,13 @@ type ConfigReadeck struct {
}
type Config struct {
Readeck ConfigReadeck `koanf:"readeck"`
Server struct {
Readeck ConfigReadeck `koanf:"readeck"`
Server struct {
Port int `koanf:"port" validate:"min=1,max=65535"`
} `koanf:"server"`
Users []User `koanf:"users" validate:"required,min=1,dive"`
LogLevel string `koanf:"log_level" validate:"oneof=error warn info debug"`
Users []User `koanf:"users" validate:"required,min=1,dive"`
LogLevel string `koanf:"log_level" validate:"oneof=error warn info debug"`
PublicURL string `koanf:"public_url" validate:"omitempty,http_url"`
}
func (c *Config) Validate() error {
+2 -17
View File
@@ -9,7 +9,6 @@ import (
"mime"
"mime/multipart"
"net/http"
"net/http/httputil" // Added import
"net/url"
"strconv"
"strings"
@@ -129,26 +128,12 @@ func (c *Client) doRequestRaw(ctx context.Context, method, path string, queryPar
req.Header.Set("Content-Type", "application/json") // Ensure Content-Type is set for requests with a body
}
// Log the outgoing request for debugging
dump, err := httputil.DumpRequestOut(req, true)
if err != nil {
c.Logger.Errorf("Failed to dump outgoing request: %v", err)
} else {
c.Logger.Debugf("Outgoing Readeck API Request:\n%s", dump)
}
start := time.Now()
resp, err := c.HTTPClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to execute request: %w", err)
}
// Log the incoming response for debugging
// dumpResp, err := httputil.DumpResponse(resp, true)
// if err != nil {
// c.Logger.Errorf("Failed to dump incoming response: %v", err)
// } else {
// c.Logger.Debugf("Incoming Readeck API Response:\n%s", dumpResp)
// }
c.Logger.Debugf("Readeck %s %s: HTTP %d in %s", req.Method, req.URL.Path, resp.StatusCode, time.Since(start))
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
defer func() { _ = resp.Body.Close() }()
+52
View File
@@ -1,10 +1,14 @@
package readeck
import (
"bytes"
"context"
"encoding/json"
"log"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
@@ -224,6 +228,54 @@ func TestCreateBookmark(t *testing.T) {
}
}
func TestRawRequestDebugLogOmitsSecrets(t *testing.T) {
const token = "private-readeck-token"
const query = "private-query-value"
const body = "private-body-value"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer "+token {
t.Errorf("Authorization header = %q, want bearer token", got)
}
if got := r.URL.Query().Get("q"); got != query {
t.Errorf("query value = %q, want %q", got, query)
}
var data map[string]string
if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
t.Errorf("decode request body: %v", err)
} else if data["data"] != body {
t.Errorf("body data = %q, want %q", data["data"], body)
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
var output bytes.Buffer
previousOutput := log.Writer()
log.SetOutput(&output)
defer log.SetOutput(previousOutput)
client, err := NewClient(server.URL, token, logger.New(logger.DEBUG), nil)
if err != nil {
t.Fatal(err)
}
resp, err := client.doRequestRaw(context.Background(), http.MethodPost, "/api/bookmarks/sync", url.Values{"q": {query}}, map[string]string{"data": body})
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
got := output.String()
if !strings.Contains(got, "Readeck POST /api/bookmarks/sync: HTTP 200 in ") {
t.Errorf("missing request summary in debug log: %q", got)
}
for _, secret := range []string{token, query, body, "Authorization"} {
if strings.Contains(got, secret) {
t.Errorf("debug log contains private request data: %q", secret)
}
}
}
func TestGetBookmarksWithIsArchived(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/bookmarks" {
+316
View File
@@ -0,0 +1,316 @@
package webserver
import (
"bytes"
"compress/gzip"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
"readeckobo/internal/app"
"readeckobo/internal/config"
"readeckobo/internal/logger"
)
func TestBookSyncRoutes(t *testing.T) {
for _, tt := range []struct {
name, path, scheme, host, upstreamPath, escapedPath, query string
wantStatus int
}{
{"custom endpoint with path prefix", "/booksync/books/v1/library/sync?page=2", "https", "books.example.com", "/api/kobo/device/v1/library/sync", "/api/kobo/device/v1/library/sync", "page=2", http.StatusAccepted},
{"custom HTTP endpoint", "/booksync/httpbook/v1/library/sync", "http", "localhost:8081", "/kobo/v1/library/sync", "/kobo/v1/library/sync", "", http.StatusAccepted},
{"escaped suffix", "/booksync/books/v1/%2Fbooks?a=1%3B2", "https", "books.example.com", "/api/kobo/device/v1//books", "/api/kobo/device/v1/%2Fbooks", "a=1%3B2", http.StatusAccepted},
{"custom endpoint root", "/booksync/books/", "https", "books.example.com", "/api/kobo/device/", "/api/kobo/device/", "", http.StatusAccepted},
{"known token uses Kobo Store", "/booksync/store/v1/library/sync?page=3", "https", "storeapi.kobo.com", "/v1/library/sync", "/v1/library/sync", "page=3", http.StatusAccepted},
{"unknown token", "/booksync/unknown/v1/library/sync", "", "", "", "", "", http.StatusNotFound},
{"partial token does not match", "/booksync/book/v1/library/sync", "", "", "", "", "", http.StatusNotFound},
} {
t.Run(tt.name, func(t *testing.T) {
var calls int
application := app.NewApp(
app.WithConfig(&config.Config{Users: []config.User{
{Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"},
{Token: "httpbook", BookSyncURL: "http://localhost:8081/kobo"},
{Token: "store"},
}}),
app.WithLogger(logger.New(logger.ERROR)),
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
calls++
if req.URL.Scheme != tt.scheme || req.URL.Host != tt.host || req.Host != tt.host || req.URL.Path != tt.upstreamPath || req.URL.EscapedPath() != tt.escapedPath || req.URL.RawQuery != tt.query {
t.Errorf("upstream URL = %s, Host = %q; want %s://%s, path %q, escaped %q, query %q", req.URL, req.Host, tt.scheme, tt.host, tt.upstreamPath, tt.escapedPath, tt.query)
}
if req.Method != http.MethodPost || req.Header.Get("X-Test") != "forwarded" {
t.Errorf("upstream method = %s, X-Test = %q", req.Method, req.Header.Get("X-Test"))
}
body, err := io.ReadAll(req.Body)
if err != nil || string(body) != "payload" {
t.Errorf("upstream body = %q, err = %v", body, err)
}
return &http.Response{StatusCode: http.StatusAccepted, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("book response"))}, nil
})),
)
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, tt.path, strings.NewReader("payload"))
req.Header.Set("X-Test", "forwarded")
NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, req)
if rr.Code != tt.wantStatus {
t.Errorf("status = %d, want %d", rr.Code, tt.wantStatus)
}
if tt.host == "" {
if calls != 0 {
t.Errorf("unknown token reached upstream (%d calls)", calls)
}
} else if calls != 1 || rr.Body.String() != "book response" {
t.Errorf("upstream calls = %d, body = %q; want one call and book response", calls, rr.Body.String())
}
})
}
}
func TestBookSyncDoesNotRetryConfiguredService(t *testing.T) {
var calls int
application := app.NewApp(
app.WithConfig(&config.Config{Users: []config.User{{Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"}}}),
app.WithLogger(logger.New(logger.ERROR)),
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
calls++
if req.URL.Host != "books.example.com" {
t.Errorf("upstream host = %q, want books.example.com", req.URL.Host)
}
return nil, errors.New("book service unavailable")
})),
)
rr := httptest.NewRecorder()
NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/booksync/books/v1/library/sync", nil))
if rr.Code != http.StatusBadGateway || calls != 1 {
t.Errorf("status = %d, upstream calls = %d; want 502 and one call", rr.Code, calls)
}
}
func TestBookSyncInitialization(t *testing.T) {
for _, tt := range []struct {
name, token, upstreamHost, upstreamPath string
gzip bool
}{
{"custom plaintext", "books", "books.example.com", "/api/kobo/device/v1/initialization", false},
{"custom gzip", "books", "books.example.com", "/api/kobo/device/v1/initialization", true},
{"store fallback", "store", "storeapi.kobo.com", "/v1/initialization", false},
} {
t.Run(tt.name, func(t *testing.T) {
body := []byte(`{"instapaper":"https://www.instapaper.com"}`)
header := http.Header{"Content-Type": {"application/json"}, "Etag": {`"old"`}}
if tt.gzip {
var b bytes.Buffer
writer := gzip.NewWriter(&b)
if _, err := writer.Write(body); err != nil {
t.Fatal(err)
}
if err := writer.Close(); err != nil {
t.Fatal(err)
}
body = b.Bytes()
header.Set("Content-Encoding", "gzip")
}
application := app.NewApp(
app.WithConfig(&config.Config{PublicURL: "https://reader.example.com", Users: []config.User{
{Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"}, {Token: "store"},
}}),
app.WithLogger(logger.New(logger.ERROR)),
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
if req.URL.Host != tt.upstreamHost || req.URL.Path != tt.upstreamPath || req.URL.RawQuery != "locale=en" || req.Header.Get("Accept-Encoding") != "identity" || req.Header.Get("Range") != "" || req.Header.Get("If-None-Match") != "" {
t.Errorf("unexpected initialization upstream: URL=%s, headers=%v", req.URL, req.Header)
}
return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(bytes.NewReader(body))}, nil
})),
)
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/booksync/"+tt.token+"/v1/initialization?locale=en", nil)
req.Header.Set("If-None-Match", `"old"`)
req.Header.Set("Range", "bytes=0-10")
NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, req)
if rr.Code != http.StatusOK || rr.Header().Get("ETag") != "" {
t.Fatalf("status = %d, headers = %v", rr.Code, rr.Header())
}
if got := rr.Header().Get("Content-Length"); got != strconv.Itoa(rr.Body.Len()) {
t.Errorf("Content-Length = %q, body size = %d", got, rr.Body.Len())
}
var result []byte
var err error
if tt.gzip {
if rr.Header().Get("Content-Encoding") != "gzip" {
t.Errorf("Content-Encoding = %q, want gzip", rr.Header().Get("Content-Encoding"))
}
reader, gzipErr := gzip.NewReader(rr.Body)
if gzipErr != nil {
t.Fatal(gzipErr)
}
result, err = io.ReadAll(reader)
_ = reader.Close()
} else {
result = rr.Body.Bytes()
}
if err != nil || string(result) != `{"instapaper":"https://reader.example.com/instapaper-proxy/instapaper"}` {
t.Errorf("rewritten body = %q, err = %v", result, err)
}
})
}
}
func TestBookSyncInitializationPreservesBookResourcesPerUser(t *testing.T) {
for _, tt := range []struct{ token, host, key string }{
{"alice", "books-alice.example.com", "alice-key"},
{"bob", "books-bob.example.com", "bob-key"},
} {
t.Run(tt.token, func(t *testing.T) {
upstream := "https://" + tt.host + "/kobo/" + tt.key
body := `{"Resources":{"instapaper_env_url":"https://www.instapaper.com/api/kobo",` +
`"image_host":"https://` + tt.host + `",` +
`"image_url_template":"` + upstream + `/v1/books/{ImageId}/thumbnail/{Width}/{Height}/false/image.jpg",` +
`"image_url_quality_template":"` + upstream + `/v1/books/{ImageId}/thumbnail/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg",` +
`"other_resource":"https://cdn.kobo.com/book.jpg"}}`
application := app.NewApp(
app.WithConfig(&config.Config{PublicURL: "https://reader.example.com", Users: []config.User{
{Token: "alice", BookSyncURL: "https://books-alice.example.com/kobo/alice-key"},
{Token: "bob", BookSyncURL: "https://books-bob.example.com/kobo/bob-key"},
}}),
app.WithLogger(logger.New(logger.ERROR)),
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
if req.URL.Host != tt.host || req.URL.Path != "/kobo/"+tt.key+"/v1/initialization" {
t.Errorf("upstream URL = %s, want %s/v1/initialization", req.URL, upstream)
}
if req.Header.Get("X-Forwarded-Host") != tt.host || req.Header.Get("X-Forwarded-Proto") != "https" {
t.Errorf("forwarded host/proto = %q/%q; want %s/https", req.Header.Get("X-Forwarded-Host"), req.Header.Get("X-Forwarded-Proto"), tt.host)
}
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}}, Body: io.NopCloser(strings.NewReader(body))}, nil
})),
)
req := httptest.NewRequest(http.MethodGet, "/booksync/"+tt.token+"/v1/initialization", nil)
req.Header.Set("X-Forwarded-Host", "untrusted.example.com")
req.Header.Set("X-Forwarded-Proto", "https")
req.Header.Set("X-Forwarded-Prefix", "/wrong")
rr := httptest.NewRecorder()
NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
var result struct {
Resources map[string]string
}
if err := json.Unmarshal(rr.Body.Bytes(), &result); err != nil {
t.Fatal(err)
}
for key, want := range map[string]string{
"instapaper_env_url": "https://reader.example.com/instapaper-proxy/instapaper/api/kobo",
"image_host": "https://" + tt.host,
"image_url_template": upstream + "/v1/books/{ImageId}/thumbnail/{Width}/{Height}/false/image.jpg",
"image_url_quality_template": upstream + "/v1/books/{ImageId}/thumbnail/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg",
"other_resource": "https://cdn.kobo.com/book.jpg",
} {
if got := result.Resources[key]; got != want {
t.Errorf("Resources[%s] = %q, want %q", key, got, want)
}
}
if !strings.Contains(rr.Body.String(), "/kobo/"+tt.key+"/v1/books/") {
t.Error("book service cover URL was not preserved")
}
})
}
}
func TestBookSyncLeavesOtherResponsesUntouched(t *testing.T) {
for _, tt := range []struct{ token, host, prefix string }{
{"alice", "books-alice.example.com", "/kobo/alice-key"},
{"bob", "books-bob.example.com", "/kobo/bob-key"},
{"store", "storeapi.kobo.com", ""},
} {
t.Run(tt.token, func(t *testing.T) {
link := "https://" + tt.host + tt.prefix + "/v1/books/book-1/file/epub?convert_kepub=true"
metadata := `{"EntitlementId":"book-1","CoverImageId":"cover-1","DownloadUrls":[{"Format":"KEPUB","Url":"` + link + `"}]}`
application := app.NewApp(
app.WithConfig(&config.Config{PublicURL: "https://reader.example.com", Users: []config.User{
{Token: "alice", BookSyncURL: "https://books-alice.example.com/kobo/alice-key"},
{Token: "bob", BookSyncURL: "https://books-bob.example.com/kobo/bob-key"},
{Token: "store"},
}}),
app.WithLogger(logger.New(logger.ERROR)),
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
if req.URL.Host != tt.host || req.Header.Get("If-None-Match") != `"old"` || req.Header.Get("Range") != "bytes=0-10" {
t.Errorf("upstream URL = %s, headers = %v", req.URL, req.Header)
}
if !strings.HasPrefix(req.URL.Path, tt.prefix+"/v1/library/") {
t.Errorf("upstream path = %q, want prefix %s/v1/library/", req.URL.Path, tt.prefix)
}
body := "[" + metadata + "]"
if strings.HasSuffix(req.URL.Path, "/sync") {
body = `[{"NewEntitlement":{"BookMetadata":` + metadata + `}}]`
}
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}, "Etag": {`"old"`}}, Body: io.NopCloser(strings.NewReader(body))}, nil
})),
)
handler := NewHandler(application, logger.New(logger.ERROR))
for _, path := range []string{"/v1/library/sync", "/v1/library/book-1/metadata"} {
req := httptest.NewRequest(http.MethodGet, "/booksync/"+tt.token+path, nil)
req.Header.Set("If-None-Match", `"old"`)
req.Header.Set("Range", "bytes=0-10")
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
wantBody := "[" + metadata + "]"
if path == "/v1/library/sync" {
wantBody = `[{"NewEntitlement":{"BookMetadata":` + metadata + `}}]`
}
if rr.Code != http.StatusOK || rr.Header().Get("Etag") != `"old"` || rr.Body.String() != wantBody {
t.Errorf("%s: status = %d, headers = %v, body = %s; want %s", path, rr.Code, rr.Header(), rr.Body.String(), wantBody)
}
}
})
}
}
func TestBookSyncStreamsDownload(t *testing.T) {
reader, writer := io.Pipe()
application := app.NewApp(
app.WithConfig(&config.Config{Users: []config.User{{Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"}}}),
app.WithLogger(logger.New(logger.ERROR)),
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
if req.URL.Path != "/api/kobo/device/v1/books/1/download" {
t.Errorf("upstream path = %q", req.URL.Path)
}
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/epub+zip"}}, Body: reader, ContentLength: -1}, nil
})),
)
server := httptest.NewServer(NewHandler(application, logger.New(logger.ERROR)))
defer server.Close()
defer func() { _ = writer.Close() }()
firstChunk := bytes.Repeat([]byte("x"), 32*1024)
result := make(chan error, 1)
go func() {
resp, err := server.Client().Get(server.URL + "/booksync/books/v1/books/1/download")
if err != nil {
result <- err
return
}
defer func() { _ = resp.Body.Close() }()
data := make([]byte, len(firstChunk))
_, err = io.ReadFull(resp.Body, data)
if err == nil && !bytes.Equal(data, firstChunk) {
err = io.ErrUnexpectedEOF
}
result <- err
}()
go func() { _, _ = writer.Write(firstChunk) }()
select {
case err := <-result:
if err != nil {
t.Fatal(err)
}
case <-time.After(3 * time.Second):
t.Fatal("book download did not stream before upstream completion")
}
}
+5 -7
View File
@@ -21,18 +21,16 @@ func (rw *responseWriter) WriteHeader(code int) {
rw.ResponseWriter.WriteHeader(code)
}
func (rw *responseWriter) Unwrap() http.ResponseWriter {
return rw.ResponseWriter
}
// LoggingMiddleware logs all incoming requests
func LoggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
rw := newResponseWriter(w)
next.ServeHTTP(rw, r)
log.Printf(
"%-7s %s %d %s",
r.Method,
r.RequestURI,
rw.statusCode,
time.Since(start),
)
log.Printf("%-7s %d %s", r.Method, rw.statusCode, time.Since(start))
})
}
+15 -9
View File
@@ -12,7 +12,12 @@ import (
func ListenAndServe(port int, application *app.App, logger *logger.Logger) {
addr := fmt.Sprintf(":%d", port)
logger.Infof("Web server starting on port %s", addr)
if err := http.ListenAndServe(addr, NewHandler(application, logger)); err != nil {
logger.Errorf("Web server failed to start: %v", err)
}
}
func NewHandler(application *app.App, logger *logger.Logger) http.Handler {
mux := http.NewServeMux()
// Register handlers
@@ -20,18 +25,19 @@ func ListenAndServe(port int, application *app.App, logger *logger.Logger) {
mux.HandleFunc("/api/kobo/download", application.HandleKoboDownload)
mux.HandleFunc("/api/kobo/send", application.HandleKoboSend)
mux.HandleFunc("/api/convert-image", application.HandleConvertImage)
mux.HandleFunc("/instapaper-proxy/storeapi/v1/initialization", application.HandleDumpAndForward)
mux.HandleFunc("/instapaper-proxy/instapaper/api/kobo/get", application.HandleKoboGet)
mux.HandleFunc("/instapaper-proxy/instapaper/api/kobo/download", application.HandleKoboDownload)
mux.HandleFunc("/instapaper-proxy/instapaper/api/kobo/send", application.HandleKoboSend)
storeProxy := http.StripPrefix("/instapaper-proxy/storeapi", http.HandlerFunc(application.HandleStoreProxy))
mux.Handle("/instapaper-proxy/storeapi/", storeProxy)
mux.Handle("/instapaper-proxy/storeapi", storeProxy)
mux.HandleFunc("/booksync/{deviceToken}/", application.HandleBookSync)
// Catch-all for unimplemented routes
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
logger.Warnf("404 Not Found: URL=%s, Method=%s, Params=%v", r.URL.Path, r.Method, r.URL.Query())
logger.Warnf("404 Not Found: Method=%s", r.Method)
http.Error(w, "404 Not Found", http.StatusNotFound)
})
// Apply logging middleware
loggedMux := LoggingMiddleware(mux)
if err := http.ListenAndServe(addr, loggedMux); err != nil {
logger.Errorf("Web server failed to start: %v", err)
}
}
return LoggingMiddleware(mux)
}
+121
View File
@@ -0,0 +1,121 @@
package webserver
import (
"bytes"
"io"
"log"
"net/http"
"net/http/httptest"
"strings"
"testing"
"readeckobo/internal/app"
"readeckobo/internal/config"
"readeckobo/internal/logger"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
func TestStoreRoutes(t *testing.T) {
application := app.NewApp(
app.WithConfig(&config.Config{PublicURL: "https://reader.example.com"}),
app.WithLogger(logger.New(logger.ERROR)),
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
if req.URL.Host != "storeapi.kobo.com" || req.Host != "storeapi.kobo.com" {
t.Errorf("upstream host = %q, Host = %q", req.URL.Host, req.Host)
}
switch req.URL.Path {
case "/", "/v1/initialization", "/v1/library/sync", "/v1//books":
default:
t.Errorf("unexpected upstream path = %q", req.URL.Path)
}
if req.URL.Path == "/v1//books" && req.URL.EscapedPath() != "/v1/%2Fbooks" {
t.Errorf("escaped upstream path = %q, want /v1/%%2Fbooks", req.URL.EscapedPath())
}
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"url":"https://www.instapaper.com"}`))}, nil
})),
)
handler := NewHandler(application, logger.New(logger.ERROR))
for _, path := range []string{"/instapaper-proxy/storeapi/v1/initialization", "/instapaper-proxy/storeapi/v1/library/sync", "/instapaper-proxy/storeapi/v1/%2Fbooks?a=1%3B2", "/instapaper-proxy/storeapi"} {
t.Run(path, func(t *testing.T) {
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, path, nil))
if rr.Code != http.StatusOK {
t.Errorf("status = %d, want 200", rr.Code)
}
})
}
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi-nope", nil))
if rr.Code != http.StatusNotFound {
t.Errorf("unknown store route status = %d, want 404", rr.Code)
}
}
func TestArticleRoutes(t *testing.T) {
readeckServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/bookmarks/sync" {
t.Errorf("unexpected Readeck request: %s", r.URL.Path)
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, "[]")
}))
defer readeckServer.Close()
application := app.NewApp(
app.WithConfig(&config.Config{
Readeck: config.ConfigReadeck{Host: readeckServer.URL},
Users: []config.User{{Token: "device", ReadeckAccessToken: "readeck"}},
}),
app.WithLogger(logger.New(logger.ERROR)),
app.WithReadeckHTTPClient(readeckServer.Client()),
)
handler := NewHandler(application, logger.New(logger.ERROR))
for _, tt := range []struct {
name, path, body, wantBody string
wantStatus int
}{
{"get", "/instapaper-proxy/instapaper/api/kobo/get", `{"access_token":"device"}`, `"status":1`, http.StatusOK},
{"download", "/instapaper-proxy/instapaper/api/kobo/download", `{"access_token":"device"}`, "Missing 'url' parameter", http.StatusBadRequest},
{"send", "/instapaper-proxy/instapaper/api/kobo/send", `{"access_token":"device","actions":[]}`, `"status":true`, http.StatusOK},
} {
t.Run(tt.name, func(t *testing.T) {
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, tt.path, strings.NewReader(tt.body))
req.Header.Set("Content-Type", "application/json")
handler.ServeHTTP(rr, req)
if rr.Code != tt.wantStatus || !strings.Contains(rr.Body.String(), tt.wantBody) {
t.Errorf("status = %d, body = %q; want %d and %q", rr.Code, rr.Body.String(), tt.wantStatus, tt.wantBody)
}
})
}
}
func TestRequestLogsDoNotExposeTokens(t *testing.T) {
var logs bytes.Buffer
old := log.Writer()
log.SetOutput(&logs)
defer log.SetOutput(old)
application := app.NewApp(
app.WithConfig(&config.Config{}),
app.WithLogger(logger.New(logger.DEBUG)),
app.WithImageHTTPClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, io.ErrUnexpectedEOF
})}),
)
handler := NewHandler(application, logger.New(logger.DEBUG))
handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/booksync/private-token/?access_token=query-secret", nil))
req := httptest.NewRequest(http.MethodPost, "/instapaper-proxy/instapaper/api/kobo/get", strings.NewReader(`{"access_token":"body-secret"}`))
handler.ServeHTTP(httptest.NewRecorder(), req)
handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/api/convert-image?url=https%3A%2F%2Fexample.com%2Fimg%3Ftoken%3Dimage-secret", nil))
for _, secret := range []string{"private-token", "query-secret", "body-secret", "image-secret"} {
if strings.Contains(logs.String(), secret) {
t.Errorf("request logs contain secret %q", secret)
}
}
}
-30
View File
@@ -1,30 +0,0 @@
# Proxy Kobo Store API so that we can modify the returned Instapaper URL.
location /instapaper-proxy/storeapi {
proxy_ssl_server_name on;
proxy_ssl_name storeapi.kobo.com;
proxy_set_header Host storeapi.kobo.com;
proxy_pass https://storeapi.kobo.com/;
}
location = /instapaper-proxy/storeapi/v1/initialization {
proxy_ssl_server_name on;
proxy_ssl_name storeapi.kobo.com;
proxy_set_header Host storeapi.kobo.com;
proxy_set_header Accept-Encoding "";
proxy_pass https://storeapi.kobo.com/;
sub_filter https://www.instapaper.com "$scheme://$host/instapaper-proxy/instapaper";
sub_filter_types application/json;
}
# Proxy to local Kobeck application
location /instapaper-proxy/instapaper/ {
proxy_pass http://readeckobo-upstream/;
proxy_set_header Host $host;
client_max_body_size 128M;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Prefix /instapaper-proxy/instapaper;
}