Support Kobo sync without custom Nginx routing #2
No files matched your search
@@ -15,3 +15,7 @@
|
||||
# llm agents
|
||||
/.gemini/
|
||||
/.claude/
|
||||
|
||||
# Local planning and sandbox configuration
|
||||
/PROJECT_NGINX.md
|
||||
/.nono.json
|
||||
@@ -9,11 +9,17 @@ when:
|
||||
include:
|
||||
- "**/*.go"
|
||||
- "**/*.gohtml"
|
||||
- "go.mod"
|
||||
- "go.sum"
|
||||
- ".golangci.yml"
|
||||
- "scripts/check-coverage.sh"
|
||||
- "bin/generate-encrypted-token.sh"
|
||||
- ".woodpecker/build-docker.yml"
|
||||
- "Dockerfile"
|
||||
exclude:
|
||||
- "**/*.md"
|
||||
on_empty: true
|
||||
- event: pull_request
|
||||
- event: tag
|
||||
ref: "refs/tags/v*.*.*"
|
||||
|
||||
@@ -50,6 +56,29 @@ steps:
|
||||
when:
|
||||
- event: tag
|
||||
|
||||
extract-pr-tag:
|
||||
image: alpine:3.22
|
||||
commands:
|
||||
- |
|
||||
branch_slug=$(printf '%s' "$CI_COMMIT_SOURCE_BRANCH" | sed 's/[^A-Za-z0-9_.-]/-/g' | cut -c1-96)
|
||||
printf 'pr-%s-%s\n' "$CI_COMMIT_PULL_REQUEST" "$branch_slug" > .tags
|
||||
cat .tags
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
build-docker-pr:
|
||||
image: woodpeckerci/plugin-docker-buildx:6.0.3
|
||||
settings:
|
||||
repo: git.eleith.com/eleith.private/readeckobo
|
||||
username: eleith
|
||||
dockerfile: Dockerfile
|
||||
registry: https://git.eleith.com
|
||||
password:
|
||||
from_secret: gitea_docker_registry
|
||||
tags_file: .tags
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
build-docker-main:
|
||||
image: woodpeckerci/plugin-docker-buildx:6.0.3
|
||||
settings:
|
||||
@@ -79,8 +108,8 @@ steps:
|
||||
- event: tag
|
||||
ref: "refs/tags/v*.*.*"
|
||||
|
||||
notify-success:
|
||||
image: deblan/woodpecker-email:3.2.0
|
||||
notify:
|
||||
image: deblan/woodpecker-email:3.3.0
|
||||
settings:
|
||||
dsn:
|
||||
from_secret: woodpecker_email_dsn
|
||||
@@ -91,23 +120,8 @@ steps:
|
||||
from_secret: woodpecker_email_from_name
|
||||
recipients:
|
||||
from_secret: woodpecker_email_recipient
|
||||
level: success
|
||||
recipients_only: true
|
||||
when:
|
||||
- status: [success]
|
||||
notify-failure:
|
||||
image: deblan/woodpecker-email:3.2.0
|
||||
settings:
|
||||
dsn:
|
||||
from_secret: woodpecker_email_dsn
|
||||
from:
|
||||
address:
|
||||
from_secret: woodpecker_email_from_address
|
||||
name:
|
||||
from_secret: woodpecker_email_from_name
|
||||
recipients:
|
||||
from_secret: woodpecker_email_recipient
|
||||
level: failure
|
||||
recipients_only: true
|
||||
when:
|
||||
- status: [failure]
|
||||
- event: push
|
||||
branch: main
|
||||
status: [failure, success]
|
||||
@@ -16,8 +16,7 @@ The project follows a standard Go project layout:
|
||||
- **`internal/models/`**: Shared data structures for Kobo and Readeck API payloads.
|
||||
|
||||
## Deployment & Infrastructure
|
||||
- **Proxy Interception**: The system relies on a reverse proxy (like Nginx) to intercept requests from the Kobo device intended for `www.instapaper.com` and redirect them to this service.
|
||||
- **Configuration**: See `nginx.conf.snippet` for the rewrite rules.
|
||||
- **Deployment**: Kobo requests can reach this app through Cloudflare Tunnel or Nginx. See `docs/CLOUDFLARE.md` and `docs/NGINX.md` for the routing and initialization rewrite examples.
|
||||
- **Docker**: The application is containerized (`Dockerfile`, `docker-compose.yml`).
|
||||
|
||||
## Development Guidelines for AI Agents
|
||||
|
||||
@@ -17,13 +17,34 @@ more.
|
||||
|
||||
## 🚀 Quick Start (for Users)
|
||||
|
||||
Getting up and running is a breeze with Docker.
|
||||
### 1. Choose how to publish
|
||||
|
||||
### 1. Configure `readeckobo`
|
||||
Choose a public HTTPS hostname (for example, `readeckobo.example.com`) and one
|
||||
deployment method:
|
||||
|
||||
First, copy `config.yaml.example` to `config.yaml` and edit it to match your setup.
|
||||
* [Cloudflare Tunnel](docs/CLOUDFLARE.md): forward requests to `readeckobo`.
|
||||
* [Nginx](docs/NGINX.md): proxy Kobo Store and article requests.
|
||||
|
||||
For a detailed explanation of all options and how to get your Readeck API token, see [docs/CONFIG.md](docs/CONFIG.md).
|
||||
Both use the same Kobo settings below.
|
||||
|
||||
### 2. Generate a device token
|
||||
|
||||
Copy `config.yaml.example` to `config.yaml`, then build the image. Find your
|
||||
Kobo's serial number under **Settings -> Device Information**. Replace
|
||||
`<YOUR_KOBO_SERIAL>` below with that number (without angle brackets):
|
||||
|
||||
```sh
|
||||
docker-compose build
|
||||
docker-compose run --rm --entrypoint /app/bin/generate-encrypted-token.sh readeckobo <YOUR_KOBO_SERIAL>
|
||||
```
|
||||
|
||||
This one-off container generates a **plain text token** for `config.yaml` and
|
||||
an **encrypted token** for the Kobo. The app is not running yet.
|
||||
|
||||
### 3. Configure `readeckobo` and Kobo
|
||||
|
||||
Edit `config.yaml` with your Readeck host and API token and the **plain text**
|
||||
device token from step 2:
|
||||
|
||||
```yaml
|
||||
server:
|
||||
@@ -32,97 +53,48 @@ log_level: info
|
||||
readeck:
|
||||
host: "https://your-readeck-instance.com"
|
||||
users:
|
||||
- token: "a-random-uuid-token-for-a-kobo"
|
||||
- token: "<THE-PLAIN-TEXT-TOKEN-FROM-THE-SCRIPT>"
|
||||
readeck_access_token: "a-readeck-api-token"
|
||||
```
|
||||
|
||||
### 2. Run with Docker
|
||||
For other options, see [docs/CONFIG.md](docs/CONFIG.md). To sync ebooks too,
|
||||
see the [Kobo ebook sync guide](docs/KOBO_SYNC.md) for the per-device
|
||||
`api_endpoint` instead of the article-only setting below.
|
||||
|
||||
Once your configuration is ready, fire it up!
|
||||
Mount your Kobo and edit `.kobo/Kobo/Kobo eReader.conf`, using the **encrypted**
|
||||
token from the script. Replace `readeckobo.example.com` with the hostname you
|
||||
chose in step 1:
|
||||
|
||||
```ini
|
||||
[OneStoreServices]
|
||||
api_endpoint=https://readeckobo.example.com/instapaper-proxy/storeapi
|
||||
instapaper_env_url=https://readeckobo.example.com
|
||||
|
||||
[Instapaper]
|
||||
AccessToken=@ByteArray(<THE-ENCRYPTED-TOKEN-FROM-THE-SCRIPT>)
|
||||
```
|
||||
|
||||
### 4. Run the app
|
||||
|
||||
```sh
|
||||
docker-compose build
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
The server will be available at `http://localhost:8080`.
|
||||
|
||||
### 3. Generate a Device Token
|
||||
|
||||
For each Kobo device, you will need a unique token. This process involves
|
||||
generating a token and then encrypting it for the Kobo device.
|
||||
|
||||
First, find your Kobo's serial number, which is available under
|
||||
**Settings -> Device Information** on your e-reader.
|
||||
|
||||
With `readeckobo` running, use this command to generate and encrypt the token,
|
||||
replacing `<YOUR_KOBO_SERIAL>` with your device's serial number:
|
||||
|
||||
```sh
|
||||
docker-compose exec readeckobo bin/generate-encrypted-token.sh <YOUR_KOBO_SERIAL>
|
||||
```
|
||||
|
||||
The script will output two important pieces of information:
|
||||
|
||||
1. A **plain text UUID token** to be used in your `config.yaml`.
|
||||
2. An **encrypted token** to be used in your Kobo's configuration file.
|
||||
|
||||
### 4. Configure Your `readeckobo` and Kobo Device
|
||||
|
||||
Follow the output from the script to configure your services.
|
||||
|
||||
1. **Update `config.yaml`**: Add the plain text UUID token to the `users`
|
||||
section of your `config.yaml`.
|
||||
|
||||
```yaml
|
||||
users:
|
||||
- token: "<THE-PLAIN-TEXT-UUID-FROM-THE-SCRIPT>"
|
||||
readeck_access_token: "a-readeck-api-token"
|
||||
```
|
||||
|
||||
2. **Update Your Kobo**: Mount your Kobo and find the
|
||||
`.kobo/Kobo/Kobo eReader.conf` file. Add or update these settings using the
|
||||
**encrypted** token from the script's output.
|
||||
|
||||
```ini
|
||||
[OneStoreServices]
|
||||
api_endpoint=https://readeckobo.example.com/instapaper-proxy/storeapi
|
||||
instapaper_env_url=https://readeckobo.example.com
|
||||
|
||||
[Instapaper]
|
||||
AccessToken=@ByteArray(<THE-ENCRYPTED-TOKEN-FROM-THE-SCRIPT>)
|
||||
```
|
||||
|
||||
Replace `https://readeckobo.example.com` with the full path to where you are deploying
|
||||
the readeckobo service to
|
||||
|
||||
### 5. Set Up a Reverse Proxy
|
||||
|
||||
`readeckobo` must be run behind a reverse proxy to handle HTTPS. It's crucial
|
||||
to proxy three specific location blocks, as shown in our `nginx.conf.snippet`
|
||||
example.
|
||||
|
||||
Your Kobo device periodically re-syncs its configuration from Kobo's servers,
|
||||
which can overwrite your custom Instapaper endpoint. The proxy rules below
|
||||
ensure this connection is preserved.
|
||||
|
||||
<!-- markdownlint-disable MD013 -->
|
||||
| Location Block | Proxies To | Purpose |
|
||||
| ----------------------------------------------- | ------------------------ | --------------------------------------------------------------------------------------------------- |
|
||||
| `/instapaper-proxy/instapaper/` | `readeckobo` application | Handles the main Instapaper API requests (sync, download, etc.) to your `readeckobo` instance. |
|
||||
| `/instapaper-proxy/storeapi/` | `storeapi.kobo.com` | Forwards general API requests to Kobo's servers. |
|
||||
| `/instapaper-proxy/storeapi/v1/initialization` | `storeapi.kobo.com` | Intercepts the Kobo configuration response to rewrite the Instapaper URL back to your proxy endpoint. |
|
||||
<!-- markdownlint-enable MD013 -->
|
||||
|
||||
Without these rules, your Kobo will eventually lose its connection to `readeckobo`.
|
||||
The app is available locally at `http://localhost:8080`. Set up your chosen
|
||||
[Cloudflare Tunnel](docs/CLOUDFLARE.md) or [Nginx](docs/NGINX.md) deployment
|
||||
before or after starting the app; the public HTTPS address must work before
|
||||
Kobo syncs. If you later edit `config.yaml`, recreate the running Docker
|
||||
container so it reads the new file; the deployment guides show the command.
|
||||
|
||||
## 🔒 A Quick Word on Security
|
||||
|
||||
A little security goes a long way.
|
||||
|
||||
* **Use HTTPS:** deploy behind a reverse proxy that provides HTTPS
|
||||
* **Stay Local:** Keep it on your local private network
|
||||
* **Kobo Password:** prevent unauthorized mounting with a Kobo password
|
||||
* **Use HTTPS:** follow either deployment guide above, and keep the app's HTTP
|
||||
port private.
|
||||
* **Protect tokens:** don't share config files or request URLs containing device
|
||||
tokens. Browser login or bot challenges may prevent Kobo sync.
|
||||
* **Kobo Password:** prevent unauthorized mounting with a Kobo password.
|
||||
|
||||
## 🧑💻 For Developers
|
||||
|
||||
|
||||
+7
-1
@@ -2,7 +2,13 @@ server:
|
||||
port: 8080
|
||||
log_level: info
|
||||
readeck:
|
||||
host: "https://your-readeck-instance.com"
|
||||
host: "https://readeck.example.com"
|
||||
# public_url: "https://readeckobo.example.com" # optional: use if proxy headers cannot supply a trusted public origin
|
||||
users:
|
||||
- token: "a-very-secret-token-for-your-kobo"
|
||||
readeck_access_token: "your-plain-text-readeck-access-token"
|
||||
# Optional: overrides the Kobo Store fallback for this device.
|
||||
# book_sync_url: "https://books.example.com/api/kobo/device-token"
|
||||
# Optional: send archived bookmarks with Kobo status "2" during incremental sync.
|
||||
# Default false keeps status "1"; neither setting deletes Readeck bookmarks.
|
||||
# remove_archived_from_kobo: false
|
||||
@@ -0,0 +1,41 @@
|
||||
# Cloudflare Tunnel
|
||||
|
||||
Use the [README quick start](../README.md) for `config.yaml` and the Kobo's Instapaper settings. This guide publishes that same setup over HTTPS without Nginx.
|
||||
|
||||
## Publish the app
|
||||
|
||||
For a simple Tunnel route, set `public_url` to your public HTTPS origin in
|
||||
`config.yaml`. This pins the rewritten Instapaper URL without relying on a
|
||||
forwarded host header the Tunnel may not overwrite:
|
||||
|
||||
```yaml
|
||||
public_url: "https://readeckobo.example.com"
|
||||
```
|
||||
|
||||
If you are changing an existing deployment, recreate the Docker service to
|
||||
reload the bind-mounted config file (`docker-compose up -d --force-recreate
|
||||
readeckobo`), or restart your non-Docker process.
|
||||
|
||||
Before making the hostname public, add a [Cloudflare WAF custom rule](https://developers.cloudflare.com/waf/custom-rules/) with **Block** action for `(http.host eq "readeckobo.example.com" and http.request.uri.path eq "/api/convert-image")`. This direct, unauthenticated endpoint fetches a caller-provided URL and is not needed by the prefixed Kobo article routes. The Tunnel can still forward all other paths without path-specific routing rules.
|
||||
|
||||
Add a [published application route](https://developers.cloudflare.com/cloudflare-one/networks/routes/add-routes/) to a connected Cloudflare Tunnel:
|
||||
|
||||
| Route setting | Example |
|
||||
| :--- | :--- |
|
||||
| Public hostname | `readeckobo.example.com` |
|
||||
| Service URL | `http://localhost:8080` |
|
||||
|
||||
Use the actual address `cloudflared` can reach: `localhost` works when it runs on the same host as the published app port, not when it runs in a separate container. Forward **all paths** to the app without a path rule. Go proxies Kobo Store traffic and rewrites initialization so Kobo continues sending Instapaper requests to `readeckobo`.
|
||||
|
||||
## Verify and secure
|
||||
|
||||
Check that initialization points back to your hostname:
|
||||
|
||||
```sh
|
||||
curl --compressed -fsS https://readeckobo.example.com/instapaper-proxy/storeapi/v1/initialization \
|
||||
| grep -F 'https://readeckobo.example.com/instapaper-proxy/instapaper'
|
||||
```
|
||||
|
||||
Then check sync on a real Kobo. Browser login or bot challenges at the edge may block its requests. `docker-compose.yml` publishes port 8080 on all interfaces: restrict direct HTTP access with a firewall, or bind the port to `127.0.0.1:8080:8080` if `cloudflared` runs on the host. If it runs in another container or host, use a private network instead. Keep device tokens and request URLs containing them private.
|
||||
|
||||
For Kobo ebook syncing with a compatible service, see [Kobo ebook sync](KOBO_SYNC.md). The same Tunnel route forwards those requests to readeckobo.
|
||||
+8
-15
@@ -1,6 +1,7 @@
|
||||
# Configuration Reference
|
||||
|
||||
This document provides a detailed reference for all available configuration options in `config.yaml`.
|
||||
Configuration options for `config.yaml`. For a starting file, see
|
||||
[config.yaml.example](../config.yaml.example) or the [quick start](../README.md#3-configure-readeckobo-and-kobo).
|
||||
|
||||
## Top-Level Options
|
||||
|
||||
@@ -10,6 +11,9 @@ This document provides a detailed reference for all available configuration opti
|
||||
| `log_level` | Output verbosity. Options: `debug`, `info`, `warn`, `error`. | String | `info` | No |
|
||||
| `readeck.host` | The full base URL of your Readeck instance. | URL String | - | **Yes** |
|
||||
| `users` | A list of Kobo-to-Readeck user mappings. | List of Objects | - | **Yes** |
|
||||
| `public_url` | Public origin used for the rewritten Instapaper URL in Kobo initialization. | URL String | Request host/forwarding headers | No |
|
||||
|
||||
Set `public_url` when your ingress cannot reliably supply the public HTTPS origin, cannot overwrite client-supplied forwarding headers, or you need one canonical hostname. Use the origin only (no path, query, or credentials). Otherwise, Go-proxied initialization uses `X-Forwarded-Host` (or the request `Host`) and `X-Forwarded-Proto` (or the connection scheme), **not** `readeck.host`. Keep the app behind a trusted ingress when relying on those headers. See [Cloudflare Tunnel](CLOUDFLARE.md) and [Nginx](NGINX.md).
|
||||
|
||||
## User Object
|
||||
|
||||
@@ -19,6 +23,8 @@ Each entry in the `users` list maps a Kobo device identity to a Readeck account.
|
||||
| :--- | :--- |
|
||||
| `token` | A unique UUID used to identify the Kobo device. This is the **Plain Text** token generated by the `bin/generate-encrypted-token.sh` script. |
|
||||
| `readeck_access_token` | The API token for the Readeck account you want to sync. |
|
||||
| `book_sync_url` | Optional per-user Kobo API endpoint for an ebook service such as Komga. When used through `/booksync/<device-token>/`, it must generate cover and download URLs the Kobo can reach directly; readeckobo does not rewrite them. See [Kobo ebook sync](KOBO_SYNC.md). |
|
||||
| `remove_archived_from_kobo` | Optional, default `false`. In incremental sync, send archived items with status `"2"` (requesting removal) instead of `"1"`. Full sync excludes them; neither setting deletes Readeck data. Verify on a real Kobo. |
|
||||
|
||||
### How to get a Readeck API Token
|
||||
|
||||
@@ -27,17 +33,4 @@ Each entry in the `users` list maps a Kobo device identity to a Readeck account.
|
||||
3. Create a new token with at least **Bookmarks: Read** permissions (Write permissions are needed for archiving/favoriting).
|
||||
4. Copy the token string into your `config.yaml`.
|
||||
|
||||
## Example `config.yaml`
|
||||
|
||||
```yaml
|
||||
server:
|
||||
port: 8080
|
||||
log_level: debug
|
||||
|
||||
readeck:
|
||||
host: "https://my-readeck.example.com"
|
||||
|
||||
users:
|
||||
- token: "550e8400-e29b-41d4-a716-446655440000" # From generate-encrypted-token.sh
|
||||
readeck_access_token: "rdk_..." # From Readeck Profile
|
||||
```
|
||||
For Kobo-compatible ebook services, see [Kobo ebook sync](KOBO_SYNC.md).
|
||||
@@ -0,0 +1,79 @@
|
||||
# Kobo ebook sync alongside Readeck articles
|
||||
|
||||
Use a per-device readeckobo API endpoint to sync Readeck articles alongside
|
||||
books from a Kobo-compatible service. Readeckobo forwards Kobo API requests to
|
||||
the selected user's ebook service and points the initialization response's
|
||||
Instapaper URL at readeckobo. Book-service URLs in initialization and later
|
||||
responses pass through unchanged. Covers and ebook downloads use the absolute
|
||||
URLs generated by the ebook service, so the Kobo contacts that service directly.
|
||||
|
||||
[Komga's Kobo sync](https://komga.org/docs/guides/kobo/) is one example.
|
||||
|
||||
## Configure the ebook service
|
||||
|
||||
For each Kobo device, set `book_sync_url` to that user's full **Kobo API
|
||||
endpoint** on the ebook service, including any authentication path. For
|
||||
example, with Komga:
|
||||
|
||||
```yaml
|
||||
users:
|
||||
- token: "<READECKOBO-PLAIN-DEVICE-TOKEN>"
|
||||
readeck_access_token: "<READECK-API-TOKEN>"
|
||||
book_sync_url: "https://books.example.com/kobo/<KOMGA_API_KEY>"
|
||||
```
|
||||
|
||||
Set `public_url` if your ingress cannot reliably provide readeckobo's
|
||||
public HTTPS origin; see [Cloudflare Tunnel](CLOUDFLARE.md) and
|
||||
[Nginx](NGINX.md).
|
||||
|
||||
Use an ebook-service hostname that both readeckobo and the Kobo can reach. Its
|
||||
initialization response should contain the literal `https://www.instapaper.com`
|
||||
in its Instapaper URL or already contain the desired readeckobo URL. Go replaces
|
||||
that literal in a successful initialization response with
|
||||
`https://reader.example.com/instapaper-proxy/instapaper`. The service must generate
|
||||
Kobo-reachable cover and download URLs because readeckobo leaves them unchanged.
|
||||
Publish the ebook service separately from readeckobo if needed.
|
||||
|
||||
Reload readeckobo after changing `config.yaml` (for Docker Compose:
|
||||
`docker-compose up -d --force-recreate readeckobo`).
|
||||
|
||||
## Publish readeckobo
|
||||
|
||||
[Cloudflare Tunnel](CLOUDFLARE.md) forwards `/booksync/` to Go with its normal
|
||||
app route. With [Nginx](NGINX.md#kobo-ebook-syncing), add the `/booksync/`
|
||||
location to readeckobo's public HTTPS server. The plain device token in the
|
||||
route selects the matching user's `book_sync_url`; readeckobo forwards the
|
||||
remaining path and query to that service. The same public hostname also serves
|
||||
article requests.
|
||||
|
||||
Check the initialization response before changing the Kobo:
|
||||
|
||||
```text
|
||||
https://reader.example.com/booksync/<READECKOBO-PLAIN-DEVICE-TOKEN>/v1/initialization
|
||||
```
|
||||
|
||||
Its `instapaper_env_url` should point to readeckobo. For a service returning
|
||||
`https://www.instapaper.com/api/kobo`, the resulting URL is
|
||||
`https://reader.example.com/instapaper-proxy/instapaper/api/kobo`. Its
|
||||
`image_host` and image templates should contain URLs reachable by the Kobo,
|
||||
whether served by the ebook service or a CDN. Treat the request URL and the service API key as secrets
|
||||
when sharing logs or screenshots.
|
||||
|
||||
## Configure the Kobo
|
||||
|
||||
In `.kobo/Kobo/Kobo eReader.conf`, set the per-device API endpoint:
|
||||
|
||||
```ini
|
||||
[OneStoreServices]
|
||||
api_endpoint=https://reader.example.com/booksync/<READECKOBO-PLAIN-DEVICE-TOKEN>
|
||||
```
|
||||
|
||||
Keep the Kobo's existing `instapaper_env_url` and encrypted Instapaper
|
||||
`AccessToken`. The token in `api_endpoint` is readeckobo's plain device token;
|
||||
`book_sync_url` holds the ebook service's endpoint and its own credential.
|
||||
Configure the ebook service to handle any Kobo Store requests you need; for
|
||||
example, Komga offers Kobo Store proxying in its Server Settings.
|
||||
|
||||
On a real Kobo, verify initialization, library sync, covers, ebook downloads,
|
||||
and Readeck articles. Check the generated service URLs and forwarding headers
|
||||
for your deployment. To roll back, restore the Kobo's previous `api_endpoint`.
|
||||
@@ -0,0 +1,81 @@
|
||||
# Nginx
|
||||
|
||||
Use the [README quick start](../README.md) for `config.yaml` and the Kobo's
|
||||
Instapaper settings.
|
||||
|
||||
## Kobo article syncing
|
||||
|
||||
Add these locations to the HTTPS `server` block for **readeckobo's public
|
||||
hostname** (the Instapaper article address, and the `api_endpoint` address when
|
||||
using ebook sync).
|
||||
The example assumes Nginx can reach `readeckobo` at `127.0.0.1:8080`; use your
|
||||
app's address if different.
|
||||
|
||||
```nginx
|
||||
# The bare Kobo Store endpoint.
|
||||
location = /instapaper-proxy/storeapi {
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_name storeapi.kobo.com;
|
||||
proxy_set_header Host storeapi.kobo.com;
|
||||
proxy_pass https://storeapi.kobo.com/;
|
||||
}
|
||||
|
||||
# Forward Store subpaths with a single leading slash upstream.
|
||||
location /instapaper-proxy/storeapi/ {
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_name storeapi.kobo.com;
|
||||
proxy_set_header Host storeapi.kobo.com;
|
||||
proxy_pass https://storeapi.kobo.com/;
|
||||
}
|
||||
|
||||
# Fetch Kobo Store initialization for the Instapaper URL rewrite.
|
||||
location = /instapaper-proxy/storeapi/v1/initialization {
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_name storeapi.kobo.com;
|
||||
proxy_set_header Host storeapi.kobo.com;
|
||||
proxy_set_header Accept-Encoding "";
|
||||
proxy_set_header If-None-Match "";
|
||||
proxy_set_header If-Modified-Since "";
|
||||
proxy_set_header Range "";
|
||||
proxy_set_header If-Range "";
|
||||
proxy_pass https://storeapi.kobo.com/v1/initialization;
|
||||
|
||||
sub_filter https://www.instapaper.com "$scheme://$host/instapaper-proxy/instapaper";
|
||||
sub_filter_types application/json;
|
||||
}
|
||||
|
||||
# Instapaper article requests go to readeckobo.
|
||||
location /instapaper-proxy/instapaper/ {
|
||||
proxy_pass http://127.0.0.1:8080/;
|
||||
proxy_set_header Host $host;
|
||||
client_max_body_size 128M;
|
||||
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-Host $host;
|
||||
proxy_set_header X-Forwarded-Prefix /instapaper-proxy/instapaper;
|
||||
}
|
||||
```
|
||||
|
||||
Keep port 8080 private: `docker-compose.yml` exposes it on all interfaces by
|
||||
default. Bind it to `127.0.0.1:8080:8080` if Nginx runs on the host, or use a
|
||||
private network if it runs elsewhere.
|
||||
|
||||
## Kobo ebook syncing
|
||||
|
||||
For [Kobo ebook sync](KOBO_SYNC.md), add this location to readeckobo's public
|
||||
HTTPS `server` block. The `proxy_pass` forwards the full
|
||||
`/booksync/<device-token>/...` path to Go, which selects the user's ebook
|
||||
service. The ebook service publishes its own Kobo-reachable cover and download
|
||||
URLs:
|
||||
|
||||
```nginx
|
||||
location /booksync/ {
|
||||
access_log off; # Request paths contain a device token.
|
||||
proxy_pass http://127.0.0.1:8080;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Host $host;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
```
|
||||
+103
-72
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/draw"
|
||||
@@ -11,8 +12,8 @@ import (
|
||||
"image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -33,6 +34,7 @@ type App struct {
|
||||
Logger *logger.Logger
|
||||
ImageHTTPClient *http.Client
|
||||
ReadeckHTTPClient *http.Client
|
||||
ProxyTransport http.RoundTripper
|
||||
}
|
||||
|
||||
func WithImageHTTPClient(client *http.Client) Option {
|
||||
@@ -69,6 +71,12 @@ func WithReadeckHTTPClient(client *http.Client) Option {
|
||||
}
|
||||
}
|
||||
|
||||
func WithProxyTransport(transport http.RoundTripper) Option {
|
||||
return func(a *App) {
|
||||
a.ProxyTransport = transport
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) handleFullSync(ctx context.Context, readeckClient *readeck.Client, req *models.KoboGetRequest) (map[string]models.KoboArticleItem, int, error) {
|
||||
count, _ := strconv.Atoi(req.Count)
|
||||
offset, _ := strconv.Atoi(req.Offset)
|
||||
@@ -136,7 +144,7 @@ func (a *App) handleFullSync(ctx context.Context, readeckClient *readeck.Client,
|
||||
return resultList, totalNonArchivedBookmarks, nil
|
||||
}
|
||||
|
||||
func (a *App) handleIncrementalSync(ctx context.Context, readeckClient *readeck.Client, since *time.Time) (map[string]models.KoboArticleItem, int, error) {
|
||||
func (a *App) handleIncrementalSync(ctx context.Context, readeckClient *readeck.Client, since *time.Time, user config.User) (map[string]models.KoboArticleItem, int, error) {
|
||||
resultList := make(map[string]models.KoboArticleItem)
|
||||
|
||||
bsyncs, err := readeckClient.GetBookmarksSync(ctx, since)
|
||||
@@ -186,6 +194,9 @@ func (a *App) handleIncrementalSync(ctx context.Context, readeckClient *readeck.
|
||||
|
||||
if bookmark.IsArchived {
|
||||
entry.Status = "1"
|
||||
if user.RemoveArchivedFromKobo {
|
||||
entry.Status = "2"
|
||||
}
|
||||
} else {
|
||||
entry.Status = "0"
|
||||
totalNonArchivedBookmarks++
|
||||
@@ -200,12 +211,12 @@ func (a *App) HandleKoboGet(w http.ResponseWriter, r *http.Request) {
|
||||
bodyBytes, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to read request body", http.StatusInternalServerError)
|
||||
a.Logger.Errorf("Error reading /api/kobo/get request body: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error reading /api/kobo/get request body: %v", err)
|
||||
return
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
|
||||
|
||||
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/get:\nMethod: %s\nURL: %s\nHeaders: %v\nBody: %s", r.Method, r.URL, r.Header, string(bodyBytes))
|
||||
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/get: Method=%s", r.Method)
|
||||
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
@@ -215,21 +226,21 @@ func (a *App) HandleKoboGet(w http.ResponseWriter, r *http.Request) {
|
||||
var req models.KoboGetRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
||||
a.Logger.Errorf("Error decoding /api/kobo/get request: %v, body: %s, URL: %s, Params: %v", err, string(bodyBytes), r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error decoding /api/kobo/get request: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
readeckToken, err := a.getReadeckToken(req.AccessToken)
|
||||
user, err := a.getUser(req.AccessToken)
|
||||
if err != nil {
|
||||
http.Error(w, "Invalid access token", http.StatusUnauthorized)
|
||||
a.Logger.Errorf("Error authenticating token for /api/kobo/get: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error authenticating token for /api/kobo/get: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
readeckClient, err := a.newReadeckClient(readeckToken)
|
||||
readeckClient, err := a.newReadeckClient(user.ReadeckAccessToken)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to initialize Readeck client", http.StatusInternalServerError)
|
||||
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/get: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/get: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -252,7 +263,7 @@ func (a *App) HandleKoboGet(w http.ResponseWriter, r *http.Request) {
|
||||
resultList, total, err = a.handleFullSync(r.Context(), readeckClient, &req)
|
||||
} else {
|
||||
a.Logger.Debugf("Handling incremental sync.")
|
||||
resultList, total, err = a.handleIncrementalSync(r.Context(), readeckClient, since)
|
||||
resultList, total, err = a.handleIncrementalSync(r.Context(), readeckClient, since, user)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -325,12 +336,12 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
|
||||
bodyBytes, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to read request body", http.StatusInternalServerError)
|
||||
a.Logger.Errorf("Error reading /api/kobo/download request body: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error reading /api/kobo/download request body: %v", err)
|
||||
return
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
|
||||
|
||||
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/download:\nMethod: %s\nURL: %s\nHeaders: %v\nBody: %s", r.Method, r.URL, r.Header, string(bodyBytes))
|
||||
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/download: Method=%s", r.Method)
|
||||
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
@@ -339,44 +350,45 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
var req models.KoboDownloadRequest
|
||||
if err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&req); err != nil {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
form, err := downloadFormValues(r, bodyBytes)
|
||||
if err != nil {
|
||||
http.Error(w, "Invalid request body or form data", http.StatusBadRequest)
|
||||
a.Logger.Errorf("Error decoding /api/kobo/download request: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error decoding /api/kobo/download request: %v", err)
|
||||
return
|
||||
}
|
||||
req.AccessToken = r.FormValue("access_token")
|
||||
req.ConsumerKey = r.FormValue("consumer_key")
|
||||
req.Images, _ = strconv.Atoi(r.FormValue("images"))
|
||||
req.Refresh, _ = strconv.Atoi(r.FormValue("refresh"))
|
||||
req.Output = r.FormValue("output")
|
||||
req.URL = r.FormValue("url")
|
||||
req.AccessToken = form.Get("access_token")
|
||||
req.ConsumerKey = form.Get("consumer_key")
|
||||
req.Images, _ = strconv.Atoi(form.Get("images"))
|
||||
req.Refresh, _ = strconv.Atoi(form.Get("refresh"))
|
||||
req.Output = form.Get("output")
|
||||
req.URL = form.Get("url")
|
||||
}
|
||||
|
||||
readeckToken, err := a.getReadeckToken(req.AccessToken)
|
||||
if err != nil {
|
||||
http.Error(w, "Invalid access token", http.StatusUnauthorized)
|
||||
a.Logger.Errorf("Error authenticating token for /api/kobo/download: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error authenticating token for /api/kobo/download: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
readeckClient, err := a.newReadeckClient(readeckToken)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to initialize Readeck client", http.StatusInternalServerError)
|
||||
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/download: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/download: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
reqURLStr := req.URL
|
||||
if reqURLStr == "" {
|
||||
http.Error(w, "Missing 'url' parameter", http.StatusBadRequest)
|
||||
a.Logger.Errorf("Error: Missing 'url' parameter in /api/kobo/download request, URL: %s, Params: %v", r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Missing 'url' parameter in /api/kobo/download request")
|
||||
return
|
||||
}
|
||||
|
||||
parsedURL, err := url.Parse(reqURLStr)
|
||||
if err != nil {
|
||||
http.Error(w, "Invalid 'url' parameter", http.StatusBadRequest)
|
||||
a.Logger.Errorf("Error: Invalid 'url' parameter in /api/kobo/download request: %v, url: %s, URL: %s, Params: %v", err, reqURLStr, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Invalid 'url' parameter in /api/kobo/download request: %T", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -392,7 +404,7 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
|
||||
isArchived := false
|
||||
bookmarks, tp, err := readeckClient.GetBookmarks(ctx, site, currentPage, &isArchived)
|
||||
if err != nil {
|
||||
a.Logger.Warnf("Error searching Readeck bookmarks for site %s, page %d in /api/kobo/download: %v, URL: %s, Params: %v", site, currentPage, err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Warnf("Error searching Readeck bookmarks for site %s, page %d in /api/kobo/download: %v", site, currentPage, err)
|
||||
break
|
||||
}
|
||||
totalPages = tp
|
||||
@@ -401,7 +413,7 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
|
||||
if bookmarks[i].URL != "" {
|
||||
match, err := compareURLs(bookmarks[i].URL, reqURLStr)
|
||||
if err != nil {
|
||||
a.Logger.Warnf("Error comparing URLs for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarks[i].ID, err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Warnf("Error comparing URLs for bookmark %s in /api/kobo/download: %T", bookmarks[i].ID, err)
|
||||
continue
|
||||
}
|
||||
if match {
|
||||
@@ -428,14 +440,14 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
|
||||
articleHTML, err := readeckClient.GetBookmarkArticle(ctx, bookmarkFound.ID)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to fetch article content", http.StatusInternalServerError)
|
||||
a.Logger.Errorf("Error fetching article content for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarkFound.ID, err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error fetching article content for bookmark %s in /api/kobo/download: %v", bookmarkFound.ID, err)
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := html.Parse(strings.NewReader(articleHTML))
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to parse article HTML", http.StatusInternalServerError)
|
||||
a.Logger.Errorf("Error parsing article HTML for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarkFound.ID, err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error parsing article HTML for bookmark %s in /api/kobo/download: %v", bookmarkFound.ID, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -474,7 +486,7 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
|
||||
var buf bytes.Buffer
|
||||
if err := html.Render(&buf, doc); err != nil {
|
||||
http.Error(w, "Failed to render modified HTML", http.StatusInternalServerError)
|
||||
a.Logger.Errorf("Error rendering modified HTML for bookmark %s in /api/kobo/download: %v, URL: %s, Params: %v", bookmarkFound.ID, err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error rendering modified HTML for bookmark %s in /api/kobo/download: %v", bookmarkFound.ID, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -485,10 +497,47 @@ func (a *App) HandleKoboDownload(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||||
a.Logger.Errorf("Error encoding response for /api/kobo/download: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error encoding response for /api/kobo/download: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func downloadFormValues(r *http.Request, body []byte) (url.Values, error) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
mediaType, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if mediaType != "application/x-www-form-urlencoded" || len(body) > 10<<20 {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Kobo sends a raw ; inside the URL value. Escape only that form field,
|
||||
// then let Go validate and decode the complete form as usual.
|
||||
fields := strings.Split(string(body), "&")
|
||||
urlFields := 0
|
||||
repaired := false
|
||||
for i, field := range fields {
|
||||
if !strings.HasPrefix(field, "url=") {
|
||||
continue
|
||||
}
|
||||
urlFields++
|
||||
if strings.Contains(field, ";") {
|
||||
fields[i] = strings.ReplaceAll(field, ";", "%3B")
|
||||
repaired = true
|
||||
}
|
||||
}
|
||||
if urlFields != 1 || !repaired {
|
||||
return nil, err
|
||||
}
|
||||
form, err := url.ParseQuery(strings.Join(fields, "&"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(form["url"]) != 1 {
|
||||
return nil, errors.New("duplicate URL form field")
|
||||
}
|
||||
return form, nil
|
||||
}
|
||||
return r.Form, nil
|
||||
}
|
||||
|
||||
func getSitesToTry(host string) []string {
|
||||
var sites []string
|
||||
parts := strings.Split(host, ".")
|
||||
@@ -518,12 +567,12 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) {
|
||||
bodyBytes, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to read request body", http.StatusInternalServerError)
|
||||
a.Logger.Errorf("Error reading /api/kobo/send request body: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error reading /api/kobo/send request body: %v", err)
|
||||
return
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
|
||||
|
||||
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/send:\nMethod: %s\nURL: %s\nHeaders: %v\nBody: %s", r.Method, r.URL, r.Header, string(bodyBytes))
|
||||
a.Logger.Debugf("Incoming Kobo Request for /api/kobo/send: Method=%s", r.Method)
|
||||
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
@@ -533,21 +582,21 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) {
|
||||
var req models.KoboSendRequest
|
||||
if err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&req); err != nil {
|
||||
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
||||
a.Logger.Errorf("Error decoding /api/kobo/send request: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error decoding /api/kobo/send request: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
readeckToken, err := a.getReadeckToken(req.AccessToken)
|
||||
if err != nil {
|
||||
http.Error(w, "Invalid access token", http.StatusUnauthorized)
|
||||
a.Logger.Errorf("Error authenticating token for /api/kobo/send: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error authenticating token for /api/kobo/send: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
readeckClient, err := a.newReadeckClient(readeckToken)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to initialize Readeck client", http.StatusInternalServerError)
|
||||
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/send: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error initializing Readeck client for /api/kobo/send: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -592,7 +641,7 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
a.Logger.Warnf("Error processing action '%s' in /api/kobo/send: %v, URL: %s, Params: %v", action, err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Warnf("Error processing action '%s' in /api/kobo/send: %v", action, err)
|
||||
actionResults[i] = false
|
||||
allSucceeded = false
|
||||
} else {
|
||||
@@ -607,7 +656,7 @@ func (a *App) HandleKoboSend(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||||
a.Logger.Errorf("Error encoding response for /api/kobo/send: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error encoding response for /api/kobo/send: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -629,25 +678,25 @@ func (a *App) HandleConvertImage(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
resp, err := client.Get(imageURL)
|
||||
if err != nil {
|
||||
a.Logger.Errorf("Failed to fetch image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Failed to fetch image in /api/convert-image: %T", err)
|
||||
a.returnPlaceholderImage(w, r, "Image fetch failed")
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := resp.Body.Close(); err != nil {
|
||||
a.Logger.Warnf("Error closing response body for image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Warnf("Error closing response body in /api/convert-image: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
a.Logger.Warnf("Failed to fetch image %s in /api/convert-image: status %d, URL: %s, Params: %v", imageURL, resp.StatusCode, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Warnf("Failed to fetch image in /api/convert-image: status %d", resp.StatusCode)
|
||||
a.returnPlaceholderImage(w, r, "Image not found")
|
||||
return
|
||||
}
|
||||
|
||||
img, _, err := image.Decode(resp.Body)
|
||||
if err != nil {
|
||||
a.Logger.Warnf("Failed to decode image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Warnf("Failed to decode image in /api/convert-image: %v", err)
|
||||
a.returnPlaceholderImage(w, r, "Image decoding failed")
|
||||
return
|
||||
}
|
||||
@@ -659,7 +708,7 @@ func (a *App) HandleConvertImage(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
w.Header().Set("Cache-Control", "public, max-age=3600")
|
||||
if err := jpeg.Encode(w, rgbImg, &jpeg.Options{Quality: 85}); err != nil {
|
||||
a.Logger.Errorf("Failed to encode JPEG for image %s in /api/convert-image: %v, URL: %s, Params: %v", imageURL, err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Failed to encode JPEG in /api/convert-image: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -681,7 +730,7 @@ func (a *App) returnPlaceholderImage(w http.ResponseWriter, r *http.Request, mes
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
w.Header().Set("Cache-control", "public, max-age=300")
|
||||
if err := jpeg.Encode(w, img, &jpeg.Options{Quality: 85}); err != nil {
|
||||
a.Logger.Errorf("Error encoding placeholder image: %v, URL: %s, Params: %v", err, r.URL.Path, r.URL.Query())
|
||||
a.Logger.Errorf("Error encoding placeholder image: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -701,41 +750,23 @@ func compareURLs(url1, url2 string) (bool, error) {
|
||||
return u1.Scheme == u2.Scheme && u1.Host == u2.Host && u1.Path == u2.Path, nil
|
||||
}
|
||||
|
||||
func (a *App) getReadeckToken(deviceToken string) (string, error) {
|
||||
func (a *App) getUser(deviceToken string) (config.User, error) {
|
||||
for _, user := range a.Config.Users {
|
||||
if user.Token == deviceToken {
|
||||
return user.ReadeckAccessToken, nil
|
||||
return user, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("unauthorized device token")
|
||||
return config.User{}, fmt.Errorf("unauthorized device token")
|
||||
}
|
||||
|
||||
func (a *App) getReadeckToken(deviceToken string) (string, error) {
|
||||
user, err := a.getUser(deviceToken)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return user.ReadeckAccessToken, nil
|
||||
}
|
||||
|
||||
func (a *App) newReadeckClient(readeckToken string) (*readeck.Client, error) {
|
||||
return readeck.NewClient(a.Config.Readeck.Host, readeckToken, a.Logger, a.ReadeckHTTPClient)
|
||||
}
|
||||
|
||||
func (a *App) HandleDumpAndForward(w http.ResponseWriter, r *http.Request) {
|
||||
a.Logger.Debugf("Dumping request from %s", r.RemoteAddr)
|
||||
a.Logger.Debugf("Method: %s", r.Method)
|
||||
a.Logger.Debugf("URL: %s", r.URL.String())
|
||||
a.Logger.Debugf("Headers: %v", r.Header)
|
||||
|
||||
bodyBytes, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to read request body", http.StatusInternalServerError)
|
||||
a.Logger.Debugf("Error reading request body: %v", err)
|
||||
return
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
|
||||
|
||||
a.Logger.Debugf("Body: %s", string(bodyBytes))
|
||||
|
||||
target, err := url.Parse("https://storeapi.kobo.com")
|
||||
if err != nil {
|
||||
a.Logger.Errorf("Error parsing target URL: %v", err)
|
||||
return
|
||||
}
|
||||
proxy := httputil.NewSingleHostReverseProxy(target)
|
||||
proxy.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
+181
-15
@@ -173,6 +173,7 @@ func TestHandleKoboGet(t *testing.T) {
|
||||
expectedStatus int
|
||||
expectedListSize int
|
||||
expectedTotal int
|
||||
expectedStatuses map[string]string
|
||||
}
|
||||
|
||||
testCases := []koboGetTestCase{
|
||||
@@ -191,6 +192,22 @@ func TestHandleKoboGet(t *testing.T) {
|
||||
expectedListSize: 1, // Only the unread item
|
||||
expectedTotal: 1,
|
||||
},
|
||||
{
|
||||
name: "full sync with opted-in user still excludes archived",
|
||||
reqBody: &models.KoboGetRequest{Count: "10", AccessToken: "remove-archived-device"},
|
||||
mockBookmarksSync: []readeck.BookmarkSync{
|
||||
{ID: "1", Type: "update"},
|
||||
{ID: "2", Type: "update"},
|
||||
},
|
||||
mockBookmarkDetails: map[string]*readeck.Bookmark{
|
||||
"1": {ID: "1", Title: "Unread"},
|
||||
"2": {ID: "2", Title: "Archived", IsArchived: true},
|
||||
},
|
||||
expectedStatus: http.StatusOK,
|
||||
expectedListSize: 1,
|
||||
expectedTotal: 1,
|
||||
expectedStatuses: map[string]string{"1": "0"},
|
||||
},
|
||||
{
|
||||
name: "full sync with favorited item",
|
||||
reqBody: &models.KoboGetRequest{Count: "10", AccessToken: mockDeviceToken}, // No 'Since'
|
||||
@@ -236,6 +253,7 @@ func TestHandleKoboGet(t *testing.T) {
|
||||
expectedStatus: http.StatusOK,
|
||||
expectedListSize: 1, // The deleted status update
|
||||
expectedTotal: 0,
|
||||
expectedStatuses: map[string]string{"1": "2"},
|
||||
},
|
||||
{
|
||||
name: "incremental sync with newly archived",
|
||||
@@ -249,6 +267,24 @@ func TestHandleKoboGet(t *testing.T) {
|
||||
expectedStatus: http.StatusOK,
|
||||
expectedListSize: 1, // The full archived item
|
||||
expectedTotal: 0,
|
||||
expectedStatuses: map[string]string{"1": "1"},
|
||||
},
|
||||
{
|
||||
name: "incremental sync removes archived for opted-in user",
|
||||
reqBody: &models.KoboGetRequest{Since: sinceValue, AccessToken: "remove-archived-device"},
|
||||
mockBookmarksSync: []readeck.BookmarkSync{
|
||||
{ID: "1", Type: "update"},
|
||||
{ID: "2", Type: "update"},
|
||||
{ID: "3", Type: "delete"},
|
||||
},
|
||||
mockBookmarkDetails: map[string]*readeck.Bookmark{
|
||||
"1": {ID: "1", Title: "Archived", IsArchived: true},
|
||||
"2": {ID: "2", Title: "Unread", IsArchived: false},
|
||||
},
|
||||
expectedStatus: http.StatusOK,
|
||||
expectedListSize: 3,
|
||||
expectedTotal: 1,
|
||||
expectedStatuses: map[string]string{"1": "2", "2": "0", "3": "2"},
|
||||
},
|
||||
{
|
||||
name: "incremental sync with GetBookmarksSync error",
|
||||
@@ -327,7 +363,10 @@ func TestHandleKoboGet(t *testing.T) {
|
||||
|
||||
app := NewApp(
|
||||
WithConfig(&config.Config{
|
||||
Users: []config.User{{Token: mockDeviceToken, ReadeckAccessToken: mockPlaintextReadeckToken}},
|
||||
Users: []config.User{
|
||||
{Token: mockDeviceToken, ReadeckAccessToken: mockPlaintextReadeckToken},
|
||||
{Token: "remove-archived-device", ReadeckAccessToken: mockPlaintextReadeckToken, RemoveArchivedFromKobo: true},
|
||||
},
|
||||
Readeck: config.ConfigReadeck{Host: mockServer.URL},
|
||||
}),
|
||||
WithLogger(testLogger),
|
||||
@@ -359,6 +398,13 @@ func TestHandleKoboGet(t *testing.T) {
|
||||
t.Errorf("expected total to be %d, got %d", tc.expectedTotal, resp.Total)
|
||||
}
|
||||
|
||||
for id, wantStatus := range tc.expectedStatuses {
|
||||
item, ok := resp.List[id]
|
||||
if !ok || item.Status != wantStatus {
|
||||
t.Errorf("item %s status = %q (found: %v), want %q", id, item.Status, ok, wantStatus)
|
||||
}
|
||||
}
|
||||
|
||||
// Specific checks for each test case
|
||||
switch tc.name {
|
||||
case "full sync with favorited item":
|
||||
@@ -366,16 +412,6 @@ func TestHandleKoboGet(t *testing.T) {
|
||||
if item.Favorite != "1" {
|
||||
t.Errorf("expected favorited item 'favorite' status to be '1', got '%s'", item.Favorite)
|
||||
}
|
||||
case "incremental sync with deleted":
|
||||
item := resp.List["1"]
|
||||
if item.Status != "2" {
|
||||
t.Errorf("expected deleted item status to be '2', got '%s'", item.Status)
|
||||
}
|
||||
case "incremental sync with newly archived":
|
||||
item := resp.List["1"]
|
||||
if item.Status != "1" {
|
||||
t.Errorf("expected archived item status to be '1', got '%s'", item.Status)
|
||||
}
|
||||
case "full sync with image item":
|
||||
item := resp.List["1"]
|
||||
if item.HasImage != "1" {
|
||||
@@ -389,6 +425,7 @@ func TestHandleKoboGet(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// koboDownloadTestCase defines the structure for test cases in TestHandleKoboDownload.
|
||||
type koboDownloadTestCase struct {
|
||||
name string
|
||||
@@ -414,6 +451,19 @@ func TestHandleKoboDownload(t *testing.T) {
|
||||
},
|
||||
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
|
||||
},
|
||||
{
|
||||
name: "successful download (JSON URL with semicolon)",
|
||||
reqBody: models.KoboDownloadRequest{
|
||||
AccessToken: mockDeviceToken,
|
||||
URL: "http://example.com/article;edition=2?page=1",
|
||||
},
|
||||
contentType: "application/json",
|
||||
expectedStatus: http.StatusOK,
|
||||
mockBookmarks: []readeck.Bookmark{
|
||||
{ID: "1", Title: "Test Article", URL: "http://example.com/article;edition=2?page=1"},
|
||||
},
|
||||
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
|
||||
},
|
||||
{
|
||||
name: "successful download (Form)",
|
||||
reqBody: url.Values{
|
||||
@@ -427,6 +477,49 @@ func TestHandleKoboDownload(t *testing.T) {
|
||||
},
|
||||
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
|
||||
},
|
||||
{
|
||||
name: "successful download (encoded semicolon in form URL path)",
|
||||
reqBody: url.Values{
|
||||
"access_token": {mockDeviceToken},
|
||||
"url": {"http://example.com/article;edition=2?page=1"},
|
||||
},
|
||||
contentType: "application/x-www-form-urlencoded",
|
||||
expectedStatus: http.StatusOK,
|
||||
mockBookmarks: []readeck.Bookmark{
|
||||
{ID: "1", Title: "Test Article", URL: "http://example.com/article;edition=2?page=1"},
|
||||
},
|
||||
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
|
||||
},
|
||||
{
|
||||
name: "successful download (raw semicolon in form URL path)",
|
||||
reqBody: "access_token=" + mockDeviceToken + "&url=http%3A%2F%2Fexample.com%2Farticle;edition=2%3Fpage%3D1",
|
||||
contentType: "application/x-www-form-urlencoded",
|
||||
expectedStatus: http.StatusOK,
|
||||
mockBookmarks: []readeck.Bookmark{
|
||||
{ID: "1", Title: "Test Article", URL: "http://example.com/article;edition=2?page=1"},
|
||||
},
|
||||
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
|
||||
},
|
||||
{
|
||||
name: "successful download (raw semicolon in form URL query)",
|
||||
reqBody: "access_token=" + mockDeviceToken + "&url=http%3A%2F%2Fexample.com%2Farticle1%3Futm_source%3Dmail;utm_medium%3Dreader",
|
||||
contentType: "application/x-www-form-urlencoded",
|
||||
expectedStatus: http.StatusOK,
|
||||
mockBookmarks: []readeck.Bookmark{
|
||||
{ID: "1", Title: "Test Article", URL: "http://example.com/article1?utm_source=mail;utm_medium=reader"},
|
||||
},
|
||||
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
|
||||
},
|
||||
{
|
||||
name: "successful download (captured Kobo form shape)",
|
||||
reqBody: "images=0&consumer_key=fake&access_token=" + mockDeviceToken + "&refresh=0&output=json&url=https://example.com/article?kobo_probe%3D1;part%3D2",
|
||||
contentType: "application/x-www-form-urlencoded",
|
||||
expectedStatus: http.StatusOK,
|
||||
mockBookmarks: []readeck.Bookmark{
|
||||
{ID: "1", Title: "Test Article", URL: "https://example.com/article?kobo_probe=1;part=2"},
|
||||
},
|
||||
mockArticle: `<html><body><h1>Test Article</h1><img src="http://example.com/image.png"></body></html>`,
|
||||
},
|
||||
{
|
||||
name: "missing url",
|
||||
reqBody: models.KoboDownloadRequest{
|
||||
@@ -436,6 +529,12 @@ func TestHandleKoboDownload(t *testing.T) {
|
||||
contentType: "application/json",
|
||||
expectedStatus: http.StatusBadRequest,
|
||||
},
|
||||
{
|
||||
name: "missing access token in raw-semicolon form",
|
||||
reqBody: "url=https://example.com/article;kobo_probe%3D1",
|
||||
contentType: "application/x-www-form-urlencoded",
|
||||
expectedStatus: http.StatusUnauthorized,
|
||||
},
|
||||
{
|
||||
name: "invalid access token",
|
||||
reqBody: models.KoboDownloadRequest{
|
||||
@@ -490,8 +589,14 @@ func TestHandleKoboDownload(t *testing.T) {
|
||||
}
|
||||
body = bytes.NewReader(jsonBody)
|
||||
case "application/x-www-form-urlencoded":
|
||||
formValues := tc.reqBody.(url.Values)
|
||||
body = strings.NewReader(formValues.Encode())
|
||||
switch form := tc.reqBody.(type) {
|
||||
case url.Values:
|
||||
body = strings.NewReader(form.Encode())
|
||||
case string:
|
||||
body = strings.NewReader(form)
|
||||
default:
|
||||
t.Fatalf("unexpected form body type %T", tc.reqBody)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/kobo/download", body)
|
||||
@@ -525,6 +630,69 @@ func TestHandleKoboDownload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFormValuesWithSemicolons(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name, body, wantURL string
|
||||
wantError bool
|
||||
}{
|
||||
{
|
||||
name: "normally encoded URL with semicolon",
|
||||
body: url.Values{"access_token": {"device"}, "url": {"https://example.com/article;edition=2?utm=a;b=c"}}.Encode(),
|
||||
wantURL: "https://example.com/article;edition=2?utm=a;b=c",
|
||||
},
|
||||
{
|
||||
name: "raw semicolon in URL path",
|
||||
body: "access_token=device&url=https%3A%2F%2Fexample.com%2Farticle;edition=2%3Fpage%3D1&output=epub",
|
||||
wantURL: "https://example.com/article;edition=2?page=1",
|
||||
},
|
||||
{
|
||||
name: "Kobo form encodes equals but leaves semicolon raw",
|
||||
body: "images=0&consumer_key=fake&access_token=device&refresh=0&output=json&url=https://example.com/article?kobo_probe%3D1;part%3D2",
|
||||
wantURL: "https://example.com/article?kobo_probe=1;part=2",
|
||||
},
|
||||
{
|
||||
name: "raw semicolon in URL query",
|
||||
body: "url=https%3A%2F%2Fexample.com%2Farticle%3Futm_source%3Dmail;utm_medium%3Dreader%26page%3D2&access_token=device",
|
||||
wantURL: "https://example.com/article?utm_source=mail;utm_medium=reader&page=2",
|
||||
},
|
||||
{
|
||||
name: "semicolon outside URL field",
|
||||
body: "access_token=device;consumer_key=other&url=https%3A%2F%2Fexample.com%2Farticle;edition=2",
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "duplicate URL field",
|
||||
body: "access_token=device&url=https%3A%2F%2Fexample.com%2Fa;edition=2&url=https%3A%2F%2Fexample.com%2Fb",
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid form without raw semicolon stays invalid",
|
||||
body: "access_token=device&url=https%3A%2F%2Fexample.com%2Fa&output=%zz",
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid percent escape",
|
||||
body: "access_token=device&url=https%3A%2F%2Fexample.com%2Fa;edition=2&output=%zz",
|
||||
wantError: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/kobo/download", strings.NewReader(tt.body))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
form, err := downloadFormValues(req, []byte(tt.body))
|
||||
if tt.wantError {
|
||||
if err == nil {
|
||||
t.Fatalf("expected a form parsing error, got %v", form)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || form.Get("url") != tt.wantURL || form.Get("access_token") != "device" {
|
||||
t.Errorf("URL = %q, token = %q, err = %v; want URL %q and token device", form.Get("url"), form.Get("access_token"), err, tt.wantURL)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// koboSendTestCase defines the structure for test cases in TestHandleKoboSend.
|
||||
type koboSendTestCase struct {
|
||||
name string
|
||||
@@ -842,5 +1010,3 @@ func TestHandleConvertImage(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
func (a *App) HandleBookSync(w http.ResponseWriter, r *http.Request) {
|
||||
token := r.PathValue("deviceToken")
|
||||
upstream, known := a.bookSyncUpstream(token)
|
||||
if !known {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
target, err := url.Parse(upstream)
|
||||
if err != nil {
|
||||
a.Logger.Errorf("Invalid configured book sync upstream: %T", err)
|
||||
http.Error(w, "Bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
http.StripPrefix("/booksync/"+token, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
a.serveKoboProxy(w, r, target, token)
|
||||
})).ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func (a *App) bookSyncUpstream(token string) (string, bool) {
|
||||
for _, user := range a.Config.Users {
|
||||
if user.Token == token {
|
||||
if user.BookSyncURL != "" {
|
||||
return user.BookSyncURL, true
|
||||
}
|
||||
return "https://" + storeAPIHost, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
storeAPIHost = "storeapi.kobo.com"
|
||||
maxInitializationBody = 8 << 20
|
||||
)
|
||||
|
||||
var koboTransport = func() *http.Transport {
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.ResponseHeaderTimeout = 20 * time.Second
|
||||
return transport
|
||||
}()
|
||||
|
||||
func (a *App) HandleStoreProxy(w http.ResponseWriter, r *http.Request) {
|
||||
a.serveKoboProxy(w, r, &url.URL{Scheme: "https", Host: storeAPIHost}, "")
|
||||
}
|
||||
|
||||
// serveKoboProxy expects the public route prefix to have been removed from r.URL.
|
||||
func (a *App) serveKoboProxy(w http.ResponseWriter, r *http.Request, target *url.URL, deviceToken string) {
|
||||
path := r.URL.Path
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
proxy := a.newKoboProxy(target, path)
|
||||
if deviceToken != "" {
|
||||
originalDirector := proxy.Director
|
||||
proxy.Director = func(req *http.Request) {
|
||||
originalDirector(req)
|
||||
// Book services can build absolute URLs from forwarded headers.
|
||||
// Use the upstream origin, not the public ingress host.
|
||||
req.Header.Del("Forwarded")
|
||||
req.Header.Del("X-Forwarded-Port")
|
||||
req.Header.Del("X-Forwarded-Prefix")
|
||||
req.Header.Set("X-Forwarded-Host", target.Host)
|
||||
req.Header.Set("X-Forwarded-Proto", target.Scheme)
|
||||
}
|
||||
}
|
||||
if path == "/v1/initialization" {
|
||||
origin, err := a.publicOrigin(r)
|
||||
if err != nil {
|
||||
http.Error(w, "Invalid public origin", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
configureInitializationProxy(proxy, origin+"/instapaper-proxy/instapaper")
|
||||
}
|
||||
proxy.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func (a *App) newKoboProxy(target *url.URL, path string) *httputil.ReverseProxy {
|
||||
proxy := httputil.NewSingleHostReverseProxy(target)
|
||||
originalDirector := proxy.Director
|
||||
proxy.Director = func(req *http.Request) {
|
||||
req.URL.Path = path
|
||||
originalDirector(req)
|
||||
req.Host = target.Host
|
||||
}
|
||||
if a.ProxyTransport != nil {
|
||||
proxy.Transport = a.ProxyTransport
|
||||
} else {
|
||||
proxy.Transport = koboTransport
|
||||
}
|
||||
proxy.ErrorHandler = func(w http.ResponseWriter, _ *http.Request, err error) {
|
||||
a.Logger.Errorf("Kobo upstream proxy failed: %T", err)
|
||||
http.Error(w, "Bad gateway", http.StatusBadGateway)
|
||||
}
|
||||
return proxy
|
||||
}
|
||||
|
||||
func configureInitializationProxy(proxy *httputil.ReverseProxy, instapaperURL string) {
|
||||
originalDirector := proxy.Director
|
||||
proxy.Director = func(req *http.Request) {
|
||||
originalDirector(req)
|
||||
// Request a complete document so its body can be rewritten.
|
||||
req.Header.Set("Accept-Encoding", "identity")
|
||||
req.Header.Del("If-None-Match")
|
||||
req.Header.Del("If-Modified-Since")
|
||||
req.Header.Del("Range")
|
||||
req.Header.Del("If-Range")
|
||||
}
|
||||
proxy.ModifyResponse = func(resp *http.Response) error {
|
||||
if resp.StatusCode == http.StatusPartialContent {
|
||||
return errors.New("partial initialization response")
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil
|
||||
}
|
||||
return rewriteInitialization(resp, instapaperURL)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) publicOrigin(r *http.Request) (string, error) {
|
||||
if a.Config.PublicURL != "" {
|
||||
return strings.TrimRight(a.Config.PublicURL, "/"), nil
|
||||
}
|
||||
|
||||
scheme := r.Header.Get("X-Forwarded-Proto")
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
if r.TLS != nil {
|
||||
scheme = "https"
|
||||
}
|
||||
}
|
||||
host := r.Header.Get("X-Forwarded-Host")
|
||||
if host == "" {
|
||||
host = r.Host
|
||||
}
|
||||
if scheme != "http" && scheme != "https" || host == "" || strings.ContainsAny(host, "/@?#, \t\r\n\"\\") {
|
||||
return "", errors.New("invalid forwarded origin")
|
||||
}
|
||||
parsed, err := url.Parse("//" + host)
|
||||
if err != nil || parsed.Hostname() == "" || parsed.Host != host {
|
||||
return "", errors.New("invalid forwarded host")
|
||||
}
|
||||
return scheme + "://" + host, nil
|
||||
}
|
||||
|
||||
func rewriteInitialization(resp *http.Response, instapaperURL string) error {
|
||||
originalBody := resp.Body
|
||||
defer func() { _ = originalBody.Close() }()
|
||||
|
||||
var reader io.Reader = originalBody
|
||||
compressed := resp.Header.Get("Content-Encoding") == "gzip"
|
||||
if compressed {
|
||||
gz, err := gzip.NewReader(originalBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = gz.Close() }()
|
||||
reader = gz
|
||||
} else if encoding := resp.Header.Get("Content-Encoding"); encoding != "" && encoding != "identity" {
|
||||
return fmt.Errorf("unsupported initialization encoding: %s", encoding)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(reader, maxInitializationBody+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(body) > maxInitializationBody {
|
||||
return errors.New("initialization response too large")
|
||||
}
|
||||
encodedURL, err := json.Marshal(instapaperURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body = bytes.ReplaceAll(body, []byte("https://www.instapaper.com"), encodedURL[1:len(encodedURL)-1])
|
||||
if compressed {
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
if _, err := gw.Write(body); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := gw.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
body = buf.Bytes()
|
||||
} else {
|
||||
resp.Header.Del("Content-Encoding")
|
||||
}
|
||||
|
||||
resp.Body = io.NopCloser(bytes.NewReader(body))
|
||||
resp.ContentLength = int64(len(body))
|
||||
resp.Header.Set("Content-Length", strconv.Itoa(len(body)))
|
||||
for _, key := range []string{"ETag", "Last-Modified", "Content-MD5", "Digest", "Content-Range", "Accept-Ranges"} {
|
||||
resp.Header.Del(key)
|
||||
}
|
||||
resp.Header.Set("Cache-Control", "private, no-store")
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"readeckobo/internal/config"
|
||||
)
|
||||
|
||||
func storeHandler(a *App) http.Handler {
|
||||
return http.StripPrefix("/instapaper-proxy/storeapi", http.HandlerFunc(a.HandleStoreProxy))
|
||||
}
|
||||
|
||||
func TestHandleStoreProxy(t *testing.T) {
|
||||
t.Run("forwards requests to the fixed store origin", func(t *testing.T) {
|
||||
body := "book sync response"
|
||||
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
|
||||
RoundTripFunc: func(req *http.Request) (*http.Response, error) {
|
||||
if req.URL.Scheme != "https" || req.URL.Host != storeAPIHost || req.Host != storeAPIHost || req.URL.Path != "/v1/library/sync" || req.URL.RawQuery != "page=2" {
|
||||
t.Errorf("unexpected upstream request: URL=%s Host=%q", req.URL, req.Host)
|
||||
}
|
||||
data, err := io.ReadAll(req.Body)
|
||||
if err != nil || string(data) != "payload" {
|
||||
t.Errorf("upstream body = %q, err = %v", data, err)
|
||||
}
|
||||
if req.Header.Get("X-Test") != "forwarded" {
|
||||
t.Errorf("X-Test = %q", req.Header.Get("X-Test"))
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusAccepted, Header: http.Header{"Content-Type": {"application/octet-stream"}}, Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
},
|
||||
}))
|
||||
req := httptest.NewRequest(http.MethodPost, "/instapaper-proxy/storeapi/v1/library/sync?page=2", strings.NewReader("payload"))
|
||||
req.Header.Set("X-Test", "forwarded")
|
||||
rr := httptest.NewRecorder()
|
||||
storeHandler(a).ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusAccepted || rr.Body.String() != body {
|
||||
t.Errorf("status = %d, body length = %d; want 202 and %d", rr.Code, rr.Body.Len(), len(body))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("streams an ordinary response before upstream completion", func(t *testing.T) {
|
||||
reader, writer := io.Pipe()
|
||||
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
|
||||
RoundTripFunc: func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{"Content-Type": {"application/octet-stream"}},
|
||||
Body: reader,
|
||||
ContentLength: -1,
|
||||
}, nil
|
||||
},
|
||||
}))
|
||||
server := httptest.NewServer(storeHandler(a))
|
||||
defer server.Close()
|
||||
defer func() { _ = writer.Close() }()
|
||||
|
||||
firstChunk := bytes.Repeat([]byte("x"), 32*1024)
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
resp, err := server.Client().Get(server.URL + "/instapaper-proxy/storeapi/v1/library/sync")
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
result <- errors.New("unexpected status")
|
||||
return
|
||||
}
|
||||
data := make([]byte, len(firstChunk))
|
||||
_, err = io.ReadFull(resp.Body, data)
|
||||
if err == nil && !bytes.Equal(data, firstChunk) {
|
||||
err = errors.New("unexpected bytes in first chunk")
|
||||
}
|
||||
result <- err
|
||||
}()
|
||||
go func() { _, _ = writer.Write(firstChunk) }()
|
||||
|
||||
select {
|
||||
case err := <-result:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("proxy did not deliver bytes before upstream finished")
|
||||
}
|
||||
})
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
gzip bool
|
||||
publicURL string
|
||||
wantOrigin string
|
||||
}{
|
||||
{name: "plaintext uses configured public origin", publicURL: "https://reader.example.com/", wantOrigin: "https://reader.example.com"},
|
||||
{name: "gzipped response uses trusted forwarded origin", gzip: true, wantOrigin: "https://reader.example.com"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
upstreamBody := []byte(`{"instapaper":"https://www.instapaper.com"}`)
|
||||
responseHeader := http.Header{
|
||||
"Content-Type": {"application/json"},
|
||||
"Etag": {`"old-tag"`},
|
||||
"Last-Modified": {"Tue, 01 Jan 2019 00:00:00 GMT"},
|
||||
}
|
||||
if tt.gzip {
|
||||
var buf bytes.Buffer
|
||||
writer := gzip.NewWriter(&buf)
|
||||
if _, err := writer.Write(upstreamBody); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upstreamBody = buf.Bytes()
|
||||
responseHeader.Set("Content-Encoding", "gzip")
|
||||
}
|
||||
responseHeader.Set("Content-Length", strconv.Itoa(len(upstreamBody)))
|
||||
a := NewApp(WithConfig(&config.Config{PublicURL: tt.publicURL}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
|
||||
RoundTripFunc: func(req *http.Request) (*http.Response, error) {
|
||||
if req.URL.Path != "/v1/initialization" || req.URL.RawQuery != "locale=en" || req.Header.Get("Accept-Encoding") != "identity" || req.Header.Get("If-None-Match") != "" || req.Header.Get("Range") != "" || req.Header.Get("If-Range") != "" {
|
||||
t.Errorf("unexpected initialization request: %s, headers: %v", req.URL, req.Header)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: responseHeader, Body: io.NopCloser(bytes.NewReader(upstreamBody))}, nil
|
||||
},
|
||||
}))
|
||||
req := httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization?locale=en", nil)
|
||||
req.Host = "untrusted.example.com"
|
||||
forwardedHost := "reader.example.com"
|
||||
if tt.publicURL != "" {
|
||||
forwardedHost = "ignored.example.com"
|
||||
}
|
||||
req.Header.Set("X-Forwarded-Host", forwardedHost)
|
||||
req.Header.Set("X-Forwarded-Proto", "https")
|
||||
req.Header.Set("If-None-Match", `"old-tag"`)
|
||||
req.Header.Set("Range", "bytes=0-50")
|
||||
req.Header.Set("If-Range", `"old-tag"`)
|
||||
rr := httptest.NewRecorder()
|
||||
storeHandler(a).ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if got := rr.Header().Get("Content-Length"); got != strconv.Itoa(rr.Body.Len()) {
|
||||
t.Errorf("Content-Length = %q, body size = %d", got, rr.Body.Len())
|
||||
}
|
||||
if rr.Header().Get("ETag") != "" || rr.Header().Get("Last-Modified") != "" || rr.Header().Get("Cache-Control") != "private, no-store" {
|
||||
t.Errorf("stale response headers: %v", rr.Header())
|
||||
}
|
||||
var result []byte
|
||||
if tt.gzip {
|
||||
if rr.Header().Get("Content-Encoding") != "gzip" {
|
||||
t.Errorf("Content-Encoding = %q", rr.Header().Get("Content-Encoding"))
|
||||
}
|
||||
reader, err := gzip.NewReader(rr.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err = io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = reader.Close()
|
||||
} else {
|
||||
if rr.Header().Get("Content-Encoding") != "" {
|
||||
t.Errorf("Content-Encoding = %q", rr.Header().Get("Content-Encoding"))
|
||||
}
|
||||
result = rr.Body.Bytes()
|
||||
}
|
||||
want := `{"instapaper":"` + tt.wantOrigin + `/instapaper-proxy/instapaper"}`
|
||||
if string(result) != want {
|
||||
t.Errorf("body = %q, want %q", result, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("preserves escaped path and query", func(t *testing.T) {
|
||||
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
|
||||
RoundTripFunc: func(req *http.Request) (*http.Response, error) {
|
||||
if req.URL.Path != "/v1//books" || req.URL.EscapedPath() != "/v1/%2Fbooks" || req.URL.RawQuery != "a=1%3B2" {
|
||||
t.Errorf("upstream URL = %q, RawPath = %q", req.URL, req.URL.RawPath)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusNoContent, Header: http.Header{}, Body: http.NoBody}, nil
|
||||
},
|
||||
}))
|
||||
rr := httptest.NewRecorder()
|
||||
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/%2Fbooks?a=1%3B2", nil))
|
||||
if rr.Code != http.StatusNoContent {
|
||||
t.Errorf("status = %d, want 204", rr.Code)
|
||||
}
|
||||
})
|
||||
|
||||
for _, tt := range []struct {
|
||||
name, encoding, body string
|
||||
}{
|
||||
{name: "rejects invalid gzip initialization", encoding: "gzip", body: "not gzip"},
|
||||
{name: "rejects unsupported initialization encoding", encoding: "br", body: "compressed"},
|
||||
{name: "limits initialization size", body: strings.Repeat("x", maxInitializationBody+1)},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
a := NewApp(WithConfig(&config.Config{PublicURL: "https://reader.example.com"}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
|
||||
RoundTripFunc: func(*http.Request) (*http.Response, error) {
|
||||
header := http.Header{}
|
||||
header.Set("Content-Encoding", tt.encoding)
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(strings.NewReader(tt.body))}, nil
|
||||
},
|
||||
}))
|
||||
rr := httptest.NewRecorder()
|
||||
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil))
|
||||
if rr.Code != http.StatusBadGateway {
|
||||
t.Errorf("status = %d, want 502", rr.Code)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("partial initialization is rejected", func(t *testing.T) {
|
||||
a := NewApp(WithConfig(&config.Config{PublicURL: "https://reader.example.com"}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
|
||||
RoundTripFunc: func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: http.StatusPartialContent, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("partial"))}, nil
|
||||
},
|
||||
}))
|
||||
rr := httptest.NewRecorder()
|
||||
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil))
|
||||
if rr.Code != http.StatusBadGateway {
|
||||
t.Errorf("status = %d, want 502", rr.Code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-OK initialization passes through without rewriting", func(t *testing.T) {
|
||||
body := `{"url":"https://www.instapaper.com"}`
|
||||
a := NewApp(WithConfig(&config.Config{PublicURL: "https://reader.example.com"}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
|
||||
RoundTripFunc: func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: http.StatusNotFound, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
},
|
||||
}))
|
||||
rr := httptest.NewRecorder()
|
||||
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil))
|
||||
if rr.Code != http.StatusNotFound || rr.Body.String() != body {
|
||||
t.Errorf("status = %d, body = %q; want 404 and unchanged body", rr.Code, rr.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("encodes rewritten URLs as valid JSON strings", func(t *testing.T) {
|
||||
urlWithQuote := `https://reader.example.com/path?value="quoted"`
|
||||
resp := &http.Response{Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"instapaper":"https://www.instapaper.com"}`))}
|
||||
if err := rewriteInitialization(resp, urlWithQuote); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded struct {
|
||||
Instapaper string `json:"instapaper"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
|
||||
t.Fatalf("rewritten response is invalid JSON: %v", err)
|
||||
}
|
||||
if decoded.Instapaper != urlWithQuote {
|
||||
t.Errorf("instapaper URL = %q, want %q", decoded.Instapaper, urlWithQuote)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects malformed forwarded origin", func(t *testing.T) {
|
||||
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger))
|
||||
req := httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil)
|
||||
req.Header.Set("X-Forwarded-Proto", "javascript")
|
||||
rr := httptest.NewRecorder()
|
||||
storeHandler(a).ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusBadGateway {
|
||||
t.Errorf("status = %d, want 502", rr.Code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects quoted forwarded host", func(t *testing.T) {
|
||||
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger))
|
||||
req := httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/initialization", nil)
|
||||
req.Header.Set("X-Forwarded-Host", `reader.example.com"injected`)
|
||||
rr := httptest.NewRecorder()
|
||||
storeHandler(a).ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusBadGateway {
|
||||
t.Errorf("status = %d, want 502", rr.Code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("returns 502 on upstream failure", func(t *testing.T) {
|
||||
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
|
||||
RoundTripFunc: func(*http.Request) (*http.Response, error) { return nil, errors.New("upstream unavailable") },
|
||||
}))
|
||||
rr := httptest.NewRecorder()
|
||||
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v1/library/sync", nil))
|
||||
if rr.Code != http.StatusBadGateway {
|
||||
t.Errorf("status = %d, want 502", rr.Code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("does not rewrite other initialization-like paths", func(t *testing.T) {
|
||||
body := `{"url":"https://www.instapaper.com"}`
|
||||
a := NewApp(WithConfig(&config.Config{}), WithLogger(testLogger), WithProxyTransport(&MockRoundTripper{
|
||||
RoundTripFunc: func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
},
|
||||
}))
|
||||
rr := httptest.NewRecorder()
|
||||
storeHandler(a).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi/v2/initialization", nil))
|
||||
if rr.Body.String() != body {
|
||||
t.Errorf("body = %q, want %q", rr.Body.String(), body)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBookSyncConfiguration(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extra string
|
||||
userExtra string
|
||||
wantErr string
|
||||
wantBookSyncURL string
|
||||
wantPublicURL string
|
||||
wantRemove bool
|
||||
}{
|
||||
{name: "no optional settings"},
|
||||
{name: "old disabled switch is ignored", extra: "book_sync: false\n"},
|
||||
{name: "old enabled switch is ignored", extra: "book_sync: true\n"},
|
||||
{name: "custom sync endpoint with path", userExtra: " book_sync_url: https://books.example.com/api/kobo/secret\n", wantBookSyncURL: "https://books.example.com/api/kobo/secret"},
|
||||
{name: "http sync endpoint is allowed", userExtra: " book_sync_url: http://localhost:8081/kobo\n", wantBookSyncURL: "http://localhost:8081/kobo"},
|
||||
{name: "public URL and archived removal", extra: "public_url: https://reader.example.com/\n", userExtra: " remove_archived_from_kobo: true\n", wantPublicURL: "https://reader.example.com/", wantRemove: true},
|
||||
{name: "invalid sync scheme", userExtra: " book_sync_url: ftp://books.example.com\n", wantErr: "BookSyncURL"},
|
||||
{name: "relative sync URL", userExtra: " book_sync_url: /api/kobo/device\n", wantErr: "BookSyncURL"},
|
||||
{name: "sync URL missing host", userExtra: " book_sync_url: https:/kobo\n", wantErr: "BookSyncURL"},
|
||||
{name: "invalid sync port", userExtra: " book_sync_url: https://books.example.com:bad/kobo\n", wantErr: "BookSyncURL"},
|
||||
{name: "invalid public scheme", extra: "public_url: ftp://reader.example.com\n", wantErr: "PublicURL"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data := "readeck:\n host: https://readeck.example.com\n" + tt.extra + "users:\n - token: device\n readeck_access_token: readeck\n" + tt.userExtra
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(data), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := Load(path)
|
||||
if tt.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("Load() error = %v, want error mentioning %q", err, tt.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := cfg.Users[0].BookSyncURL; got != tt.wantBookSyncURL {
|
||||
t.Errorf("BookSyncURL = %q, want %q", got, tt.wantBookSyncURL)
|
||||
}
|
||||
if cfg.PublicURL != tt.wantPublicURL {
|
||||
t.Errorf("PublicURL = %q, want %q", cfg.PublicURL, tt.wantPublicURL)
|
||||
}
|
||||
if cfg.Users[0].RemoveArchivedFromKobo != tt.wantRemove {
|
||||
t.Errorf("RemoveArchivedFromKobo = %v, want %v", cfg.Users[0].RemoveArchivedFromKobo, tt.wantRemove)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -12,8 +12,10 @@ import (
|
||||
)
|
||||
|
||||
type User struct {
|
||||
Token string `koanf:"token" validate:"required"`
|
||||
ReadeckAccessToken string `koanf:"readeck_access_token" validate:"required"`
|
||||
Token string `koanf:"token" validate:"required"`
|
||||
ReadeckAccessToken string `koanf:"readeck_access_token" validate:"required"`
|
||||
BookSyncURL string `koanf:"book_sync_url" validate:"omitempty,http_url"`
|
||||
RemoveArchivedFromKobo bool `koanf:"remove_archived_from_kobo"`
|
||||
}
|
||||
|
||||
type ConfigReadeck struct {
|
||||
@@ -21,12 +23,13 @@ type ConfigReadeck struct {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Readeck ConfigReadeck `koanf:"readeck"`
|
||||
Server struct {
|
||||
Readeck ConfigReadeck `koanf:"readeck"`
|
||||
Server struct {
|
||||
Port int `koanf:"port" validate:"min=1,max=65535"`
|
||||
} `koanf:"server"`
|
||||
Users []User `koanf:"users" validate:"required,min=1,dive"`
|
||||
LogLevel string `koanf:"log_level" validate:"oneof=error warn info debug"`
|
||||
Users []User `koanf:"users" validate:"required,min=1,dive"`
|
||||
LogLevel string `koanf:"log_level" validate:"oneof=error warn info debug"`
|
||||
PublicURL string `koanf:"public_url" validate:"omitempty,http_url"`
|
||||
}
|
||||
|
||||
func (c *Config) Validate() error {
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httputil" // Added import
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -129,26 +128,12 @@ func (c *Client) doRequestRaw(ctx context.Context, method, path string, queryPar
|
||||
req.Header.Set("Content-Type", "application/json") // Ensure Content-Type is set for requests with a body
|
||||
}
|
||||
|
||||
// Log the outgoing request for debugging
|
||||
dump, err := httputil.DumpRequestOut(req, true)
|
||||
if err != nil {
|
||||
c.Logger.Errorf("Failed to dump outgoing request: %v", err)
|
||||
} else {
|
||||
c.Logger.Debugf("Outgoing Readeck API Request:\n%s", dump)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
resp, err := c.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to execute request: %w", err)
|
||||
}
|
||||
|
||||
// Log the incoming response for debugging
|
||||
// dumpResp, err := httputil.DumpResponse(resp, true)
|
||||
// if err != nil {
|
||||
// c.Logger.Errorf("Failed to dump incoming response: %v", err)
|
||||
// } else {
|
||||
// c.Logger.Debugf("Incoming Readeck API Response:\n%s", dumpResp)
|
||||
// }
|
||||
c.Logger.Debugf("Readeck %s %s: HTTP %d in %s", req.Method, req.URL.Path, resp.StatusCode, time.Since(start))
|
||||
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
package readeck
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -224,6 +228,54 @@ func TestCreateBookmark(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawRequestDebugLogOmitsSecrets(t *testing.T) {
|
||||
const token = "private-readeck-token"
|
||||
const query = "private-query-value"
|
||||
const body = "private-body-value"
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer "+token {
|
||||
t.Errorf("Authorization header = %q, want bearer token", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("q"); got != query {
|
||||
t.Errorf("query value = %q, want %q", got, query)
|
||||
}
|
||||
var data map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
|
||||
t.Errorf("decode request body: %v", err)
|
||||
} else if data["data"] != body {
|
||||
t.Errorf("body data = %q, want %q", data["data"], body)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
var output bytes.Buffer
|
||||
previousOutput := log.Writer()
|
||||
log.SetOutput(&output)
|
||||
defer log.SetOutput(previousOutput)
|
||||
|
||||
client, err := NewClient(server.URL, token, logger.New(logger.DEBUG), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := client.doRequestRaw(context.Background(), http.MethodPost, "/api/bookmarks/sync", url.Values{"q": {query}}, map[string]string{"data": body})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
got := output.String()
|
||||
if !strings.Contains(got, "Readeck POST /api/bookmarks/sync: HTTP 200 in ") {
|
||||
t.Errorf("missing request summary in debug log: %q", got)
|
||||
}
|
||||
for _, secret := range []string{token, query, body, "Authorization"} {
|
||||
if strings.Contains(got, secret) {
|
||||
t.Errorf("debug log contains private request data: %q", secret)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBookmarksWithIsArchived(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/bookmarks" {
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
package webserver
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"readeckobo/internal/app"
|
||||
"readeckobo/internal/config"
|
||||
"readeckobo/internal/logger"
|
||||
)
|
||||
|
||||
func TestBookSyncRoutes(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name, path, scheme, host, upstreamPath, escapedPath, query string
|
||||
wantStatus int
|
||||
}{
|
||||
{"custom endpoint with path prefix", "/booksync/books/v1/library/sync?page=2", "https", "books.example.com", "/api/kobo/device/v1/library/sync", "/api/kobo/device/v1/library/sync", "page=2", http.StatusAccepted},
|
||||
{"custom HTTP endpoint", "/booksync/httpbook/v1/library/sync", "http", "localhost:8081", "/kobo/v1/library/sync", "/kobo/v1/library/sync", "", http.StatusAccepted},
|
||||
{"escaped suffix", "/booksync/books/v1/%2Fbooks?a=1%3B2", "https", "books.example.com", "/api/kobo/device/v1//books", "/api/kobo/device/v1/%2Fbooks", "a=1%3B2", http.StatusAccepted},
|
||||
{"custom endpoint root", "/booksync/books/", "https", "books.example.com", "/api/kobo/device/", "/api/kobo/device/", "", http.StatusAccepted},
|
||||
{"known token uses Kobo Store", "/booksync/store/v1/library/sync?page=3", "https", "storeapi.kobo.com", "/v1/library/sync", "/v1/library/sync", "page=3", http.StatusAccepted},
|
||||
{"unknown token", "/booksync/unknown/v1/library/sync", "", "", "", "", "", http.StatusNotFound},
|
||||
{"partial token does not match", "/booksync/book/v1/library/sync", "", "", "", "", "", http.StatusNotFound},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var calls int
|
||||
application := app.NewApp(
|
||||
app.WithConfig(&config.Config{Users: []config.User{
|
||||
{Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"},
|
||||
{Token: "httpbook", BookSyncURL: "http://localhost:8081/kobo"},
|
||||
{Token: "store"},
|
||||
}}),
|
||||
app.WithLogger(logger.New(logger.ERROR)),
|
||||
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if req.URL.Scheme != tt.scheme || req.URL.Host != tt.host || req.Host != tt.host || req.URL.Path != tt.upstreamPath || req.URL.EscapedPath() != tt.escapedPath || req.URL.RawQuery != tt.query {
|
||||
t.Errorf("upstream URL = %s, Host = %q; want %s://%s, path %q, escaped %q, query %q", req.URL, req.Host, tt.scheme, tt.host, tt.upstreamPath, tt.escapedPath, tt.query)
|
||||
}
|
||||
if req.Method != http.MethodPost || req.Header.Get("X-Test") != "forwarded" {
|
||||
t.Errorf("upstream method = %s, X-Test = %q", req.Method, req.Header.Get("X-Test"))
|
||||
}
|
||||
body, err := io.ReadAll(req.Body)
|
||||
if err != nil || string(body) != "payload" {
|
||||
t.Errorf("upstream body = %q, err = %v", body, err)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusAccepted, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("book response"))}, nil
|
||||
})),
|
||||
)
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, tt.path, strings.NewReader("payload"))
|
||||
req.Header.Set("X-Test", "forwarded")
|
||||
NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, req)
|
||||
if rr.Code != tt.wantStatus {
|
||||
t.Errorf("status = %d, want %d", rr.Code, tt.wantStatus)
|
||||
}
|
||||
if tt.host == "" {
|
||||
if calls != 0 {
|
||||
t.Errorf("unknown token reached upstream (%d calls)", calls)
|
||||
}
|
||||
} else if calls != 1 || rr.Body.String() != "book response" {
|
||||
t.Errorf("upstream calls = %d, body = %q; want one call and book response", calls, rr.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBookSyncDoesNotRetryConfiguredService(t *testing.T) {
|
||||
var calls int
|
||||
application := app.NewApp(
|
||||
app.WithConfig(&config.Config{Users: []config.User{{Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"}}}),
|
||||
app.WithLogger(logger.New(logger.ERROR)),
|
||||
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if req.URL.Host != "books.example.com" {
|
||||
t.Errorf("upstream host = %q, want books.example.com", req.URL.Host)
|
||||
}
|
||||
return nil, errors.New("book service unavailable")
|
||||
})),
|
||||
)
|
||||
rr := httptest.NewRecorder()
|
||||
NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/booksync/books/v1/library/sync", nil))
|
||||
if rr.Code != http.StatusBadGateway || calls != 1 {
|
||||
t.Errorf("status = %d, upstream calls = %d; want 502 and one call", rr.Code, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBookSyncInitialization(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name, token, upstreamHost, upstreamPath string
|
||||
gzip bool
|
||||
}{
|
||||
{"custom plaintext", "books", "books.example.com", "/api/kobo/device/v1/initialization", false},
|
||||
{"custom gzip", "books", "books.example.com", "/api/kobo/device/v1/initialization", true},
|
||||
{"store fallback", "store", "storeapi.kobo.com", "/v1/initialization", false},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
body := []byte(`{"instapaper":"https://www.instapaper.com"}`)
|
||||
header := http.Header{"Content-Type": {"application/json"}, "Etag": {`"old"`}}
|
||||
if tt.gzip {
|
||||
var b bytes.Buffer
|
||||
writer := gzip.NewWriter(&b)
|
||||
if _, err := writer.Write(body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body = b.Bytes()
|
||||
header.Set("Content-Encoding", "gzip")
|
||||
}
|
||||
application := app.NewApp(
|
||||
app.WithConfig(&config.Config{PublicURL: "https://reader.example.com", Users: []config.User{
|
||||
{Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"}, {Token: "store"},
|
||||
}}),
|
||||
app.WithLogger(logger.New(logger.ERROR)),
|
||||
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
if req.URL.Host != tt.upstreamHost || req.URL.Path != tt.upstreamPath || req.URL.RawQuery != "locale=en" || req.Header.Get("Accept-Encoding") != "identity" || req.Header.Get("Range") != "" || req.Header.Get("If-None-Match") != "" {
|
||||
t.Errorf("unexpected initialization upstream: URL=%s, headers=%v", req.URL, req.Header)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(bytes.NewReader(body))}, nil
|
||||
})),
|
||||
)
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/booksync/"+tt.token+"/v1/initialization?locale=en", nil)
|
||||
req.Header.Set("If-None-Match", `"old"`)
|
||||
req.Header.Set("Range", "bytes=0-10")
|
||||
NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK || rr.Header().Get("ETag") != "" {
|
||||
t.Fatalf("status = %d, headers = %v", rr.Code, rr.Header())
|
||||
}
|
||||
if got := rr.Header().Get("Content-Length"); got != strconv.Itoa(rr.Body.Len()) {
|
||||
t.Errorf("Content-Length = %q, body size = %d", got, rr.Body.Len())
|
||||
}
|
||||
var result []byte
|
||||
var err error
|
||||
if tt.gzip {
|
||||
if rr.Header().Get("Content-Encoding") != "gzip" {
|
||||
t.Errorf("Content-Encoding = %q, want gzip", rr.Header().Get("Content-Encoding"))
|
||||
}
|
||||
reader, gzipErr := gzip.NewReader(rr.Body)
|
||||
if gzipErr != nil {
|
||||
t.Fatal(gzipErr)
|
||||
}
|
||||
result, err = io.ReadAll(reader)
|
||||
_ = reader.Close()
|
||||
} else {
|
||||
result = rr.Body.Bytes()
|
||||
}
|
||||
if err != nil || string(result) != `{"instapaper":"https://reader.example.com/instapaper-proxy/instapaper"}` {
|
||||
t.Errorf("rewritten body = %q, err = %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBookSyncInitializationPreservesBookResourcesPerUser(t *testing.T) {
|
||||
for _, tt := range []struct{ token, host, key string }{
|
||||
{"alice", "books-alice.example.com", "alice-key"},
|
||||
{"bob", "books-bob.example.com", "bob-key"},
|
||||
} {
|
||||
t.Run(tt.token, func(t *testing.T) {
|
||||
upstream := "https://" + tt.host + "/kobo/" + tt.key
|
||||
body := `{"Resources":{"instapaper_env_url":"https://www.instapaper.com/api/kobo",` +
|
||||
`"image_host":"https://` + tt.host + `",` +
|
||||
`"image_url_template":"` + upstream + `/v1/books/{ImageId}/thumbnail/{Width}/{Height}/false/image.jpg",` +
|
||||
`"image_url_quality_template":"` + upstream + `/v1/books/{ImageId}/thumbnail/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg",` +
|
||||
`"other_resource":"https://cdn.kobo.com/book.jpg"}}`
|
||||
application := app.NewApp(
|
||||
app.WithConfig(&config.Config{PublicURL: "https://reader.example.com", Users: []config.User{
|
||||
{Token: "alice", BookSyncURL: "https://books-alice.example.com/kobo/alice-key"},
|
||||
{Token: "bob", BookSyncURL: "https://books-bob.example.com/kobo/bob-key"},
|
||||
}}),
|
||||
app.WithLogger(logger.New(logger.ERROR)),
|
||||
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
if req.URL.Host != tt.host || req.URL.Path != "/kobo/"+tt.key+"/v1/initialization" {
|
||||
t.Errorf("upstream URL = %s, want %s/v1/initialization", req.URL, upstream)
|
||||
}
|
||||
if req.Header.Get("X-Forwarded-Host") != tt.host || req.Header.Get("X-Forwarded-Proto") != "https" {
|
||||
t.Errorf("forwarded host/proto = %q/%q; want %s/https", req.Header.Get("X-Forwarded-Host"), req.Header.Get("X-Forwarded-Proto"), tt.host)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}}, Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
})),
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodGet, "/booksync/"+tt.token+"/v1/initialization", nil)
|
||||
req.Header.Set("X-Forwarded-Host", "untrusted.example.com")
|
||||
req.Header.Set("X-Forwarded-Proto", "https")
|
||||
req.Header.Set("X-Forwarded-Prefix", "/wrong")
|
||||
rr := httptest.NewRecorder()
|
||||
NewHandler(application, logger.New(logger.ERROR)).ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var result struct {
|
||||
Resources map[string]string
|
||||
}
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"instapaper_env_url": "https://reader.example.com/instapaper-proxy/instapaper/api/kobo",
|
||||
"image_host": "https://" + tt.host,
|
||||
"image_url_template": upstream + "/v1/books/{ImageId}/thumbnail/{Width}/{Height}/false/image.jpg",
|
||||
"image_url_quality_template": upstream + "/v1/books/{ImageId}/thumbnail/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg",
|
||||
"other_resource": "https://cdn.kobo.com/book.jpg",
|
||||
} {
|
||||
if got := result.Resources[key]; got != want {
|
||||
t.Errorf("Resources[%s] = %q, want %q", key, got, want)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "/kobo/"+tt.key+"/v1/books/") {
|
||||
t.Error("book service cover URL was not preserved")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBookSyncLeavesOtherResponsesUntouched(t *testing.T) {
|
||||
for _, tt := range []struct{ token, host, prefix string }{
|
||||
{"alice", "books-alice.example.com", "/kobo/alice-key"},
|
||||
{"bob", "books-bob.example.com", "/kobo/bob-key"},
|
||||
{"store", "storeapi.kobo.com", ""},
|
||||
} {
|
||||
t.Run(tt.token, func(t *testing.T) {
|
||||
link := "https://" + tt.host + tt.prefix + "/v1/books/book-1/file/epub?convert_kepub=true"
|
||||
metadata := `{"EntitlementId":"book-1","CoverImageId":"cover-1","DownloadUrls":[{"Format":"KEPUB","Url":"` + link + `"}]}`
|
||||
application := app.NewApp(
|
||||
app.WithConfig(&config.Config{PublicURL: "https://reader.example.com", Users: []config.User{
|
||||
{Token: "alice", BookSyncURL: "https://books-alice.example.com/kobo/alice-key"},
|
||||
{Token: "bob", BookSyncURL: "https://books-bob.example.com/kobo/bob-key"},
|
||||
{Token: "store"},
|
||||
}}),
|
||||
app.WithLogger(logger.New(logger.ERROR)),
|
||||
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
if req.URL.Host != tt.host || req.Header.Get("If-None-Match") != `"old"` || req.Header.Get("Range") != "bytes=0-10" {
|
||||
t.Errorf("upstream URL = %s, headers = %v", req.URL, req.Header)
|
||||
}
|
||||
if !strings.HasPrefix(req.URL.Path, tt.prefix+"/v1/library/") {
|
||||
t.Errorf("upstream path = %q, want prefix %s/v1/library/", req.URL.Path, tt.prefix)
|
||||
}
|
||||
body := "[" + metadata + "]"
|
||||
if strings.HasSuffix(req.URL.Path, "/sync") {
|
||||
body = `[{"NewEntitlement":{"BookMetadata":` + metadata + `}}]`
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}, "Etag": {`"old"`}}, Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
})),
|
||||
)
|
||||
handler := NewHandler(application, logger.New(logger.ERROR))
|
||||
for _, path := range []string{"/v1/library/sync", "/v1/library/book-1/metadata"} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/booksync/"+tt.token+path, nil)
|
||||
req.Header.Set("If-None-Match", `"old"`)
|
||||
req.Header.Set("Range", "bytes=0-10")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
wantBody := "[" + metadata + "]"
|
||||
if path == "/v1/library/sync" {
|
||||
wantBody = `[{"NewEntitlement":{"BookMetadata":` + metadata + `}}]`
|
||||
}
|
||||
if rr.Code != http.StatusOK || rr.Header().Get("Etag") != `"old"` || rr.Body.String() != wantBody {
|
||||
t.Errorf("%s: status = %d, headers = %v, body = %s; want %s", path, rr.Code, rr.Header(), rr.Body.String(), wantBody)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBookSyncStreamsDownload(t *testing.T) {
|
||||
reader, writer := io.Pipe()
|
||||
application := app.NewApp(
|
||||
app.WithConfig(&config.Config{Users: []config.User{{Token: "books", BookSyncURL: "https://books.example.com/api/kobo/device"}}}),
|
||||
app.WithLogger(logger.New(logger.ERROR)),
|
||||
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
if req.URL.Path != "/api/kobo/device/v1/books/1/download" {
|
||||
t.Errorf("upstream path = %q", req.URL.Path)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/epub+zip"}}, Body: reader, ContentLength: -1}, nil
|
||||
})),
|
||||
)
|
||||
server := httptest.NewServer(NewHandler(application, logger.New(logger.ERROR)))
|
||||
defer server.Close()
|
||||
defer func() { _ = writer.Close() }()
|
||||
|
||||
firstChunk := bytes.Repeat([]byte("x"), 32*1024)
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
resp, err := server.Client().Get(server.URL + "/booksync/books/v1/books/1/download")
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
data := make([]byte, len(firstChunk))
|
||||
_, err = io.ReadFull(resp.Body, data)
|
||||
if err == nil && !bytes.Equal(data, firstChunk) {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
result <- err
|
||||
}()
|
||||
go func() { _, _ = writer.Write(firstChunk) }()
|
||||
select {
|
||||
case err := <-result:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("book download did not stream before upstream completion")
|
||||
}
|
||||
}
|
||||
@@ -21,18 +21,16 @@ func (rw *responseWriter) WriteHeader(code int) {
|
||||
rw.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (rw *responseWriter) Unwrap() http.ResponseWriter {
|
||||
return rw.ResponseWriter
|
||||
}
|
||||
|
||||
// LoggingMiddleware logs all incoming requests
|
||||
func LoggingMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
rw := newResponseWriter(w)
|
||||
next.ServeHTTP(rw, r)
|
||||
log.Printf(
|
||||
"%-7s %s %d %s",
|
||||
r.Method,
|
||||
r.RequestURI,
|
||||
rw.statusCode,
|
||||
time.Since(start),
|
||||
)
|
||||
log.Printf("%-7s %d %s", r.Method, rw.statusCode, time.Since(start))
|
||||
})
|
||||
}
|
||||
@@ -12,7 +12,12 @@ import (
|
||||
func ListenAndServe(port int, application *app.App, logger *logger.Logger) {
|
||||
addr := fmt.Sprintf(":%d", port)
|
||||
logger.Infof("Web server starting on port %s", addr)
|
||||
if err := http.ListenAndServe(addr, NewHandler(application, logger)); err != nil {
|
||||
logger.Errorf("Web server failed to start: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func NewHandler(application *app.App, logger *logger.Logger) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// Register handlers
|
||||
@@ -20,18 +25,19 @@ func ListenAndServe(port int, application *app.App, logger *logger.Logger) {
|
||||
mux.HandleFunc("/api/kobo/download", application.HandleKoboDownload)
|
||||
mux.HandleFunc("/api/kobo/send", application.HandleKoboSend)
|
||||
mux.HandleFunc("/api/convert-image", application.HandleConvertImage)
|
||||
mux.HandleFunc("/instapaper-proxy/storeapi/v1/initialization", application.HandleDumpAndForward)
|
||||
mux.HandleFunc("/instapaper-proxy/instapaper/api/kobo/get", application.HandleKoboGet)
|
||||
mux.HandleFunc("/instapaper-proxy/instapaper/api/kobo/download", application.HandleKoboDownload)
|
||||
mux.HandleFunc("/instapaper-proxy/instapaper/api/kobo/send", application.HandleKoboSend)
|
||||
storeProxy := http.StripPrefix("/instapaper-proxy/storeapi", http.HandlerFunc(application.HandleStoreProxy))
|
||||
mux.Handle("/instapaper-proxy/storeapi/", storeProxy)
|
||||
mux.Handle("/instapaper-proxy/storeapi", storeProxy)
|
||||
mux.HandleFunc("/booksync/{deviceToken}/", application.HandleBookSync)
|
||||
|
||||
// Catch-all for unimplemented routes
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
logger.Warnf("404 Not Found: URL=%s, Method=%s, Params=%v", r.URL.Path, r.Method, r.URL.Query())
|
||||
logger.Warnf("404 Not Found: Method=%s", r.Method)
|
||||
http.Error(w, "404 Not Found", http.StatusNotFound)
|
||||
})
|
||||
|
||||
// Apply logging middleware
|
||||
loggedMux := LoggingMiddleware(mux)
|
||||
|
||||
if err := http.ListenAndServe(addr, loggedMux); err != nil {
|
||||
logger.Errorf("Web server failed to start: %v", err)
|
||||
}
|
||||
}
|
||||
return LoggingMiddleware(mux)
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package webserver
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"readeckobo/internal/app"
|
||||
"readeckobo/internal/config"
|
||||
"readeckobo/internal/logger"
|
||||
)
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
|
||||
|
||||
func TestStoreRoutes(t *testing.T) {
|
||||
application := app.NewApp(
|
||||
app.WithConfig(&config.Config{PublicURL: "https://reader.example.com"}),
|
||||
app.WithLogger(logger.New(logger.ERROR)),
|
||||
app.WithProxyTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
if req.URL.Host != "storeapi.kobo.com" || req.Host != "storeapi.kobo.com" {
|
||||
t.Errorf("upstream host = %q, Host = %q", req.URL.Host, req.Host)
|
||||
}
|
||||
switch req.URL.Path {
|
||||
case "/", "/v1/initialization", "/v1/library/sync", "/v1//books":
|
||||
default:
|
||||
t.Errorf("unexpected upstream path = %q", req.URL.Path)
|
||||
}
|
||||
if req.URL.Path == "/v1//books" && req.URL.EscapedPath() != "/v1/%2Fbooks" {
|
||||
t.Errorf("escaped upstream path = %q, want /v1/%%2Fbooks", req.URL.EscapedPath())
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"url":"https://www.instapaper.com"}`))}, nil
|
||||
})),
|
||||
)
|
||||
handler := NewHandler(application, logger.New(logger.ERROR))
|
||||
for _, path := range []string{"/instapaper-proxy/storeapi/v1/initialization", "/instapaper-proxy/storeapi/v1/library/sync", "/instapaper-proxy/storeapi/v1/%2Fbooks?a=1%3B2", "/instapaper-proxy/storeapi"} {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, path, nil))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
})
|
||||
}
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/instapaper-proxy/storeapi-nope", nil))
|
||||
if rr.Code != http.StatusNotFound {
|
||||
t.Errorf("unknown store route status = %d, want 404", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArticleRoutes(t *testing.T) {
|
||||
readeckServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/bookmarks/sync" {
|
||||
t.Errorf("unexpected Readeck request: %s", r.URL.Path)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, "[]")
|
||||
}))
|
||||
defer readeckServer.Close()
|
||||
|
||||
application := app.NewApp(
|
||||
app.WithConfig(&config.Config{
|
||||
Readeck: config.ConfigReadeck{Host: readeckServer.URL},
|
||||
Users: []config.User{{Token: "device", ReadeckAccessToken: "readeck"}},
|
||||
}),
|
||||
app.WithLogger(logger.New(logger.ERROR)),
|
||||
app.WithReadeckHTTPClient(readeckServer.Client()),
|
||||
)
|
||||
handler := NewHandler(application, logger.New(logger.ERROR))
|
||||
for _, tt := range []struct {
|
||||
name, path, body, wantBody string
|
||||
wantStatus int
|
||||
}{
|
||||
{"get", "/instapaper-proxy/instapaper/api/kobo/get", `{"access_token":"device"}`, `"status":1`, http.StatusOK},
|
||||
{"download", "/instapaper-proxy/instapaper/api/kobo/download", `{"access_token":"device"}`, "Missing 'url' parameter", http.StatusBadRequest},
|
||||
{"send", "/instapaper-proxy/instapaper/api/kobo/send", `{"access_token":"device","actions":[]}`, `"status":true`, http.StatusOK},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, tt.path, strings.NewReader(tt.body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
handler.ServeHTTP(rr, req)
|
||||
if rr.Code != tt.wantStatus || !strings.Contains(rr.Body.String(), tt.wantBody) {
|
||||
t.Errorf("status = %d, body = %q; want %d and %q", rr.Code, rr.Body.String(), tt.wantStatus, tt.wantBody)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestLogsDoNotExposeTokens(t *testing.T) {
|
||||
var logs bytes.Buffer
|
||||
old := log.Writer()
|
||||
log.SetOutput(&logs)
|
||||
defer log.SetOutput(old)
|
||||
|
||||
application := app.NewApp(
|
||||
app.WithConfig(&config.Config{}),
|
||||
app.WithLogger(logger.New(logger.DEBUG)),
|
||||
app.WithImageHTTPClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
})}),
|
||||
)
|
||||
handler := NewHandler(application, logger.New(logger.DEBUG))
|
||||
handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/booksync/private-token/?access_token=query-secret", nil))
|
||||
req := httptest.NewRequest(http.MethodPost, "/instapaper-proxy/instapaper/api/kobo/get", strings.NewReader(`{"access_token":"body-secret"}`))
|
||||
handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||
handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/api/convert-image?url=https%3A%2F%2Fexample.com%2Fimg%3Ftoken%3Dimage-secret", nil))
|
||||
for _, secret := range []string{"private-token", "query-secret", "body-secret", "image-secret"} {
|
||||
if strings.Contains(logs.String(), secret) {
|
||||
t.Errorf("request logs contain secret %q", secret)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
# Proxy Kobo Store API so that we can modify the returned Instapaper URL.
|
||||
location /instapaper-proxy/storeapi {
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_name storeapi.kobo.com;
|
||||
proxy_set_header Host storeapi.kobo.com;
|
||||
proxy_pass https://storeapi.kobo.com/;
|
||||
}
|
||||
|
||||
location = /instapaper-proxy/storeapi/v1/initialization {
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_name storeapi.kobo.com;
|
||||
proxy_set_header Host storeapi.kobo.com;
|
||||
proxy_set_header Accept-Encoding "";
|
||||
proxy_pass https://storeapi.kobo.com/;
|
||||
|
||||
sub_filter https://www.instapaper.com "$scheme://$host/instapaper-proxy/instapaper";
|
||||
sub_filter_types application/json;
|
||||
}
|
||||
|
||||
# Proxy to local Kobeck application
|
||||
location /instapaper-proxy/instapaper/ {
|
||||
proxy_pass http://readeckobo-upstream/;
|
||||
proxy_set_header Host $host;
|
||||
client_max_body_size 128M;
|
||||
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-Prefix /instapaper-proxy/instapaper;
|
||||
}
|
||||
Reference in new issue
Block a user