From b2dfd7616573e4553af3e90b1f6156f9ca1ed3b2 Mon Sep 17 00:00:00 2001 From: Diego Ripley Date: Sun, 8 Mar 2026 13:12:47 -0400 Subject: [PATCH] Add vibe code Cloudflare data ingestor to Source Cooperative --- .../05_cloudflare_http_ingestor/.env.example | 7 + .../05_cloudflare_http_ingestor/.gitignore | 8 + scripts/05_cloudflare_http_ingestor/README.md | 145 +++ .../05_cloudflare_http_ingestor/package.json | 17 + .../pnpm-lock.yaml | 862 ++++++++++++++++++ .../05_cloudflare_http_ingestor/src/index.js | 317 +++++++ .../05_cloudflare_http_ingestor/wrangler.toml | 23 + 7 files changed, 1379 insertions(+) create mode 100644 scripts/05_cloudflare_http_ingestor/.env.example create mode 100644 scripts/05_cloudflare_http_ingestor/.gitignore create mode 100644 scripts/05_cloudflare_http_ingestor/README.md create mode 100644 scripts/05_cloudflare_http_ingestor/package.json create mode 100644 scripts/05_cloudflare_http_ingestor/pnpm-lock.yaml create mode 100644 scripts/05_cloudflare_http_ingestor/src/index.js create mode 100644 scripts/05_cloudflare_http_ingestor/wrangler.toml diff --git a/scripts/05_cloudflare_http_ingestor/.env.example b/scripts/05_cloudflare_http_ingestor/.env.example new file mode 100644 index 0000000..9422ce9 --- /dev/null +++ b/scripts/05_cloudflare_http_ingestor/.env.example @@ -0,0 +1,7 @@ +# Copy to .env and fill in real values. +# Used by Wrangler for local development (wrangler dev). +# For deployed Workers, set secrets via: pnpm wrangler secret put + +AUTH_TOKEN="your-auth-token" +AWS_ACCESS_KEY_ID="AKIAxxxxxxxxxxxxxxxxxxxx" +AWS_SECRET_ACCESS_KEY="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/scripts/05_cloudflare_http_ingestor/.gitignore b/scripts/05_cloudflare_http_ingestor/.gitignore new file mode 100644 index 0000000..8586d4a --- /dev/null +++ b/scripts/05_cloudflare_http_ingestor/.gitignore @@ -0,0 +1,8 @@ +node_modules/ +dist/ +.wrangler/ + +# Secret files – never commit +.env +.env.* +!.env.example diff --git a/scripts/05_cloudflare_http_ingestor/README.md b/scripts/05_cloudflare_http_ingestor/README.md new file mode 100644 index 0000000..f410c46 --- /dev/null +++ b/scripts/05_cloudflare_http_ingestor/README.md @@ -0,0 +1,145 @@ +# cf-data-ingestor + +A Cloudflare Worker that acts as a secure proxy: it downloads a file from a +URL provided in a JSON payload and streams it directly into an S3 bucket in +`us-west-2`, keeping memory usage constant regardless of file size. + +## Architecture + +``` +Client POST ──▶ Worker ──stream──▶ S3 PutObject / Multipart + │ + ├─ Auth check (Bearer token) + ├─ Fetch source URL (custom User-Agent) + └─ Sign with AWS Sig V4 (aws4fetch) +``` + +**Two upload paths are used automatically:** + +| Source provides `Content-Length`? | Upload method | Memory overhead | +|---|---|---| +| Yes | Single streaming `PUT` | ~0 (pipe-through) | +| No | Multipart upload in 5 MiB chunks | ≤ 5 MiB | + +## Setup + +### 1. Install dependencies + +```bash +pnpm install +``` + +### 2. Configure `wrangler.toml` + +Edit the `[vars]` section: + +```toml +[vars] +S3_BUCKET = "us-west-2.opendata.source.coop" +S3_REGION = "us-west-2" +S3_ENDPOINT = "" +``` + +`S3_ENDPOINT` should be left empty when targeting AWS S3 (path-style +addressing is used automatically). Set it only for non-AWS S3-compatible +services — `https://` is prepended automatically if omitted. + +### 3. Set secrets + +Copy the example `.env` file and fill in your values: + +```bash +cp .env.example .env +``` + +```env +AUTH_TOKEN="your-auth-token" +AWS_ACCESS_KEY_ID="AKIAxxxxxxxxxxxxxxxxxxxx" +AWS_SECRET_ACCESS_KEY="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +``` + +Wrangler automatically loads the `.env` file during local development +(`pnpm run dev`). For **deployed** Workers, push each secret with: + +```bash +pnpm wrangler secret put AUTH_TOKEN +pnpm wrangler secret put AWS_ACCESS_KEY_ID +pnpm wrangler secret put AWS_SECRET_ACCESS_KEY +``` + +### 4. Deploy + +```bash +pnpm run deploy +``` + +## Usage + +### Request + +**Method:** `POST` +**Content-Type:** `application/json` +**Authorization:** `Bearer ` + +**Payload parameters:** + +| Field | Required | Description | +|---|---|---| +| `download_url` | Yes | Direct link to the source file | +| `user_agent` | Yes | User-Agent string for the download request | +| `key_prefix` | No | Destination path within the S3 bucket | + +### Example + +```bash +curl -X POST https://cf-data-ingestor..workers.dev \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "download_url": "https://diffusion.mern.gouv.qc.ca/diffusion/RGQ/Imagerie/Orthomosaique/Generique/Mosa30rvb0015_30cm_Rvb/Mtm9/Jpeg2000/mos_14_31n02_se_30cm_f09.JP2", + "user_agent": "Data for Canada - d4c-datapkg-orthoimagery", + "key_prefix": "dataforcanada/d4c-datapkg-orthoimagery/archive/ca-qc_government_and_municipalities_of_quebec-2026A000224_d4c-datapkg-orthoimagery_orthorectified_imagery_from_quebec" + }' +``` + +### Successful response + +```json +{ + "ok": true, + "bucket": "us-west-2.opendata.source.coop", + "key": "dataforcanada/d4c-datapkg-orthoimagery/archive/ca-qc_government_and_municipalities_of_quebec-2026A000224_d4c-datapkg-orthoimagery_orthorectified_imagery_from_quebec/mos_14_31n02_se_30cm_f09.JP2", + "content_type": "application/x-msdownload", + "size_bytes": 773722941 +} +``` + +### Error responses + +| Status | Meaning | +|--------|---------| +| 401 | Missing or invalid Bearer token | +| 405 | Non-POST method | +| 415 | Content-Type is not `application/json` | +| 400 | Malformed JSON or missing fields | +| 502 | Source download or S3 upload failed | + +## S3 Object Key + +Only the **filename** is extracted from the `download_url` and placed under the `key_prefix`. The source URL's directory hierarchy is not preserved. + +``` +download_url: https://diffusion.mern.gouv.qc.ca/diffusion/RGQ/Imagerie/Orthomosaique/Generique/Mosa30rvb0015_30cm_Rvb/Mtm9/Jpeg2000/mos_14_31n02_se_30cm_f09.JP2 +key_prefix: "dataforcanada/d4c-datapkg-orthoimagery/archive/ca-qc_government_and_municipalities_of_quebec-2026A000224_d4c-datapkg-orthoimagery_orthorectified_imagery_from_quebec" +→ key: dataforcanada/d4c-datapkg-orthoimagery/archive/ca-qc_government_and_municipalities_of_quebec-2026A000224_d4c-datapkg-orthoimagery_orthorectified_imagery_from_quebec/mos_14_31n02_se_30cm_f09.JP2 +``` + +If `key_prefix` is omitted or empty, the file uploads to the bucket root. + +## Local Development + +```bash +pnpm run dev +``` + +Then POST to `http://localhost:8787`. Wrangler reads secrets from the `.env` file you created in step 3. You can also create environment-specific overrides (e.g. `.env.staging`) — see the [Cloudflare docs](https://developers.cloudflare.com/workers/configuration/secrets/#local-development-with-secrets) for the full `.env` precedence rules. diff --git a/scripts/05_cloudflare_http_ingestor/package.json b/scripts/05_cloudflare_http_ingestor/package.json new file mode 100644 index 0000000..67d9792 --- /dev/null +++ b/scripts/05_cloudflare_http_ingestor/package.json @@ -0,0 +1,17 @@ +{ + "name": "cf-data-ingestor", + "version": "1.0.0", + "description": "Cloudflare Worker that proxies file downloads to an S3 bucket using streaming.", + "main": "src/index.js", + "type": "module", + "scripts": { + "dev": "wrangler dev", + "deploy": "wrangler deploy" + }, + "dependencies": { + "aws4fetch": "^1.0.20" + }, + "devDependencies": { + "wrangler": "4.71.0" + } +} diff --git a/scripts/05_cloudflare_http_ingestor/pnpm-lock.yaml b/scripts/05_cloudflare_http_ingestor/pnpm-lock.yaml new file mode 100644 index 0000000..a936a5a --- /dev/null +++ b/scripts/05_cloudflare_http_ingestor/pnpm-lock.yaml @@ -0,0 +1,862 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + aws4fetch: + specifier: ^1.0.20 + version: 1.0.20 + devDependencies: + wrangler: + specifier: 4.71.0 + version: 4.71.0 + +packages: + + '@cloudflare/kv-asset-handler@0.4.2': + resolution: {integrity: sha512-SIOD2DxrRRwQ+jgzlXCqoEFiKOFqaPjhnNTGKXSRLvp1HiOvapLaFG2kEr9dYQTYe8rKrd9uvDUzmAITeNyaHQ==} + engines: {node: '>=18.0.0'} + + '@cloudflare/unenv-preset@2.15.0': + resolution: {integrity: sha512-EGYmJaGZKWl+X8tXxcnx4v2bOZSjQeNI5dWFeXivgX9+YCT69AkzHHwlNbVpqtEUTbew8eQurpyOpeN8fg00nw==} + peerDependencies: + unenv: 2.0.0-rc.24 + workerd: 1.20260301.1 || ~1.20260302.1 || ~1.20260303.1 || ~1.20260304.1 || >1.20260305.0 <2.0.0-0 + peerDependenciesMeta: + workerd: + optional: true + + '@cloudflare/workerd-darwin-64@1.20260301.1': + resolution: {integrity: sha512-+kJvwociLrvy1JV9BAvoSVsMEIYD982CpFmo/yMEvBwxDIjltYsLTE8DLi0mCkGsQ8Ygidv2fD9wavzXeiY7OQ==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@cloudflare/workerd-darwin-arm64@1.20260301.1': + resolution: {integrity: sha512-PPIetY3e67YBr9O4UhILK8nbm5TqUDl14qx4rwFNrRSBOvlzuczzbd4BqgpAtbGVFxKp1PWpjAnBvGU/OI/tLQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@cloudflare/workerd-linux-64@1.20260301.1': + resolution: {integrity: sha512-Gu5vaVTZuYl3cHa+u5CDzSVDBvSkfNyuAHi6Mdfut7TTUdcb3V5CIcR/mXRSyMXzEy9YxEWIfdKMxOMBjupvYQ==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@cloudflare/workerd-linux-arm64@1.20260301.1': + resolution: {integrity: sha512-igL1pkyCXW6GiGpjdOAvqMi87UW0LMc/+yIQe/CSzuZJm5GzXoAMrwVTkCFnikk6JVGELrM5x0tGYlxa0sk5Iw==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@cloudflare/workerd-windows-64@1.20260301.1': + resolution: {integrity: sha512-Q0wMJ4kcujXILwQKQFc1jaYamVsNvjuECzvRrTI8OxGFMx2yq9aOsswViE4X1gaS2YQQ5u0JGwuGi5WdT1Lt7A==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + + '@cspotcode/source-map-support@0.8.1': + resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} + engines: {node: '>=12'} + + '@emnapi/runtime@1.8.1': + resolution: {integrity: sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==} + + '@esbuild/aix-ppc64@0.27.3': + resolution: {integrity: sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.27.3': + resolution: {integrity: sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.27.3': + resolution: {integrity: sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.27.3': + resolution: {integrity: sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.27.3': + resolution: {integrity: sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.3': + resolution: {integrity: sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.27.3': + resolution: {integrity: sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.3': + resolution: {integrity: sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.27.3': + resolution: {integrity: sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.27.3': + resolution: {integrity: sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.27.3': + resolution: {integrity: sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.27.3': + resolution: {integrity: sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.27.3': + resolution: {integrity: sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.27.3': + resolution: {integrity: sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.3': + resolution: {integrity: sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.27.3': + resolution: {integrity: sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.27.3': + resolution: {integrity: sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.27.3': + resolution: {integrity: sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.3': + resolution: {integrity: sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.27.3': + resolution: {integrity: sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.3': + resolution: {integrity: sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.27.3': + resolution: {integrity: sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.27.3': + resolution: {integrity: sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.27.3': + resolution: {integrity: sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.27.3': + resolution: {integrity: sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.27.3': + resolution: {integrity: sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.34.5': + resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.34.5': + resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.2.4': + resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.2.4': + resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.2.4': + resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linux-arm@1.2.4': + resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} + cpu: [arm] + os: [linux] + + '@img/sharp-libvips-linux-ppc64@1.2.4': + resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} + cpu: [ppc64] + os: [linux] + + '@img/sharp-libvips-linux-riscv64@1.2.4': + resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} + cpu: [riscv64] + os: [linux] + + '@img/sharp-libvips-linux-s390x@1.2.4': + resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} + cpu: [s390x] + os: [linux] + + '@img/sharp-libvips-linux-x64@1.2.4': + resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} + cpu: [x64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} + cpu: [x64] + os: [linux] + + '@img/sharp-linux-arm64@0.34.5': + resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linux-arm@0.34.5': + resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + + '@img/sharp-linux-ppc64@0.34.5': + resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ppc64] + os: [linux] + + '@img/sharp-linux-riscv64@0.34.5': + resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [riscv64] + os: [linux] + + '@img/sharp-linux-s390x@0.34.5': + resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [s390x] + os: [linux] + + '@img/sharp-linux-x64@0.34.5': + resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-linuxmusl-arm64@0.34.5': + resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linuxmusl-x64@0.34.5': + resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-wasm32@0.34.5': + resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.34.5': + resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.34.5': + resolution: {integrity: sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.34.5': + resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@jridgewell/trace-mapping@0.3.9': + resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + + '@poppinss/colors@4.1.6': + resolution: {integrity: sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==} + + '@poppinss/dumper@0.6.5': + resolution: {integrity: sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==} + + '@poppinss/exception@1.2.3': + resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} + + '@sindresorhus/is@7.2.0': + resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} + engines: {node: '>=18'} + + '@speed-highlight/core@1.2.14': + resolution: {integrity: sha512-G4ewlBNhUtlLvrJTb88d2mdy2KRijzs4UhnlrOSRT4bmjh/IqNElZa3zkrZ+TC47TwtlDWzVLFADljF1Ijp5hA==} + + aws4fetch@1.0.20: + resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==} + + blake3-wasm@2.1.5: + resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} + + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + error-stack-parser-es@1.0.5: + resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} + + esbuild@0.27.3: + resolution: {integrity: sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==} + engines: {node: '>=18'} + hasBin: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + kleur@4.1.5: + resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} + engines: {node: '>=6'} + + miniflare@4.20260301.1: + resolution: {integrity: sha512-fqkHx0QMKswRH9uqQQQOU/RoaS3Wjckxy3CUX3YGJr0ZIMu7ObvI+NovdYi6RIsSPthNtq+3TPmRNxjeRiasog==} + engines: {node: '>=18.0.0'} + hasBin: true + + path-to-regexp@6.3.0: + resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + semver@7.7.4: + resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} + engines: {node: '>=10'} + hasBin: true + + sharp@0.34.5: + resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + undici@7.18.2: + resolution: {integrity: sha512-y+8YjDFzWdQlSE9N5nzKMT3g4a5UBX1HKowfdXh0uvAnTaqqwqB92Jt4UXBAeKekDs5IaDKyJFR4X1gYVCgXcw==} + engines: {node: '>=20.18.1'} + + unenv@2.0.0-rc.24: + resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} + + workerd@1.20260301.1: + resolution: {integrity: sha512-oterQ1IFd3h7PjCfT4znSFOkJCvNQ6YMOyZ40YsnO3nrSpgB4TbJVYWFOnyJAw71/RQuupfVqZZWKvsy8GO3fw==} + engines: {node: '>=16'} + hasBin: true + + wrangler@4.71.0: + resolution: {integrity: sha512-j6pSGAncOLNQDRzqtp0EqzYj52CldDP7uz/C9cxVrIgqa5p+cc0b4pIwnapZZAGv9E1Loa3tmPD0aXonH7KTkw==} + engines: {node: '>=20.0.0'} + hasBin: true + peerDependencies: + '@cloudflare/workers-types': ^4.20260226.1 + peerDependenciesMeta: + '@cloudflare/workers-types': + optional: true + + ws@8.18.0: + resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + youch-core@0.3.3: + resolution: {integrity: sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==} + + youch@4.1.0-beta.10: + resolution: {integrity: sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==} + +snapshots: + + '@cloudflare/kv-asset-handler@0.4.2': {} + + '@cloudflare/unenv-preset@2.15.0(unenv@2.0.0-rc.24)(workerd@1.20260301.1)': + dependencies: + unenv: 2.0.0-rc.24 + optionalDependencies: + workerd: 1.20260301.1 + + '@cloudflare/workerd-darwin-64@1.20260301.1': + optional: true + + '@cloudflare/workerd-darwin-arm64@1.20260301.1': + optional: true + + '@cloudflare/workerd-linux-64@1.20260301.1': + optional: true + + '@cloudflare/workerd-linux-arm64@1.20260301.1': + optional: true + + '@cloudflare/workerd-windows-64@1.20260301.1': + optional: true + + '@cspotcode/source-map-support@0.8.1': + dependencies: + '@jridgewell/trace-mapping': 0.3.9 + + '@emnapi/runtime@1.8.1': + dependencies: + tslib: 2.8.1 + optional: true + + '@esbuild/aix-ppc64@0.27.3': + optional: true + + '@esbuild/android-arm64@0.27.3': + optional: true + + '@esbuild/android-arm@0.27.3': + optional: true + + '@esbuild/android-x64@0.27.3': + optional: true + + '@esbuild/darwin-arm64@0.27.3': + optional: true + + '@esbuild/darwin-x64@0.27.3': + optional: true + + '@esbuild/freebsd-arm64@0.27.3': + optional: true + + '@esbuild/freebsd-x64@0.27.3': + optional: true + + '@esbuild/linux-arm64@0.27.3': + optional: true + + '@esbuild/linux-arm@0.27.3': + optional: true + + '@esbuild/linux-ia32@0.27.3': + optional: true + + '@esbuild/linux-loong64@0.27.3': + optional: true + + '@esbuild/linux-mips64el@0.27.3': + optional: true + + '@esbuild/linux-ppc64@0.27.3': + optional: true + + '@esbuild/linux-riscv64@0.27.3': + optional: true + + '@esbuild/linux-s390x@0.27.3': + optional: true + + '@esbuild/linux-x64@0.27.3': + optional: true + + '@esbuild/netbsd-arm64@0.27.3': + optional: true + + '@esbuild/netbsd-x64@0.27.3': + optional: true + + '@esbuild/openbsd-arm64@0.27.3': + optional: true + + '@esbuild/openbsd-x64@0.27.3': + optional: true + + '@esbuild/openharmony-arm64@0.27.3': + optional: true + + '@esbuild/sunos-x64@0.27.3': + optional: true + + '@esbuild/win32-arm64@0.27.3': + optional: true + + '@esbuild/win32-ia32@0.27.3': + optional: true + + '@esbuild/win32-x64@0.27.3': + optional: true + + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.2.4 + optional: true + + '@img/sharp-darwin-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.2.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.2.4': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-s390x@1.2.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + optional: true + + '@img/sharp-linux-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.2.4 + optional: true + + '@img/sharp-linux-arm@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.2.4 + optional: true + + '@img/sharp-linux-ppc64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.2.4 + optional: true + + '@img/sharp-linux-riscv64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.2.4 + optional: true + + '@img/sharp-linux-s390x@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.2.4 + optional: true + + '@img/sharp-linux-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + optional: true + + '@img/sharp-wasm32@0.34.5': + dependencies: + '@emnapi/runtime': 1.8.1 + optional: true + + '@img/sharp-win32-arm64@0.34.5': + optional: true + + '@img/sharp-win32-ia32@0.34.5': + optional: true + + '@img/sharp-win32-x64@0.34.5': + optional: true + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@jridgewell/trace-mapping@0.3.9': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@poppinss/colors@4.1.6': + dependencies: + kleur: 4.1.5 + + '@poppinss/dumper@0.6.5': + dependencies: + '@poppinss/colors': 4.1.6 + '@sindresorhus/is': 7.2.0 + supports-color: 10.2.2 + + '@poppinss/exception@1.2.3': {} + + '@sindresorhus/is@7.2.0': {} + + '@speed-highlight/core@1.2.14': {} + + aws4fetch@1.0.20: {} + + blake3-wasm@2.1.5: {} + + cookie@1.1.1: {} + + detect-libc@2.1.2: {} + + error-stack-parser-es@1.0.5: {} + + esbuild@0.27.3: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.3 + '@esbuild/android-arm': 0.27.3 + '@esbuild/android-arm64': 0.27.3 + '@esbuild/android-x64': 0.27.3 + '@esbuild/darwin-arm64': 0.27.3 + '@esbuild/darwin-x64': 0.27.3 + '@esbuild/freebsd-arm64': 0.27.3 + '@esbuild/freebsd-x64': 0.27.3 + '@esbuild/linux-arm': 0.27.3 + '@esbuild/linux-arm64': 0.27.3 + '@esbuild/linux-ia32': 0.27.3 + '@esbuild/linux-loong64': 0.27.3 + '@esbuild/linux-mips64el': 0.27.3 + '@esbuild/linux-ppc64': 0.27.3 + '@esbuild/linux-riscv64': 0.27.3 + '@esbuild/linux-s390x': 0.27.3 + '@esbuild/linux-x64': 0.27.3 + '@esbuild/netbsd-arm64': 0.27.3 + '@esbuild/netbsd-x64': 0.27.3 + '@esbuild/openbsd-arm64': 0.27.3 + '@esbuild/openbsd-x64': 0.27.3 + '@esbuild/openharmony-arm64': 0.27.3 + '@esbuild/sunos-x64': 0.27.3 + '@esbuild/win32-arm64': 0.27.3 + '@esbuild/win32-ia32': 0.27.3 + '@esbuild/win32-x64': 0.27.3 + + fsevents@2.3.3: + optional: true + + kleur@4.1.5: {} + + miniflare@4.20260301.1: + dependencies: + '@cspotcode/source-map-support': 0.8.1 + sharp: 0.34.5 + undici: 7.18.2 + workerd: 1.20260301.1 + ws: 8.18.0 + youch: 4.1.0-beta.10 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + path-to-regexp@6.3.0: {} + + pathe@2.0.3: {} + + semver@7.7.4: {} + + sharp@0.34.5: + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.7.4 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.34.5 + '@img/sharp-darwin-x64': 0.34.5 + '@img/sharp-libvips-darwin-arm64': 1.2.4 + '@img/sharp-libvips-darwin-x64': 1.2.4 + '@img/sharp-libvips-linux-arm': 1.2.4 + '@img/sharp-libvips-linux-arm64': 1.2.4 + '@img/sharp-libvips-linux-ppc64': 1.2.4 + '@img/sharp-libvips-linux-riscv64': 1.2.4 + '@img/sharp-libvips-linux-s390x': 1.2.4 + '@img/sharp-libvips-linux-x64': 1.2.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + '@img/sharp-linux-arm': 0.34.5 + '@img/sharp-linux-arm64': 0.34.5 + '@img/sharp-linux-ppc64': 0.34.5 + '@img/sharp-linux-riscv64': 0.34.5 + '@img/sharp-linux-s390x': 0.34.5 + '@img/sharp-linux-x64': 0.34.5 + '@img/sharp-linuxmusl-arm64': 0.34.5 + '@img/sharp-linuxmusl-x64': 0.34.5 + '@img/sharp-wasm32': 0.34.5 + '@img/sharp-win32-arm64': 0.34.5 + '@img/sharp-win32-ia32': 0.34.5 + '@img/sharp-win32-x64': 0.34.5 + + supports-color@10.2.2: {} + + tslib@2.8.1: + optional: true + + undici@7.18.2: {} + + unenv@2.0.0-rc.24: + dependencies: + pathe: 2.0.3 + + workerd@1.20260301.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20260301.1 + '@cloudflare/workerd-darwin-arm64': 1.20260301.1 + '@cloudflare/workerd-linux-64': 1.20260301.1 + '@cloudflare/workerd-linux-arm64': 1.20260301.1 + '@cloudflare/workerd-windows-64': 1.20260301.1 + + wrangler@4.71.0: + dependencies: + '@cloudflare/kv-asset-handler': 0.4.2 + '@cloudflare/unenv-preset': 2.15.0(unenv@2.0.0-rc.24)(workerd@1.20260301.1) + blake3-wasm: 2.1.5 + esbuild: 0.27.3 + miniflare: 4.20260301.1 + path-to-regexp: 6.3.0 + unenv: 2.0.0-rc.24 + workerd: 1.20260301.1 + optionalDependencies: + fsevents: 2.3.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + ws@8.18.0: {} + + youch-core@0.3.3: + dependencies: + '@poppinss/exception': 1.2.3 + error-stack-parser-es: 1.0.5 + + youch@4.1.0-beta.10: + dependencies: + '@poppinss/colors': 4.1.6 + '@poppinss/dumper': 0.6.5 + '@speed-highlight/core': 1.2.14 + cookie: 1.1.1 + youch-core: 0.3.3 diff --git a/scripts/05_cloudflare_http_ingestor/src/index.js b/scripts/05_cloudflare_http_ingestor/src/index.js new file mode 100644 index 0000000..633b0cd --- /dev/null +++ b/scripts/05_cloudflare_http_ingestor/src/index.js @@ -0,0 +1,317 @@ +import { AwsClient } from "aws4fetch"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- +const MIN_PART_SIZE = 5 * 1024 * 1024; // 5 MiB – S3 minimum for multipart parts + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Return a plain-text Response with the given status. */ +function textResponse(body, status = 200) { + return new Response(body, { + status, + headers: { "Content-Type": "text/plain; charset=utf-8" }, + }); +} + +/** Return a JSON Response. */ +function jsonResponse(obj, status = 200) { + return new Response(JSON.stringify(obj), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +/** Derive the S3 object key from the download URL (filename only). */ +function objectKeyFromUrl(downloadUrl, prefix) { + const pathname = new URL(downloadUrl).pathname; + const filename = pathname.split("/").pop() || "unnamed"; + return prefix ? `${prefix.replace(/\/+$/, "")}/${filename}` : filename; +} + +/** Build a pre-configured AwsClient for S3 in the target region. */ +function makeAwsClient(env) { + return new AwsClient({ + accessKeyId: env.AWS_ACCESS_KEY_ID, + secretAccessKey: env.AWS_SECRET_ACCESS_KEY, + region: env.S3_REGION || "us-west-2", + service: "s3", + }); +} + +/** + * Construct the S3 endpoint URL for a given bucket + key. + * Uses path-style addressing (s3.region.amazonaws.com/bucket/key) which is + * required for bucket names containing dots (e.g. "us-west-2.opendata.source.coop"). + * If a custom S3_ENDPOINT is set, it is used as the base instead. + */ +function s3Url(bucket, key, region, endpoint) { + let base; + if (endpoint) { + // Normalise: strip trailing slashes, prepend https:// if no scheme given + base = endpoint.replace(/\/+$/, ""); + if (!/^https?:\/\//i.test(base)) { + base = `https://${base}`; + } + } else { + base = `https://s3.${region}.amazonaws.com`; + } + return `${base}/${bucket}/${encodeURI(key)}`; +} + +// --------------------------------------------------------------------------- +// S3 Upload – Single PUT (streaming, requires known Content-Length) +// --------------------------------------------------------------------------- + +async function putObjectStreaming(aws, bucket, region, key, body, contentLength, contentType, endpoint) { + const url = s3Url(bucket, key, region, endpoint); + const headers = { + "Content-Type": contentType || "application/octet-stream", + "Content-Length": String(contentLength), + "x-amz-acl": "bucket-owner-full-control", + }; + + const resp = await aws.fetch(url, { + method: "PUT", + headers, + body, // ReadableStream – streamed directly, no buffering + }); + + if (!resp.ok) { + const text = await resp.text(); + throw new Error(`S3 PUT failed (${resp.status}): ${text}`); + } + return resp; +} + +// --------------------------------------------------------------------------- +// S3 Upload – Multipart (streaming, for unknown Content-Length) +// --------------------------------------------------------------------------- + +async function initiateMultipart(aws, bucket, region, key, contentType, endpoint) { + const url = `${s3Url(bucket, key, region, endpoint)}?uploads`; + const resp = await aws.fetch(url, { + method: "POST", + headers: { + "Content-Type": contentType || "application/octet-stream", + "x-amz-acl": "bucket-owner-full-control", + }, + }); + if (!resp.ok) { + const text = await resp.text(); + throw new Error(`Initiate multipart failed (${resp.status}): ${text}`); + } + const xml = await resp.text(); + const match = xml.match(/(.+?)<\/UploadId>/); + if (!match) throw new Error("Could not parse UploadId from response"); + return match[1]; +} + +async function uploadPart(aws, bucket, region, key, uploadId, partNumber, body, length, endpoint) { + const url = `${s3Url(bucket, key, region, endpoint)}?partNumber=${partNumber}&uploadId=${encodeURIComponent(uploadId)}`; + const resp = await aws.fetch(url, { + method: "PUT", + headers: { "Content-Length": String(length) }, + body, + }); + if (!resp.ok) { + const text = await resp.text(); + throw new Error(`Upload part ${partNumber} failed (${resp.status}): ${text}`); + } + const etag = resp.headers.get("ETag"); + return etag; +} + +async function completeMultipart(aws, bucket, region, key, uploadId, parts, endpoint) { + const partsXml = parts + .map((p) => `${p.partNumber}${p.etag}`) + .join(""); + const xmlBody = `${partsXml}`; + const url = `${s3Url(bucket, key, region, endpoint)}?uploadId=${encodeURIComponent(uploadId)}`; + const resp = await aws.fetch(url, { + method: "POST", + headers: { "Content-Type": "application/xml" }, + body: xmlBody, + }); + if (!resp.ok) { + const text = await resp.text(); + throw new Error(`Complete multipart failed (${resp.status}): ${text}`); + } + return resp; +} + +async function abortMultipart(aws, bucket, region, key, uploadId, endpoint) { + const url = `${s3Url(bucket, key, region, endpoint)}?uploadId=${encodeURIComponent(uploadId)}`; + try { + await aws.fetch(url, { method: "DELETE" }); + } catch { + // best-effort cleanup + } +} + +/** + * Read from a ReadableStream in ≥ MIN_PART_SIZE chunks and upload each as an + * S3 multipart part. Memory usage stays bounded to ~MIN_PART_SIZE at a time. + */ +async function multipartStreamUpload(aws, bucket, region, key, stream, contentType, endpoint) { + const uploadId = await initiateMultipart(aws, bucket, region, key, contentType, endpoint); + const parts = []; + let partNumber = 1; + let buffer = []; + let bufferSize = 0; + + const reader = stream.getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer.push(value); + bufferSize += value.byteLength; + + // Flush when we've accumulated enough for a part + if (bufferSize >= MIN_PART_SIZE) { + const blob = new Blob(buffer); + const etag = await uploadPart(aws, bucket, region, key, uploadId, partNumber, blob, bufferSize, endpoint); + parts.push({ partNumber, etag }); + partNumber++; + buffer = []; + bufferSize = 0; + } + } + + // Upload remaining bytes as the final part + if (bufferSize > 0) { + const blob = new Blob(buffer); + const etag = await uploadPart(aws, bucket, region, key, uploadId, partNumber, blob, bufferSize, endpoint); + parts.push({ partNumber, etag }); + } + + await completeMultipart(aws, bucket, region, key, uploadId, parts, endpoint); + } catch (err) { + await abortMultipart(aws, bucket, region, key, uploadId, endpoint); + throw err; + } +} + +// --------------------------------------------------------------------------- +// Main handler +// --------------------------------------------------------------------------- + +export default { + async fetch(request, env) { + // ---- Method check ---- + if (request.method !== "POST") { + return textResponse("Method Not Allowed. Use POST.", 405); + } + + // ---- Auth check ---- + const authHeader = request.headers.get("Authorization") || ""; + const token = authHeader.startsWith("Bearer ") ? authHeader.slice(7) : ""; + if (!token || token !== env.AUTH_TOKEN) { + return textResponse("Unauthorized", 401); + } + + // ---- Content-Type check ---- + const ct = request.headers.get("Content-Type") || ""; + if (!ct.includes("application/json")) { + return textResponse("Content-Type must be application/json", 415); + } + + // ---- Parse body ---- + let payload; + try { + payload = await request.json(); + } catch { + return textResponse("Invalid JSON body", 400); + } + + const { download_url, user_agent, key_prefix } = payload; + if (!download_url || !user_agent) { + return jsonResponse( + { error: "'download_url' and 'user_agent' are required." }, + 400, + ); + } + + // Validate the download URL + let parsedUrl; + try { + parsedUrl = new URL(download_url); + if (!["http:", "https:"].includes(parsedUrl.protocol)) throw new Error(); + } catch { + return jsonResponse({ error: "Invalid download_url" }, 400); + } + + // ---- Fetch the source file (streaming) ---- + let sourceResp; + try { + sourceResp = await fetch(download_url, { + headers: { "User-Agent": user_agent }, + redirect: "follow", + }); + } catch (err) { + return jsonResponse({ error: `Download failed: ${err.message}` }, 502); + } + + if (!sourceResp.ok) { + return jsonResponse( + { error: `Source returned HTTP ${sourceResp.status}` }, + 502, + ); + } + + // ---- Prepare S3 parameters ---- + const bucket = env.S3_BUCKET; + const region = env.S3_REGION || "us-west-2"; + const endpoint = env.S3_ENDPOINT || ""; + const key = objectKeyFromUrl(download_url, key_prefix || ""); + const sourceContentType = + sourceResp.headers.get("Content-Type") || "application/octet-stream"; + const contentLength = sourceResp.headers.get("Content-Length"); + + const aws = makeAwsClient(env); + + // ---- Upload to S3 ---- + try { + if (contentLength && Number(contentLength) > 0) { + // Known size → single streaming PUT (zero extra memory) + await putObjectStreaming( + aws, + bucket, + region, + key, + sourceResp.body, + contentLength, + sourceContentType, + endpoint, + ); + } else { + // Unknown size → multipart streaming upload (≤ 5 MiB buffer) + await multipartStreamUpload( + aws, + bucket, + region, + key, + sourceResp.body, + sourceContentType, + endpoint, + ); + } + } catch (err) { + return jsonResponse({ error: `S3 upload failed: ${err.message}` }, 502); + } + + return jsonResponse({ + ok: true, + bucket, + key, + content_type: sourceContentType, + ...(contentLength ? { size_bytes: Number(contentLength) } : {}), + }); + }, +}; diff --git a/scripts/05_cloudflare_http_ingestor/wrangler.toml b/scripts/05_cloudflare_http_ingestor/wrangler.toml new file mode 100644 index 0000000..678c084 --- /dev/null +++ b/scripts/05_cloudflare_http_ingestor/wrangler.toml @@ -0,0 +1,23 @@ +name = "cf-data-ingestor" +main = "src/index.js" +compatibility_date = "2026-03-08" +routes = [ + { pattern = "cf-data-ingestor.labs.dataforcanada.org", custom_domain = true } +] + +# ------------------------------------------------------------------ +# Secrets (set via `wrangler secret put `): +# AUTH_TOKEN +# AWS_ACCESS_KEY_ID +# AWS_SECRET_ACCESS_KEY +# ------------------------------------------------------------------ + +# Non-secret environment variables +[vars] +S3_BUCKET = "us-west-2.opendata.source.coop" +S3_REGION = "us-west-2" +S3_ENDPOINT = "" + +[placement] +# TODO: parametize so user can specifically set where they want the worker to run +mode = "smart"