diff --git a/.devcontainer/check-image.sh b/.devcontainer/check-image.sh new file mode 100755 index 0000000..5596026 --- /dev/null +++ b/.devcontainer/check-image.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# initializeCommand for the dev container: runs on the *host* before the +# container is created. The image is built locally (it embeds proprietary +# SDKs, so it is not on a registry); without this check a missing image +# surfaces as a confusing "pull access denied" from Docker. +set -euo pipefail + +image="${1:?usage: check-image.sh IMAGE:TAG}" + +if ! docker image inspect "${image}" >/dev/null 2>&1; then + cat >&2 < dataforcanada/gdal-ecw-mrsid:3.13.3 (GDAL + ECW + MrSID) + // docker/python/build.sh -> dataforcanada/gdal-ecw-mrsid-python:3.13.3 (+ uv venv: rioxarray, rasterio, ...) + // linux/amd64 only. To bump GDAL, rebuild both stages and change this tag. + "image": "dataforcanada/gdal-ecw-mrsid-python:3.13.3", + // Fail with a clear message if the image has not been built yet. + "initializeCommand": "bash ${localWorkspaceFolder}/.devcontainer/check-image.sh dataforcanada/gdal-ecw-mrsid-python:3.13.3", + // Non-root user created in docker/python/Dockerfile; its UID/GID is remapped + // to the host user's on first start so bind-mounted files keep sane ownership. + "remoteUser": "d4c", + "updateRemoteUserUID": true, + // Assert (not print) that ECW + MrSID are usable from GDAL and rasterio. + "postCreateCommand": "verify-gdal-drivers", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "charliermarsh.ruff", + "ms-toolsai.jupyter", + "tamasfe.even-better-toml", + "ms-azuretools.vscode-containers", + "anthropic.claude-code" + ], + "settings": { + "python.defaultInterpreterPath": "/opt/venv/bin/python", + "python.terminal.activateEnvironment": false, + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff" + }, + "files.exclude": { + "**/__pycache__": true, + "**/.ruff_cache": true, + "**/.pytest_cache": true + } + } + } + } +} diff --git a/.gitignore b/.gitignore index b06b16a..f8afd6f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,11 @@ *.sdw *.prj +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class # GDAL source checkout used by docker/gdal/build.sh docker/gdal/src/ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e21c79e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "workbench.editorAssociations": { + "*.copilotmd": "vscode.markdown.preview.editor", + "*.parquet": "duckdb.dataFileViewer", + "*.csv": "duckdb.dataFileViewer", + "*.tsv": "duckdb.dataFileViewer", + "*.xlsx": "duckdb.dataFileViewer" + } +} \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..9587a0c --- /dev/null +++ b/docker/README.md @@ -0,0 +1,229 @@ +# Building the GDAL + ECW + MrSID images for the dev container + +The dev container in [`.devcontainer/devcontainer.json`](../.devcontainer/devcontainer.json) +uses a locally built image (it embeds proprietary SDKs, so it is not published +to a registry). Build it in two stages, from the repo root: + +```bash +docker/gdal/build.sh # stage 1: GDAL 3.13.3 + ECW + MrSID (~18 min cold on 16 cores; 3.6 GB) +docker/python/build.sh # stage 2: uv venv with rioxarray etc. (~1.5 min; 4.5 GB) +``` + +Then in VS Code: **Dev Containers: Reopen in Container**. If the image is +missing, the container's `initializeCommand` stops with a message pointing +back here instead of a cryptic "pull access denied". + +| Stage | Image | Contents | +| --- | --- | --- | +| 1 | `dataforcanada/gdal-ecw-mrsid:3.13.3` | OSGeo's `ubuntu-full` GDAL build (Ubuntu 26.04, PROJ 9.8.1, all PROJ grids) plus the **ECW** (libecwj2 3.3) and **MrSID** (DSDK 9.5.5) drivers. Built from GDAL's own `docker/ubuntu-full/Dockerfile`, unmodified. | +| 2 | `dataforcanada/gdal-ecw-mrsid-python:3.13.3` | `FROM` stage 1. `uv`-managed venv at `/opt/venv` (system Python 3.14) with `rioxarray`, `rasterio` (compiled against the image's GDAL), `xarray`, `dask[array]`, `numpy`. Non-root user `d4c`. This is what the dev container runs. | + +## Prerequisites + +- Docker with BuildKit/buildx (Docker 23+; tested with 29.x) on an **x86_64 + host**. Both SDKs are x86_64 glibc binaries; the scripts refuse to run + elsewhere. (Apple Silicon: possible under emulation with + `D4C_ALLOW_EMULATED_BUILD=1`, but expect many hours.) +- ~15 GB free disk for build cache + images, and a decent network connection: + stage 1 downloads Ubuntu dev packages, ~10 source tarballs, the two SDK + zips and ~600 MB of PROJ grids. +- `git` (stage 1 shallow-clones GDAL into `docker/gdal/src/`, git-ignored). + +## Stage 1 — `docker/gdal/build.sh` + +What it does: + +1. Builds a tiny builder base image `dataforcanada/gdal-builder-base:26.04` + from [`gdal/builder-base.Dockerfile`](gdal/builder-base.Dockerfile) — see + [Why a builder base image](#why-a-builder-base-image) below. +2. Shallow-clones `https://github.com/OSGeo/gdal.git` at tag `v3.13.3`. +3. Runs `docker buildx build` on GDAL's `docker/ubuntu-full/Dockerfile` with + the build context at the checkout root (the Dockerfile does + `COPY --link . gdal/`), passing only build args the upstream file already + defines: + + ```bash + docker buildx build \ + --platform linux/amd64 \ + --file docker/ubuntu-full/Dockerfile \ + --build-arg BASE_IMAGE=dataforcanada/gdal-builder-base:26.04 \ + --build-arg GDAL_VERSION=v3.13.3 \ + --build-arg GDAL_BUILD_IS_RELEASE=YES \ + --build-arg PROJ_VERSION=9.8.1 \ + --build-arg WITH_ECW=yes \ + --build-arg WITH_MRSID=yes \ + --build-arg WITH_DEBUG_SYMBOLS=no \ + --build-arg WITH_CCACHE=1 \ + --tag dataforcanada/gdal-ecw-mrsid:3.13.3 \ + --load . + ``` + +4. Asserts the result: `gdal-config --version` must equal `3.13.3` and + `gdalinfo --formats` must list `ECW` and `MrSID`. A failed assertion exits + non-zero — do not use an image that failed here. + +Environment knobs: `GDAL_VERSION` (default `3.13.3`, no leading `v`), +`PROJ_VERSION` (`9.8.1`), `IMAGE`, `TAG`, `WITH_CCACHE` (default `1`; set to +empty to disable), `GDAL_SRC_DIR`. For a scrollable log: +`BUILDKIT_PROGRESS=plain docker/gdal/build.sh 2>&1 | tee stage1.log`. + +### What to expect + +The builder stage compiles kealib, mongo-c-driver, mongocxx, TileDB, +libOpenDRIVE, libqb3, libjxl, arrow-adbc (Go), PROJ twice (once only to run +`projsync`, which downloads every PROJ grid) and finally GDAL with LTO. +Measured on this machine (16 cores, 30 GB RAM, NVMe): **17.8 min** end to end +from a cold cache, of which GDAL itself is ~4 min, adbc ~3 min, the grid +download ~3 min, TileDB ~2 min. With a warm ccache but *every* layer +invalidated (base image changed) it was 8.5 min. A 4-core laptop should plan +for 1–2 hours cold. The final image is 3.56 GB (the PROJ grids are most of +it). The Go module downloads in the adbc step have no retry logic upstream; if +that step fails with a `proxy.golang.org … read` error, just run the script +again — everything before it is a layer-cache hit. + +### Rebuilds, `WITH_CCACHE` and `RSYNC_REMOTE` + +Docker's layer cache does most of the work and needs nothing from you. Build +args are declared in order in the upstream Dockerfile, so a rebuild re-runs +only the layers from the first changed arg onward: + +- change `GDAL_VERSION` only → only the GDAL compile re-runs; PROJ, TileDB, + libjxl, adbc… are layer-cache hits; +- change `PROJ_VERSION` → PROJ and GDAL re-run; +- change `WITH_ECW`/`WITH_MRSID`/`WITH_CCACHE` → everything from the SDK + download step onward re-runs (TileDB and friends are earlier and stay cached); +- change `BASE_IMAGE` → the whole builder stage re-runs. + +`WITH_CCACHE=1` (on by default here) runs every compile through `ccache`, +with the cache kept in BuildKit cache mounts (`--mount=type=cache,id=ubuntu-full-*`) +that persist on this machine until `docker builder prune`. It is worth +keeping on: when the builder base changed during setup and *every* layer was +invalidated, TileDB still rebuilt in 2.5 s instead of 104 s and libjxl in +22 s instead of 50 s. For a patch-level GDAL bump most translation units are +unchanged, so the GDAL step drops to a few minutes (only the LTO link is not +cacheable). Note upstream caps the third-party caches at 100 MB and GDAL's at +1 GB. + +`RSYNC_REMOTE` is **not** worth it on a single workstation. It rsyncs those +ccache directories to/from an rsync daemon (OSGeo's `build.sh` runs one in a +container on `--network host`) so the cache survives builder prunes and can be +shared across machines or CI. BuildKit cache mounts already give the same +benefit locally with zero setup. + +PROJ grids: OSGeo passes `PROJ_DATUMGRID_LATEST_LAST_MODIFIED` (the +`Last-Modified` header of cdn.proj.org) purely to bust that layer's cache when +new grids are published. This script doesn't, so the grid set is frozen until +something above that layer changes — fine for reproducibility; pass +`--build-arg PROJ_DATUMGRID_LATEST_LAST_MODIFIED="$(date)"` yourself if you +want fresh grids. + +### Why a builder base image + +GDAL's `docker/ubuntu-full/bh-gdal.sh` only passes `-DECW_ROOT` / +`-DMRSID_ROOT` to CMake when `uname -p` prints `x86_64`. Ubuntu 26.04's +default coreutils are uutils (Rust), whose `uname -p` prints `unknown`, so on +stock `ubuntu:26.04` both SDKs are downloaded and then silently ignored: the +build "succeeds" and the image has no ECW/MrSID driver (this is how the first +attempt here failed; OSGeo's CI never enables the proprietary SDKs, so nothing +upstream catches it). + +Switching the system to GNU coreutils (`coreutils-from-gnu`) does not help: +`build-essential` on 26.04 hard-depends on `coreutils-from-uutils` and is the +first thing the upstream Dockerfile installs, which flips `uname` straight +back. Ubuntu does ship GNU coreutils alongside as `gnu-coreutils` +(`/usr/bin/gnuuname` etc.), and apt never touches `/usr/local`, so +`gdal/builder-base.Dockerfile` is just `ubuntu:26.04` plus +`ln -s /usr/bin/gnuuname /usr/local/bin/uname` — GNU `uname` wins PATH lookup +for the rest of the build. It is passed as the upstream `BASE_IMAGE` arg, i.e. +it only affects the builder stage; the runtime image (`TARGET_BASE_IMAGE`) is +stock `ubuntu:26.04` and the GDAL checkout is never modified. Drop it once +upstream changes that test to `uname -m` (still `uname -p` on `master` as of +2026-09). + +## Stage 2 — `docker/python/build.sh` + +Builds [`python/Dockerfile`](python/Dockerfile) `FROM dataforcanada/gdal-ecw-mrsid:${GDAL_VERSION}` +and then re-runs the assertions against the finished image as the runtime +user. Environment knobs: `GDAL_VERSION`, `GDAL_IMAGE`, `IMAGE`, `TAG`, +`USER_UID`/`USER_GID` (default 1000/1000). + +Key points, all enforced inside the Dockerfile so the image cannot be built +with them violated: + +- **rasterio is compiled from source** with `uv pip install --no-binary rasterio`. + rasterio's PyPI wheels bundle their own libgdal (the 1.5.1 wheel ships GDAL + 3.12.4, without ECW/MrSID) that would shadow the image's GDAL; the sdist + build links against it via `gdal-config` instead. Flag spelling matters: + `uv pip install` takes pip-style `--no-binary `, while + `--no-binary-package ` belongs to `uv sync`/`uv add`. The image also + installs [`python/uv.toml`](python/uv.toml) as `/etc/uv/uv.toml`, listing + `rasterio fiona pyogrio gdal` for *both* interfaces, so anything you install + later inside the container is forced from source too (fiona/pyogrio wheels + bundle GDAL as well). Note `UV_NO_BINARY_PACKAGE` is not used: it only + affects `uv sync`/`uv add`, not `uv pip install` (verified on uv 0.12.13). +- **No `libgdal-dev` from apt.** The base image already ships `gdal-config`, + the headers and `libgdal.so` for the pinned GDAL; Ubuntu's `libgdal-dev` + would add a second, driver-less GDAL next to it — exactly the shadowing this + image exists to prevent (GDAL's own `docker/README.md` warns against it). + Only `build-essential` and `python3-dev` are added, and the build refuses to + continue if any `libgdal*` apt package is installed. +- Packages are pinned in [`python/requirements.txt`](python/requirements.txt); + the venv uses the system Python 3.14 (`UV_PYTHON_DOWNLOADS=never`). +- The last build step runs `verify-gdal-drivers` + ([`python/verify_gdal_drivers.py`](python/verify_gdal_drivers.py)), which + asserts: + 1. `gdal-config --version` == the pinned GDAL; + 2. `gdalinfo --formats` lists ECW and MrSID; + 3. `rasterio.__gdal_version__` == the pinned GDAL; + 4. `rasterio.Env().drivers()` includes ECW and MrSID; + 5. exactly one `libgdal` is mapped into the Python process and it is the + system one (no wheel-bundled copy); + 6. rioxarray round-trips an in-memory GeoTIFF. + + `verify-gdal-drivers` is on `PATH` in the image; run it any time (the dev + container runs it as `postCreateCommand`). + +## Using it in the dev container + +`.devcontainer/devcontainer.json` points at `dataforcanada/gdal-ecw-mrsid-python:3.13.3` +directly — no build on the VS Code side. `remoteUser` is `d4c`; VS Code remaps +its UID/GID to yours on first start so files in the bind-mounted workspace keep +sane ownership. The venv at `/opt/venv` is owned by that user, so +`uv pip install ` works inside the container without root (and honours +the no-binary rule above). + +Headless check with the Dev Containers CLI: + +```bash +npx --yes @devcontainers/cli@latest up --workspace-folder . +npx --yes @devcontainers/cli@latest exec --workspace-folder . verify-gdal-drivers +``` + +## Bumping GDAL + +1. `GDAL_VERSION=3.13.4 docker/gdal/build.sh` (also bump `PROJ_VERSION` if a + new PROJ is out). +2. `GDAL_VERSION=3.13.4 docker/python/build.sh`. +3. Change the tag in `.devcontainer/devcontainer.json` (two places) and + the `GDAL_VERSION` defaults in the two scripts and `python/Dockerfile`. +4. Rebuild the dev container. + +## Troubleshooting + +- **`ECW driver missing from 'gdalinfo --formats'` after stage 1** — the + GDAL configure step ran with uutils `uname`. Make sure the script built and + passed `dataforcanada/gdal-builder-base:26.04` as `BASE_IMAGE` (see above), + and in the build log look for `-DECW_ROOT=/opt/libecwj2-3.3 + -DMRSID_ROOT=/opt/Raster_DSDK` on the `GDAL_CMAKE_EXTRA_OPTS` echo line + and `Found ECW` / `Found MRSID` from CMake. +- **`rasterio.__gdal_version__` mismatch in stage 2** (e.g. it reports 3.12.4) + — a rasterio wheel got installed. Check that `--no-binary rasterio` and + `/etc/uv/uv.toml` are in effect (`uv pip install --dry-run -v rasterio` + should log `Selecting: rasterio==… (rasterio-….tar.gz)`) and that no + `libgdal*` apt package is present. +- **"this image is linux/amd64 only"** — you are building on a non-x86_64 + daemon; see Prerequisites. +- **Stale grids / wrong PROJ** — see the PROJ notes under stage 1. +- **Free space** — `docker builder prune` clears the BuildKit cache + (including ccache); `docker image rm dataforcanada/gdal-ecw-mrsid:3.13.3` + etc. removes images. diff --git a/docker/gdal/build.sh b/docker/gdal/build.sh new file mode 100755 index 0000000..c85efad --- /dev/null +++ b/docker/gdal/build.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# +# Stage 1: GDAL "ubuntu-full" image with the proprietary ECW + MrSID SDKs. +# +# Uses OSGeo's docker/ubuntu-full/Dockerfile *unmodified*, pinned to a GDAL +# release tag. Nothing in the GDAL checkout is patched; everything is driven by +# build args the upstream Dockerfile already exposes (WITH_ECW / WITH_MRSID +# download the SDKs themselves). +# +# Note: checking out the tag is NOT enough to pin the build. bh-gdal.sh wipes +# the copied checkout and downloads whatever GDAL_VERSION says (default: +# master). The checkout only contributes docker/ubuntu-full/*; the pin lives +# in --build-arg GDAL_VERSION=v, which is also how OSGeo builds releases. +# +# Note: the *builder* stage runs on builder-base.Dockerfile (ubuntu:26.04 with +# GNU `uname` first on PATH) via the upstream BASE_IMAGE arg, because +# bh-gdal.sh gates the ECW/MrSID CMake flags on `uname -p`, which prints +# "unknown" under Ubuntu 26.04's default uutils coreutils. See that file for +# details. The runtime image (TARGET_BASE_IMAGE) is stock ubuntu:26.04. +# +# Result: ${IMAGE}:${TAG} (default dataforcanada/gdal-ecw-mrsid:3.13.3) +# +# Usage: +# docker/gdal/build.sh # build + verify +# GDAL_VERSION=3.13.4 docker/gdal/build.sh # bump GDAL +# WITH_CCACHE= docker/gdal/build.sh # disable ccache +# BUILDKIT_PROGRESS=plain docker/gdal/build.sh 2>&1 | tee build.log +# +set -euo pipefail + +GDAL_VERSION="${GDAL_VERSION:-3.13.3}" # GDAL release tag, without the leading "v" +PROJ_VERSION="${PROJ_VERSION:-9.8.1}" # PROJ release tag (upstream default is "master") +IMAGE="${IMAGE:-dataforcanada/gdal-ecw-mrsid}" +TAG="${TAG:-${GDAL_VERSION}}" +WITH_CCACHE="${WITH_CCACHE-1}" # set to "" to disable +GDAL_REPO="${GDAL_REPO:-https://github.com/OSGeo/gdal.git}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SRC_DIR="${GDAL_SRC_DIR:-${SCRIPT_DIR}/src}" # git-ignored +PLATFORM=linux/amd64 + +die() { echo "ERROR: $*" >&2; exit 1; } + +# --- linux/amd64 only --------------------------------------------------------- +# Both SDKs are x86_64 glibc binaries and the upstream Dockerfile silently +# skips them on any other architecture (uname -m gate), which would yield an +# image *without* ECW/MrSID after an hours-long build. Fail fast instead. +daemon_arch="$(docker info --format '{{.Architecture}}')" +if [ "${daemon_arch}" != "x86_64" ] && [ "${D4C_ALLOW_EMULATED_BUILD:-}" != "1" ]; then + die "the Docker daemon runs on ${daemon_arch}, but this image is linux/amd64 only + (the ECW and MrSID SDKs are x86_64 binaries and OSGeo's Dockerfile only + installs them when 'uname -m' is x86_64). Build on an x86_64 host, or + set D4C_ALLOW_EMULATED_BUILD=1 to force a (very slow) QEMU build." +fi + +# --- source checkout at the pinned tag --------------------------------------- +if [ -d "${SRC_DIR}/.git" ]; then + git -C "${SRC_DIR}" fetch --depth 1 origin "refs/tags/v${GDAL_VERSION}:refs/tags/v${GDAL_VERSION}" + git -C "${SRC_DIR}" checkout -q "v${GDAL_VERSION}" +else + git clone --depth 1 --branch "v${GDAL_VERSION}" "${GDAL_REPO}" "${SRC_DIR}" +fi +# Sanity: make sure we are on the tag we think we are. +[ "$(git -C "${SRC_DIR}" describe --tags --exact-match)" = "v${GDAL_VERSION}" ] \ + || die "checkout in ${SRC_DIR} is not at v${GDAL_VERSION}" + +# --- builder base image (GNU uname on PATH; see builder-base.Dockerfile) ------ +BUILDER_BASE_IMAGE="${BUILDER_BASE_IMAGE:-dataforcanada/gdal-builder-base:26.04}" +echo ">>> Building ${BUILDER_BASE_IMAGE}" +docker buildx build \ + --platform "${PLATFORM}" \ + --file "${SCRIPT_DIR}/builder-base.Dockerfile" \ + --tag "${BUILDER_BASE_IMAGE}" \ + --load \ + "${SCRIPT_DIR}" + +# --- build -------------------------------------------------------------------- +# The Dockerfile does `COPY --link . gdal/`, so the build context must be the +# repo root and the Dockerfile referenced with -f. +echo ">>> Building ${IMAGE}:${TAG} (GDAL v${GDAL_VERSION}, PROJ ${PROJ_VERSION}, ECW + MrSID)" +cd "${SRC_DIR}" +docker buildx build \ + --platform "${PLATFORM}" \ + --file docker/ubuntu-full/Dockerfile \ + --build-arg BASE_IMAGE="${BUILDER_BASE_IMAGE}" \ + --build-arg GDAL_VERSION="v${GDAL_VERSION}" \ + --build-arg GDAL_BUILD_IS_RELEASE=YES \ + --build-arg PROJ_VERSION="${PROJ_VERSION}" \ + --build-arg WITH_ECW=yes \ + --build-arg WITH_MRSID=yes \ + --build-arg WITH_DEBUG_SYMBOLS=no \ + ${WITH_CCACHE:+--build-arg WITH_CCACHE=1} \ + --tag "${IMAGE}:${TAG}" \ + --load \ + . + +# --- verify (assert, don't print) -------------------------------------------- +echo ">>> Verifying ${IMAGE}:${TAG}" +run() { docker run --rm --platform "${PLATFORM}" "${IMAGE}:${TAG}" "$@"; } + +got_version="$(run gdal-config --version)" +[ "${got_version}" = "${GDAL_VERSION}" ] \ + || die "gdal-config --version is '${got_version}', expected '${GDAL_VERSION}'" + +formats="$(run gdalinfo --formats)" +grep -Eq '^ +ECW -raster-' <<<"${formats}" || die "ECW driver missing from 'gdalinfo --formats'" +grep -Eq '^ +MrSID -raster-' <<<"${formats}" || die "MrSID driver missing from 'gdalinfo --formats'" + +echo ">>> OK: ${IMAGE}:${TAG} — GDAL ${got_version} with ECW and MrSID" +grep -E '^ +(ECW|JP2ECW|MrSID|JP2MrSID) ' <<<"${formats}" diff --git a/docker/gdal/builder-base.Dockerfile b/docker/gdal/builder-base.Dockerfile new file mode 100644 index 0000000..c214cfd --- /dev/null +++ b/docker/gdal/builder-base.Dockerfile @@ -0,0 +1,33 @@ +# syntax=docker/dockerfile:1 +# +# Builder base for docker/gdal/build.sh: ubuntu:26.04 with GNU `uname` first +# on PATH. +# +# Why this exists: GDAL's docker/ubuntu-full/bh-gdal.sh only passes +# -DECW_ROOT / -DMRSID_ROOT to CMake when `uname -p` prints "x86_64". The +# Dockerfile's own SDK-download steps test `uname -m` (fine), but Ubuntu 26.04's +# default coreutils are uutils (Rust), whose `uname -p` prints "unknown". So on +# stock ubuntu:26.04 both SDKs are downloaded and then silently ignored by the +# GDAL configure step (OSGeo's CI never enables the proprietary SDKs, so +# nothing upstream catches it). +# +# Switching the whole system to GNU coreutils (coreutils-from-gnu) does not +# survive the upstream build: build-essential on 26.04 hard-depends on +# coreutils-from-uutils and is the first thing the upstream Dockerfile +# installs. Ubuntu does ship GNU coreutils alongside, as gnu-coreutils with +# gnu-prefixed binaries (/usr/bin/gnuuname), and apt never touches /usr/local, +# so a symlink there wins PATH lookup for the rest of the build. +# +# This image is passed as the upstream Dockerfile's BASE_IMAGE build arg, i.e. +# it is only the *builder* stage. TARGET_BASE_IMAGE (the runtime image) stays +# stock ubuntu:26.04. Nothing in the GDAL checkout is modified. +# +# Drop this once upstream changes that test to `uname -m` +# (docker/ubuntu-full/bh-gdal.sh, still `uname -p` on master as of 2026-09). + +ARG UBUNTU_IMAGE=ubuntu:26.04 +FROM ${UBUNTU_IMAGE} + +RUN test -x /usr/bin/gnuuname \ + && ln -s /usr/bin/gnuuname /usr/local/bin/uname \ + && test "$(uname -p)" = "x86_64" diff --git a/docker/python/Dockerfile b/docker/python/Dockerfile new file mode 100644 index 0000000..db14a72 --- /dev/null +++ b/docker/python/Dockerfile @@ -0,0 +1,120 @@ +# syntax=docker/dockerfile:1 +# +# Stage 2: Python environment (uv-managed venv with rioxarray/rasterio/xarray/ +# dask/numpy) on top of the stage-1 GDAL + ECW + MrSID image. +# +# To bump GDAL, change GDAL_VERSION (the base image tag) and rebuild. Nothing +# else here encodes the GDAL version: rasterio is compiled against whatever +# `gdal-config` the base image ships, and the build asserts that it matches. +# +# Build (from the repo root): +# docker/python/build.sh +# # or by hand: +# docker buildx build --platform linux/amd64 \ +# --build-arg GDAL_VERSION=3.13.3 \ +# -t dataforcanada/gdal-ecw-mrsid-python:3.13.3 docker/python + +ARG GDAL_VERSION=3.13.3 +ARG GDAL_IMAGE=dataforcanada/gdal-ecw-mrsid +ARG UV_VERSION=0.12.13 + +# uv ships as a static binary in a scratch image; copying it in is the +# documented install method for Docker and avoids a curl|sh. +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv + +FROM ${GDAL_IMAGE}:${GDAL_VERSION} +ARG GDAL_VERSION +ARG TARGETARCH + +# ---- linux/amd64 only -------------------------------------------------------- +# The ECW and MrSID SDKs are x86_64 binaries and the upstream GDAL Dockerfile +# only installs them on x86_64, so on any other architecture the base image +# would silently lack both drivers. Fail here, before doing any work. +RUN if [ "${TARGETARCH:-}" != "amd64" ] || [ "$(uname -m)" != "x86_64" ]; then \ + echo "ERROR: this image is linux/amd64 only (TARGETARCH=${TARGETARCH:-unset}, uname -m=$(uname -m))." >&2; \ + echo " The ECW and MrSID SDKs are x86_64 binaries; build with --platform linux/amd64 on an x86_64 host." >&2; \ + exit 1; \ + fi + +# ---- build deps for compiling rasterio against the image's GDAL ------------- +# Deliberately NOT installing libgdal-dev: the base image already provides +# gdal-config, the headers and libgdal.so for the pinned GDAL. Ubuntu's +# libgdal-dev would install a *second*, driver-less GDAL next to it, which is +# exactly the shadowing this image exists to prevent (see the warning in +# GDAL's docker/README.md). git is for the dev-container use case. +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + export DEBIAN_FRONTEND=noninteractive \ + && apt-get update -y \ + && apt-get install -y --no-install-recommends \ + build-essential \ + python3-dev \ + git \ + openssh-client + +# Guard: gdal-config must be the pinned GDAL, and no apt GDAL may be present. +RUN test "$(gdal-config --version)" = "${GDAL_VERSION}" \ + || { echo "ERROR: gdal-config --version is '$(gdal-config --version)', expected '${GDAL_VERSION}'" >&2; exit 1; } \ + && if dpkg-query -W -f '${Status} ${Package}\n' 'libgdal*' 2>/dev/null | grep '^install ok installed'; then \ + echo "ERROR: apt-provided GDAL packages are installed (above); they would shadow the built GDAL. Refusing to build." >&2; \ + exit 1; \ + fi + +COPY --from=uv /uv /uvx /usr/local/bin/ +# Image-wide default: never install wheels that bundle libgdal (see file). +COPY uv.toml /etc/uv/uv.toml + +# ---- non-root runtime user --------------------------------------------------- +# Ubuntu 26.04 ships a stock "ubuntu" user at uid 1000; drop it so USER_UID can +# default to 1000 (what dev-container UID remapping and most hosts expect). +ARG USERNAME=d4c +ARG USER_UID=1000 +ARG USER_GID=${USER_UID} +RUN if id ubuntu >/dev/null 2>&1; then userdel -r ubuntu; fi \ + && groupadd --gid "${USER_GID}" "${USERNAME}" \ + && useradd --uid "${USER_UID}" --gid "${USER_GID}" --create-home --shell /bin/bash "${USERNAME}" \ + && mkdir -p /opt/venv \ + && chown "${USER_UID}:${USER_GID}" /opt/venv \ + && echo "source /usr/share/bash-completion/bash_completion" >> "/home/${USERNAME}/.bashrc" + +COPY --chmod=0755 verify_gdal_drivers.py /usr/local/bin/verify-gdal-drivers + +# VIRTUAL_ENV/PATH: the venv is the default python for everything downstream. +# UV_PYTHON_DOWNLOADS=never: always use the system interpreter, never a +# uv-managed one. +# GDAL_CONFIG: what rasterio's setup.py consults; explicit rather than PATH luck. +# D4C_GDAL_VERSION: what verify-gdal-drivers checks against at runtime. +# (Not named GDAL_VERSION: rasterio's setup.py treats that as an override.) +# (No UV_NO_BINARY_PACKAGE here on purpose: it does not apply to +# `uv pip install`; /etc/uv/uv.toml covers both interfaces.) +ENV VIRTUAL_ENV=/opt/venv \ + PATH=/opt/venv/bin:${PATH} \ + UV_PYTHON_DOWNLOADS=never \ + UV_LINK_MODE=copy \ + UV_COMPILE_BYTECODE=1 \ + GDAL_CONFIG=/usr/bin/gdal-config \ + D4C_GDAL_VERSION=${GDAL_VERSION} + +USER ${USERNAME} +WORKDIR /home/${USERNAME} + +# ---- venv + packages --------------------------------------------------------- +# --no-binary rasterio is the critical bit: rasterio's PyPI wheels bundle their +# own libgdal (built without ECW/MrSID). Building the sdist makes it link +# against this image's libgdal via gdal-config. (`uv pip install` takes the +# pip-style `--no-binary `; `--no-binary-package` is the `uv sync`/`uv add` +# spelling. /etc/uv/uv.toml already says the same; the flag is belt-and-braces.) +COPY --chown=${USER_UID}:${USER_GID} requirements.txt /tmp/requirements.txt +RUN --mount=type=cache,target=/home/${USERNAME}/.cache/uv,uid=${USER_UID},gid=${USER_GID} \ + uv venv --python /usr/bin/python3 "${VIRTUAL_ENV}" \ + && uv pip install --no-binary rasterio --requirements /tmp/requirements.txt \ + && rm /tmp/requirements.txt + +# ---- verification: the build fails unless all of these hold ------------------ +# gdalinfo --formats lists ECW and MrSID +# rasterio.__gdal_version__ == GDAL_VERSION +# rasterio.Env().drivers() includes ECW and MrSID +# (+ exactly one libgdal, the system one, is loaded; rioxarray round-trips) +RUN verify-gdal-drivers --expect-gdal "${GDAL_VERSION}" + +CMD ["/bin/bash", "-l"] diff --git a/docker/python/build.sh b/docker/python/build.sh new file mode 100755 index 0000000..964f2aa --- /dev/null +++ b/docker/python/build.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# Stage 2: Python (uv venv with rioxarray/rasterio/xarray/dask/numpy) on top of +# the stage-1 GDAL + ECW + MrSID image. Requires stage 1 to exist locally: +# docker/gdal/build.sh +# +# Result: ${IMAGE}:${TAG} (default dataforcanada/gdal-ecw-mrsid-python:3.13.3) +# +# Usage: +# docker/python/build.sh +# GDAL_VERSION=3.13.4 docker/python/build.sh # after building stage 1 for it +# USER_UID=$(id -u) USER_GID=$(id -g) docker/python/build.sh +# +# The build itself runs verify-gdal-drivers as its last step, so a successful +# build already implies: ECW + MrSID in gdalinfo --formats, rasterio linked to +# GDAL ${GDAL_VERSION}, and ECW + MrSID in rasterio.Env().drivers(). +# +set -euo pipefail + +GDAL_VERSION="${GDAL_VERSION:-3.13.3}" +GDAL_IMAGE="${GDAL_IMAGE:-dataforcanada/gdal-ecw-mrsid}" +IMAGE="${IMAGE:-dataforcanada/gdal-ecw-mrsid-python}" +TAG="${TAG:-${GDAL_VERSION}}" +USER_UID="${USER_UID:-1000}" +USER_GID="${USER_GID:-${USER_UID}}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLATFORM=linux/amd64 + +die() { echo "ERROR: $*" >&2; exit 1; } + +docker image inspect "${GDAL_IMAGE}:${GDAL_VERSION}" >/dev/null 2>&1 \ + || die "base image ${GDAL_IMAGE}:${GDAL_VERSION} not found locally — run docker/gdal/build.sh first" + +echo ">>> Building ${IMAGE}:${TAG} FROM ${GDAL_IMAGE}:${GDAL_VERSION}" +docker buildx build \ + --platform "${PLATFORM}" \ + --build-arg GDAL_VERSION="${GDAL_VERSION}" \ + --build-arg GDAL_IMAGE="${GDAL_IMAGE}" \ + --build-arg USER_UID="${USER_UID}" \ + --build-arg USER_GID="${USER_GID}" \ + --tag "${IMAGE}:${TAG}" \ + --load \ + "${SCRIPT_DIR}" + +# Re-run the assertions against the finished image, as the runtime user. +echo ">>> Verifying ${IMAGE}:${TAG}" +docker run --rm --platform "${PLATFORM}" "${IMAGE}:${TAG}" verify-gdal-drivers --expect-gdal "${GDAL_VERSION}" +echo ">>> OK: ${IMAGE}:${TAG}" diff --git a/docker/python/requirements.txt b/docker/python/requirements.txt new file mode 100644 index 0000000..76eae41 --- /dev/null +++ b/docker/python/requirements.txt @@ -0,0 +1,14 @@ +# Python stack for the stage-2 image (docker/python/Dockerfile). +# +# rasterio is *always* built from source here so it links against the image's +# GDAL (with ECW + MrSID) instead of the libgdal bundled in its PyPI wheels. +# The Dockerfile enforces that with `uv pip install --no-binary rasterio` and +# /etc/uv/uv.toml makes later installs in the image do the same. +# +# Versions pinned 2026-09-14 (all support Python 3.14, the Ubuntu 26.04 system +# interpreter this image uses). Bump deliberately, then rebuild + re-verify. +numpy==2.5.3 +rasterio==1.5.1 +rioxarray==0.23.0 +xarray==2026.7.0 +dask[array]==2026.8.0 diff --git a/docker/python/uv.toml b/docker/python/uv.toml new file mode 100644 index 0000000..aa0c269 --- /dev/null +++ b/docker/python/uv.toml @@ -0,0 +1,16 @@ +# System-wide uv config for the image, installed at /etc/uv/uv.toml. +# +# Packages whose PyPI wheels bundle their own libgdal. A bundled libgdal would +# shadow the image's GDAL (built with ECW + MrSID) and silently lose both +# drivers, so uv must always build these from source against `gdal-config`. +# +# Two keys because uv has two interfaces with different option names: +# - [pip] no-binary -> `uv pip install ...` +# - no-binary-package -> `uv sync` / `uv add` / `uv lock` / `uv run` +# (UV_NO_BINARY_PACKAGE only affects the second one — verified on uv 0.12.13.) +# A project's own uv.toml/pyproject.toml can override this; it is a default. + +no-binary-package = ["rasterio", "fiona", "pyogrio", "gdal"] + +[pip] +no-binary = ["rasterio", "fiona", "pyogrio", "gdal"] diff --git a/docker/python/verify_gdal_drivers.py b/docker/python/verify_gdal_drivers.py new file mode 100644 index 0000000..c226190 --- /dev/null +++ b/docker/python/verify_gdal_drivers.py @@ -0,0 +1,136 @@ +#!/opt/venv/bin/python +"""Assert that GDAL + rasterio in this environment are the ones we built. + +Every check is an assertion: the first failure exits non-zero with a message +naming what was expected and what was found. Nothing is merely printed for a +human to eyeball. Used as a `RUN` step in docker/python/Dockerfile (so the +image build fails), as the dev container's postCreateCommand, and runnable by +hand at any time: `verify-gdal-drivers [--expect-gdal X.Y.Z]`. + +Checks + 1. `gdal-config --version` == expected GDAL version (the base image is the + pinned one, and gdal-config is the one rasterio was compiled against). + 2. `gdalinfo --formats` lists ECW and MrSID. + 3. `rasterio.__gdal_version__` == expected GDAL version. + 4. `rasterio.Env().drivers()` includes ECW and MrSID. + 5. The libgdal mapped into this Python process is the system one, and there + is exactly one — i.e. no wheel-bundled libgdal shadowing it. + 6. rioxarray can open a raster through that stack (in-memory GeoTIFF only; + no ECW/MrSID files are touched). + +The expected version comes from --expect-gdal, else $D4C_GDAL_VERSION (baked +into the image from the GDAL_VERSION build arg), else `gdal-config --version`. +""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys + +REQUIRED_DRIVERS = ("ECW", "MrSID") + + +def fail(msg: str) -> None: + print(f"FAIL: {msg}", file=sys.stderr) + sys.exit(1) + + +def ok(msg: str) -> None: + print(f"ok: {msg}") + + +def run(*cmd: str) -> str: + try: + return subprocess.check_output(cmd, text=True, stderr=subprocess.STDOUT).strip() + except (OSError, subprocess.CalledProcessError) as exc: + fail(f"{' '.join(cmd)}: {exc}") + raise # unreachable; keeps type checkers happy + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + parser.add_argument("--expect-gdal", metavar="X.Y.Z", help="expected GDAL version") + args = parser.parse_args() + + gdal_config = os.environ.get("GDAL_CONFIG", "gdal-config") + config_version = run(gdal_config, "--version") + expected = args.expect_gdal or os.environ.get("D4C_GDAL_VERSION") or config_version + if not re.fullmatch(r"\d+\.\d+\.\d+", expected): + fail(f"expected GDAL version {expected!r} is not of the form X.Y.Z") + + # 1. gdal-config is the pinned GDAL + if config_version != expected: + fail(f"{gdal_config} --version is {config_version!r}, expected {expected!r}") + ok(f"gdal-config --version == {expected}") + + # 2. gdalinfo --formats lists both proprietary drivers + formats = run("gdalinfo", "--formats") + for drv in REQUIRED_DRIVERS: + if not re.search(rf"^\s+{re.escape(drv)} -raster-", formats, re.MULTILINE): + fail(f"{drv} missing from 'gdalinfo --formats'") + ok("gdalinfo --formats lists ECW and MrSID") + + # 3. rasterio links against the pinned GDAL + try: + import rasterio + except ImportError as exc: + fail(f"cannot import rasterio: {exc}") + if rasterio.__gdal_version__ != expected: + fail( + f"rasterio.__gdal_version__ is {rasterio.__gdal_version__!r}, expected " + f"{expected!r} — rasterio is not using the image's GDAL " + "(was a PyPI wheel with bundled libgdal installed?)" + ) + ok(f"rasterio {rasterio.__version__} reports GDAL {rasterio.__gdal_version__}") + + # 4. rasterio sees the drivers (drivers() needs an *entered* Env) + with rasterio.Env() as env: + drivers = env.drivers() + for drv in REQUIRED_DRIVERS: + if drv not in drivers: + fail(f"{drv} missing from rasterio.Env().drivers() ({len(drivers)} drivers registered)") + ok("rasterio.Env().drivers() includes ECW and MrSID") + + # 5. exactly one libgdal in this process, and it is the system one + with open("/proc/self/maps") as maps: + libgdal = sorted( + {line.split()[-1] for line in maps if "/libgdal" in line and line.split()[-1].startswith("/")} + ) + if len(libgdal) != 1: + fail(f"expected exactly one libgdal mapped into the process, found {libgdal}") + if not libgdal[0].startswith("/usr/lib/") or "site-packages" in libgdal[0]: + fail(f"libgdal is loaded from {libgdal[0]}, not from the system GDAL under /usr/lib/") + ok(f"single system libgdal in process: {libgdal[0]}") + + # 6. rioxarray works end-to-end on an in-memory GeoTIFF + try: + import numpy as np + import rioxarray + from rasterio.transform import from_origin + except ImportError as exc: + fail(f"cannot import the raster stack: {exc}") + path = "/vsimem/verify.tif" + with rasterio.Env(): + data = np.arange(16, dtype="uint8").reshape(1, 4, 4) + with rasterio.open( + path, "w", driver="GTiff", width=4, height=4, count=1, dtype="uint8", + crs="EPSG:3857", transform=from_origin(0, 4, 1, 1), + ) as dst: + dst.write(data) + # Context manager: an open dataset left to the garbage collector gets + # finalized during interpreter teardown and prints a spurious + # "Error in sys.excepthook" at exit. + with rioxarray.open_rasterio(path) as src: + da = src.load() + if da.shape != (1, 4, 4) or int(da.sum()) != int(data.sum()) or da.rio.crs.to_epsg() != 3857: + fail(f"rioxarray round-trip mismatch: shape={da.shape} crs={da.rio.crs}") + ok(f"rioxarray {rioxarray.__version__} round-trips an in-memory raster") + + print(f"OK: GDAL {expected} with ECW + MrSID, rasterio {rasterio.__version__} linked against it") + + +if __name__ == "__main__": + main()