dataforcanada/d4c-datapkg-orthoimagery
1
0
Fork 0
mirror of https://github.com/dataforcanada/d4c-datapkg-orthoimagery.git synced 2026-10-02 12:15:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

Changes to Docker image

Browse Source
This commit is contained in:
Diego Ripley
2026-09-26 10:11:13 -04:00
parent ddec949b03
commit 6dd4c9413f
7 changed files with 2876 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/python/Dockerfile
+36 -14
View File
@@ -1,18 +1,26 @@
# syntax=docker/dockerfile:1
#
# Stage 2: Python environment (uv-managed venv with rioxarray/rasterio/xarray/
# dask/numpy) on top of the stage-1 GDAL + ECW + MrSID image.
# Stage 2: Python environment (uv-managed venv holding the repo's
# pyproject.toml + uv.lock dependency set — rioxarray, rasterio, xarray, dask,
# numpy, portolan-cli) on top of the stage-1 GDAL + ECW + MrSID image.
#
# To bump GDAL, change GDAL_VERSION (the base image tag) and rebuild. Nothing
# else here encodes the GDAL version: rasterio is compiled against whatever
# `gdal-config` the base image ships, and the build asserts that it matches.
# To change Python packages, edit ../../pyproject.toml (via `uv add`) and commit
# the regenerated ../../uv.lock; `uv sync --locked` below will pick them up.
#
# The build context is the REPO ROOT, not this directory, because pyproject.toml
# and uv.lock live there. .dockerignore keeps that context tiny — data/ and
# scripts/ hold hundreds of GB of imagery and are excluded.
#
# Build (from the repo root):
# docker/python/build.sh
# # or by hand:
# docker buildx build --platform linux/amd64 \
# --build-arg GDAL_VERSION=3.13.3 \
# -t dataforcanada/gdal-ecw-mrsid-python:3.13.3 docker/python
# --file docker/python/Dockerfile \
# -t dataforcanada/gdal-ecw-mrsid-python:3.13.3 .
ARG GDAL_VERSION=3.13.3
ARG GDAL_IMAGE=dataforcanada/gdal-ecw-mrsid
@@ -62,7 +70,7 @@ RUN test "$(gdal-config --version)" = "${GDAL_VERSION}" \
COPY --from=uv /uv /uvx /usr/local/bin/
# Image-wide default: never install wheels that bundle libgdal (see file).
COPY uv.toml /etc/uv/uv.toml
COPY docker/python/uv.toml /etc/uv/uv.toml
# ---- non-root runtime user ---------------------------------------------------
# Ubuntu 26.04 ships a stock "ubuntu" user at uid 1000; drop it so USER_UID can
@@ -77,18 +85,24 @@ RUN if id ubuntu >/dev/null 2>&1; then userdel -r ubuntu; fi \
&& chown "${USER_UID}:${USER_GID}" /opt/venv \
&& echo "source /usr/share/bash-completion/bash_completion" >> "/home/${USERNAME}/.bashrc"
COPY --chmod=0755 verify_gdal_drivers.py /usr/local/bin/verify-gdal-drivers
COPY --chmod=0755 docker/python/verify_gdal_drivers.py /usr/local/bin/verify-gdal-drivers
# VIRTUAL_ENV/PATH: the venv is the default python for everything downstream.
# UV_PROJECT_ENVIRONMENT: point uv's *project* interface at that same venv, so
# `uv sync`/`uv add`/`uv run` in the repo update /opt/venv instead of creating
# a ./.venv next to pyproject.toml — both in this build and, more importantly,
# in the dev container, where the workspace is bind-mounted.
# UV_PYTHON_DOWNLOADS=never: always use the system interpreter, never a
# uv-managed one.
# GDAL_CONFIG: what rasterio's setup.py consults; explicit rather than PATH luck.
# D4C_GDAL_VERSION: what verify-gdal-drivers checks against at runtime.
# (Not named GDAL_VERSION: rasterio's setup.py treats that as an override.)
# (No UV_NO_BINARY_PACKAGE here on purpose: it does not apply to
# `uv pip install`; /etc/uv/uv.toml covers both interfaces.)
# `uv pip install`. /etc/uv/uv.toml covers both interfaces image-wide, and
# pyproject.toml's [tool.uv] covers the project interface for this repo.)
ENV VIRTUAL_ENV=/opt/venv \
PATH=/opt/venv/bin:${PATH} \
UV_PROJECT_ENVIRONMENT=/opt/venv \
UV_PYTHON_DOWNLOADS=never \
UV_LINK_MODE=copy \
UV_COMPILE_BYTECODE=1 \
@@ -99,21 +113,29 @@ USER ${USERNAME}
WORKDIR /home/${USERNAME}
# ---- venv + packages ---------------------------------------------------------
# --no-binary rasterio is the critical bit: rasterio's PyPI wheels bundle their
# own libgdal (built without ECW/MrSID). Building the sdist makes it link
# against this image's libgdal via gdal-config. (`uv pip install` takes the
# pip-style `--no-binary <pkg>`; `--no-binary-package` is the `uv sync`/`uv add`
# spelling. /etc/uv/uv.toml already says the same; the flag is belt-and-braces.)
COPY --chown=${USER_UID}:${USER_GID} requirements.txt /tmp/requirements.txt
# The repo's pyproject.toml + uv.lock are the single source of truth for this
# environment; `--locked` fails the build if they have drifted apart, so the
# image can never be built from an unlocked (i.e. unreproducible) dependency
# set. The lock pins the whole transitive tree, not just the direct deps.
#
# --no-binary-package is the critical bit: the PyPI wheels for rasterio and
# pyogrio (the latter pulled in by portolan-cli) bundle their own libgdal, built
# without ECW/MrSID. Building the sdists makes them link against this image's
# libgdal via gdal-config. Both pyproject.toml's [tool.uv] and /etc/uv/uv.toml
# already say this; the flags are belt-and-braces. (`uv sync`/`uv add` take
# `--no-binary-package <pkg>`; `uv pip install` takes pip-style `--no-binary`.)
COPY --chown=${USER_UID}:${USER_GID} pyproject.toml uv.lock /tmp/project/
RUN --mount=type=cache,target=/home/${USERNAME}/.cache/uv,uid=${USER_UID},gid=${USER_GID} \
uv venv --python /usr/bin/python3 "${VIRTUAL_ENV}" \
&& uv pip install --no-binary rasterio --requirements /tmp/requirements.txt \
&& rm /tmp/requirements.txt
&& uv sync --project /tmp/project --locked --no-dev \
--no-binary-package rasterio --no-binary-package pyogrio \
&& rm -rf /tmp/project
# ---- verification: the build fails unless all of these hold ------------------
# gdalinfo --formats lists ECW and MrSID
# rasterio.__gdal_version__ == GDAL_VERSION
# rasterio.Env().drivers() includes ECW and MrSID
# pyogrio.__gdal_version_string__ == GDAL_VERSION
# (+ exactly one libgdal, the system one, is loaded; rioxarray round-trips)
RUN verify-gdal-drivers --expect-gdal "${GDAL_VERSION}"
docker/python/build.sh
+16 -3
View File
@@ -1,9 +1,14 @@
#!/usr/bin/env bash
#
# Stage 2: Python (uv venv with rioxarray/rasterio/xarray/dask/numpy) on top of
# the stage-1 GDAL + ECW + MrSID image. Requires stage 1 to exist locally:
# Stage 2: Python (uv venv holding the repo's pyproject.toml + uv.lock set:
# rioxarray, rasterio, xarray, dask, numpy, portolan-cli) on top of the stage-1
# GDAL + ECW + MrSID image. Requires stage 1 to exist locally:
# docker/gdal/build.sh
#
# The build context is the repo root (that is where pyproject.toml and uv.lock
# live); .dockerignore keeps it to a handful of files rather than the hundreds
# of GB of imagery under data/ and scripts/.
#
# Result: ${IMAGE}:${TAG} (default dataforcanada/gdal-ecw-mrsid-python:3.13.3)
#
# Usage:
@@ -24,6 +29,7 @@ TAG="${TAG:-${GDAL_VERSION}}"
USER_UID="${USER_UID:-1000}"
USER_GID="${USER_GID:-${USER_UID}}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
PLATFORM=linux/amd64
die() { echo "ERROR: $*" >&2; exit 1; }
@@ -31,6 +37,12 @@ die() { echo "ERROR: $*" >&2; exit 1; }
docker image inspect "${GDAL_IMAGE}:${GDAL_VERSION}" >/dev/null 2>&1 \
|| die "base image ${GDAL_IMAGE}:${GDAL_VERSION} not found locally — run docker/gdal/build.sh first"
# The image is built straight from these two; `uv sync --locked` in the
# Dockerfile refuses to build if they have drifted apart.
for f in pyproject.toml uv.lock; do
[ -f "${REPO_ROOT}/${f}" ] || die "${REPO_ROOT}/${f} not found — the image is built from it"
done
echo ">>> Building ${IMAGE}:${TAG} FROM ${GDAL_IMAGE}:${GDAL_VERSION}"
docker buildx build \
--platform "${PLATFORM}" \
@@ -39,8 +51,9 @@ docker buildx build \
--build-arg USER_UID="${USER_UID}" \
--build-arg USER_GID="${USER_GID}" \
--tag "${IMAGE}:${TAG}" \
--file "${SCRIPT_DIR}/Dockerfile" \
--load \
"${SCRIPT_DIR}"
"${REPO_ROOT}"
# Re-run the assertions against the finished image, as the runtime user.
echo ">>> Verifying ${IMAGE}:${TAG}"
docker/python/verify_gdal_drivers.py
+23 -5
View File
@@ -13,9 +13,13 @@ Checks
2. `gdalinfo --formats` lists ECW and MrSID.
3. `rasterio.__gdal_version__` == expected GDAL version.
4. `rasterio.Env().drivers()` includes ECW and MrSID.
5. The libgdal mapped into this Python process is the system one, and there
is exactly one — i.e. no wheel-bundled libgdal shadowing it.
6. rioxarray can open a raster through that stack (in-memory GeoTIFF only;
5. `pyogrio.__gdal_version_string__` == expected GDAL version. pyogrio is the
other GDAL-linked extension in the image (pulled in by portolan-cli) and
its wheels bundle libgdal too.
6. The libgdal mapped into this Python process is the system one, and there
is exactly one — i.e. no wheel-bundled libgdal shadowing it. This runs
after both extensions are imported, so it covers rasterio and pyogrio.
7. rioxarray can open a raster through that stack (in-memory GeoTIFF only;
no ECW/MrSID files are touched).
The expected version comes from --expect-gdal, else $D4C_GDAL_VERSION (baked
@@ -94,7 +98,21 @@ def main() -> None:
fail(f"{drv} missing from rasterio.Env().drivers() ({len(drivers)} drivers registered)")
ok("rasterio.Env().drivers() includes ECW and MrSID")
# 5. exactly one libgdal in this process, and it is the system one
# 5. pyogrio links against the pinned GDAL too. Imported here, before the
# libgdal count below, so a wheel-bundled libgdal of its own is caught.
try:
import pyogrio
except ImportError as exc:
fail(f"cannot import pyogrio: {exc}")
if pyogrio.__gdal_version_string__ != expected:
fail(
f"pyogrio.__gdal_version_string__ is {pyogrio.__gdal_version_string__!r}, "
f"expected {expected!r} — pyogrio is not using the image's GDAL "
"(was a PyPI wheel with bundled libgdal installed?)"
)
ok(f"pyogrio {pyogrio.__version__} reports GDAL {pyogrio.__gdal_version_string__}")
# 6. exactly one libgdal in this process, and it is the system one
with open("/proc/self/maps") as maps:
libgdal = sorted(
{line.split()[-1] for line in maps if "/libgdal" in line and line.split()[-1].startswith("/")}
@@ -105,7 +123,7 @@ def main() -> None:
fail(f"libgdal is loaded from {libgdal[0]}, not from the system GDAL under /usr/lib/")
ok(f"single system libgdal in process: {libgdal[0]}")
# 6. rioxarray works end-to-end on an in-memory GeoTIFF
# 7. rioxarray works end-to-end on an in-memory GeoTIFF
try:
import numpy as np
import rioxarray