mirror of
https://github.com/dataforcanada/d4c-datapkg-orthoimagery.git
synced 2026-10-02 20:25:44 +02:00
Work on #152
This commit is contained in:
Diego Ripley
@@ -0,0 +1,120 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
#
|
||||
# Stage 2: Python environment (uv-managed venv with rioxarray/rasterio/xarray/
|
||||
# dask/numpy) on top of the stage-1 GDAL + ECW + MrSID image.
|
||||
#
|
||||
# To bump GDAL, change GDAL_VERSION (the base image tag) and rebuild. Nothing
|
||||
# else here encodes the GDAL version: rasterio is compiled against whatever
|
||||
# `gdal-config` the base image ships, and the build asserts that it matches.
|
||||
#
|
||||
# Build (from the repo root):
|
||||
# docker/python/build.sh
|
||||
# # or by hand:
|
||||
# docker buildx build --platform linux/amd64 \
|
||||
# --build-arg GDAL_VERSION=3.13.3 \
|
||||
# -t dataforcanada/gdal-ecw-mrsid-python:3.13.3 docker/python
|
||||
|
||||
ARG GDAL_VERSION=3.13.3
|
||||
ARG GDAL_IMAGE=dataforcanada/gdal-ecw-mrsid
|
||||
ARG UV_VERSION=0.12.13
|
||||
|
||||
# uv ships as a static binary in a scratch image; copying it in is the
|
||||
# documented install method for Docker and avoids a curl|sh.
|
||||
FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv
|
||||
|
||||
FROM ${GDAL_IMAGE}:${GDAL_VERSION}
|
||||
ARG GDAL_VERSION
|
||||
ARG TARGETARCH
|
||||
|
||||
# ---- linux/amd64 only --------------------------------------------------------
|
||||
# The ECW and MrSID SDKs are x86_64 binaries and the upstream GDAL Dockerfile
|
||||
# only installs them on x86_64, so on any other architecture the base image
|
||||
# would silently lack both drivers. Fail here, before doing any work.
|
||||
RUN if [ "${TARGETARCH:-}" != "amd64" ] || [ "$(uname -m)" != "x86_64" ]; then \
|
||||
echo "ERROR: this image is linux/amd64 only (TARGETARCH=${TARGETARCH:-unset}, uname -m=$(uname -m))." >&2; \
|
||||
echo " The ECW and MrSID SDKs are x86_64 binaries; build with --platform linux/amd64 on an x86_64 host." >&2; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# ---- build deps for compiling rasterio against the image's GDAL -------------
|
||||
# Deliberately NOT installing libgdal-dev: the base image already provides
|
||||
# gdal-config, the headers and libgdal.so for the pinned GDAL. Ubuntu's
|
||||
# libgdal-dev would install a *second*, driver-less GDAL next to it, which is
|
||||
# exactly the shadowing this image exists to prevent (see the warning in
|
||||
# GDAL's docker/README.md). git is for the dev-container use case.
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
export DEBIAN_FRONTEND=noninteractive \
|
||||
&& apt-get update -y \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
python3-dev \
|
||||
git \
|
||||
openssh-client
|
||||
|
||||
# Guard: gdal-config must be the pinned GDAL, and no apt GDAL may be present.
|
||||
RUN test "$(gdal-config --version)" = "${GDAL_VERSION}" \
|
||||
|| { echo "ERROR: gdal-config --version is '$(gdal-config --version)', expected '${GDAL_VERSION}'" >&2; exit 1; } \
|
||||
&& if dpkg-query -W -f '${Status} ${Package}\n' 'libgdal*' 2>/dev/null | grep '^install ok installed'; then \
|
||||
echo "ERROR: apt-provided GDAL packages are installed (above); they would shadow the built GDAL. Refusing to build." >&2; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
COPY --from=uv /uv /uvx /usr/local/bin/
|
||||
# Image-wide default: never install wheels that bundle libgdal (see file).
|
||||
COPY uv.toml /etc/uv/uv.toml
|
||||
|
||||
# ---- non-root runtime user ---------------------------------------------------
|
||||
# Ubuntu 26.04 ships a stock "ubuntu" user at uid 1000; drop it so USER_UID can
|
||||
# default to 1000 (what dev-container UID remapping and most hosts expect).
|
||||
ARG USERNAME=d4c
|
||||
ARG USER_UID=1000
|
||||
ARG USER_GID=${USER_UID}
|
||||
RUN if id ubuntu >/dev/null 2>&1; then userdel -r ubuntu; fi \
|
||||
&& groupadd --gid "${USER_GID}" "${USERNAME}" \
|
||||
&& useradd --uid "${USER_UID}" --gid "${USER_GID}" --create-home --shell /bin/bash "${USERNAME}" \
|
||||
&& mkdir -p /opt/venv \
|
||||
&& chown "${USER_UID}:${USER_GID}" /opt/venv \
|
||||
&& echo "source /usr/share/bash-completion/bash_completion" >> "/home/${USERNAME}/.bashrc"
|
||||
|
||||
COPY --chmod=0755 verify_gdal_drivers.py /usr/local/bin/verify-gdal-drivers
|
||||
|
||||
# VIRTUAL_ENV/PATH: the venv is the default python for everything downstream.
|
||||
# UV_PYTHON_DOWNLOADS=never: always use the system interpreter, never a
|
||||
# uv-managed one.
|
||||
# GDAL_CONFIG: what rasterio's setup.py consults; explicit rather than PATH luck.
|
||||
# D4C_GDAL_VERSION: what verify-gdal-drivers checks against at runtime.
|
||||
# (Not named GDAL_VERSION: rasterio's setup.py treats that as an override.)
|
||||
# (No UV_NO_BINARY_PACKAGE here on purpose: it does not apply to
|
||||
# `uv pip install`; /etc/uv/uv.toml covers both interfaces.)
|
||||
ENV VIRTUAL_ENV=/opt/venv \
|
||||
PATH=/opt/venv/bin:${PATH} \
|
||||
UV_PYTHON_DOWNLOADS=never \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_COMPILE_BYTECODE=1 \
|
||||
GDAL_CONFIG=/usr/bin/gdal-config \
|
||||
D4C_GDAL_VERSION=${GDAL_VERSION}
|
||||
|
||||
USER ${USERNAME}
|
||||
WORKDIR /home/${USERNAME}
|
||||
|
||||
# ---- venv + packages ---------------------------------------------------------
|
||||
# --no-binary rasterio is the critical bit: rasterio's PyPI wheels bundle their
|
||||
# own libgdal (built without ECW/MrSID). Building the sdist makes it link
|
||||
# against this image's libgdal via gdal-config. (`uv pip install` takes the
|
||||
# pip-style `--no-binary <pkg>`; `--no-binary-package` is the `uv sync`/`uv add`
|
||||
# spelling. /etc/uv/uv.toml already says the same; the flag is belt-and-braces.)
|
||||
COPY --chown=${USER_UID}:${USER_GID} requirements.txt /tmp/requirements.txt
|
||||
RUN --mount=type=cache,target=/home/${USERNAME}/.cache/uv,uid=${USER_UID},gid=${USER_GID} \
|
||||
uv venv --python /usr/bin/python3 "${VIRTUAL_ENV}" \
|
||||
&& uv pip install --no-binary rasterio --requirements /tmp/requirements.txt \
|
||||
&& rm /tmp/requirements.txt
|
||||
|
||||
# ---- verification: the build fails unless all of these hold ------------------
|
||||
# gdalinfo --formats lists ECW and MrSID
|
||||
# rasterio.__gdal_version__ == GDAL_VERSION
|
||||
# rasterio.Env().drivers() includes ECW and MrSID
|
||||
# (+ exactly one libgdal, the system one, is loaded; rioxarray round-trips)
|
||||
RUN verify-gdal-drivers --expect-gdal "${GDAL_VERSION}"
|
||||
|
||||
CMD ["/bin/bash", "-l"]
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Stage 2: Python (uv venv with rioxarray/rasterio/xarray/dask/numpy) on top of
|
||||
# the stage-1 GDAL + ECW + MrSID image. Requires stage 1 to exist locally:
|
||||
# docker/gdal/build.sh
|
||||
#
|
||||
# Result: ${IMAGE}:${TAG} (default dataforcanada/gdal-ecw-mrsid-python:3.13.3)
|
||||
#
|
||||
# Usage:
|
||||
# docker/python/build.sh
|
||||
# GDAL_VERSION=3.13.4 docker/python/build.sh # after building stage 1 for it
|
||||
# USER_UID=$(id -u) USER_GID=$(id -g) docker/python/build.sh
|
||||
#
|
||||
# The build itself runs verify-gdal-drivers as its last step, so a successful
|
||||
# build already implies: ECW + MrSID in gdalinfo --formats, rasterio linked to
|
||||
# GDAL ${GDAL_VERSION}, and ECW + MrSID in rasterio.Env().drivers().
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
GDAL_VERSION="${GDAL_VERSION:-3.13.3}"
|
||||
GDAL_IMAGE="${GDAL_IMAGE:-dataforcanada/gdal-ecw-mrsid}"
|
||||
IMAGE="${IMAGE:-dataforcanada/gdal-ecw-mrsid-python}"
|
||||
TAG="${TAG:-${GDAL_VERSION}}"
|
||||
USER_UID="${USER_UID:-1000}"
|
||||
USER_GID="${USER_GID:-${USER_UID}}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PLATFORM=linux/amd64
|
||||
|
||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||
|
||||
docker image inspect "${GDAL_IMAGE}:${GDAL_VERSION}" >/dev/null 2>&1 \
|
||||
|| die "base image ${GDAL_IMAGE}:${GDAL_VERSION} not found locally — run docker/gdal/build.sh first"
|
||||
|
||||
echo ">>> Building ${IMAGE}:${TAG} FROM ${GDAL_IMAGE}:${GDAL_VERSION}"
|
||||
docker buildx build \
|
||||
--platform "${PLATFORM}" \
|
||||
--build-arg GDAL_VERSION="${GDAL_VERSION}" \
|
||||
--build-arg GDAL_IMAGE="${GDAL_IMAGE}" \
|
||||
--build-arg USER_UID="${USER_UID}" \
|
||||
--build-arg USER_GID="${USER_GID}" \
|
||||
--tag "${IMAGE}:${TAG}" \
|
||||
--load \
|
||||
"${SCRIPT_DIR}"
|
||||
|
||||
# Re-run the assertions against the finished image, as the runtime user.
|
||||
echo ">>> Verifying ${IMAGE}:${TAG}"
|
||||
docker run --rm --platform "${PLATFORM}" "${IMAGE}:${TAG}" verify-gdal-drivers --expect-gdal "${GDAL_VERSION}"
|
||||
echo ">>> OK: ${IMAGE}:${TAG}"
|
||||
@@ -0,0 +1,14 @@
|
||||
# Python stack for the stage-2 image (docker/python/Dockerfile).
|
||||
#
|
||||
# rasterio is *always* built from source here so it links against the image's
|
||||
# GDAL (with ECW + MrSID) instead of the libgdal bundled in its PyPI wheels.
|
||||
# The Dockerfile enforces that with `uv pip install --no-binary rasterio` and
|
||||
# /etc/uv/uv.toml makes later installs in the image do the same.
|
||||
#
|
||||
# Versions pinned 2026-09-14 (all support Python 3.14, the Ubuntu 26.04 system
|
||||
# interpreter this image uses). Bump deliberately, then rebuild + re-verify.
|
||||
numpy==2.5.3
|
||||
rasterio==1.5.1
|
||||
rioxarray==0.23.0
|
||||
xarray==2026.7.0
|
||||
dask[array]==2026.8.0
|
||||
@@ -0,0 +1,16 @@
|
||||
# System-wide uv config for the image, installed at /etc/uv/uv.toml.
|
||||
#
|
||||
# Packages whose PyPI wheels bundle their own libgdal. A bundled libgdal would
|
||||
# shadow the image's GDAL (built with ECW + MrSID) and silently lose both
|
||||
# drivers, so uv must always build these from source against `gdal-config`.
|
||||
#
|
||||
# Two keys because uv has two interfaces with different option names:
|
||||
# - [pip] no-binary -> `uv pip install ...`
|
||||
# - no-binary-package -> `uv sync` / `uv add` / `uv lock` / `uv run`
|
||||
# (UV_NO_BINARY_PACKAGE only affects the second one — verified on uv 0.12.13.)
|
||||
# A project's own uv.toml/pyproject.toml can override this; it is a default.
|
||||
|
||||
no-binary-package = ["rasterio", "fiona", "pyogrio", "gdal"]
|
||||
|
||||
[pip]
|
||||
no-binary = ["rasterio", "fiona", "pyogrio", "gdal"]
|
||||
@@ -0,0 +1,136 @@
|
||||
#!/opt/venv/bin/python
|
||||
"""Assert that GDAL + rasterio in this environment are the ones we built.
|
||||
|
||||
Every check is an assertion: the first failure exits non-zero with a message
|
||||
naming what was expected and what was found. Nothing is merely printed for a
|
||||
human to eyeball. Used as a `RUN` step in docker/python/Dockerfile (so the
|
||||
image build fails), as the dev container's postCreateCommand, and runnable by
|
||||
hand at any time: `verify-gdal-drivers [--expect-gdal X.Y.Z]`.
|
||||
|
||||
Checks
|
||||
1. `gdal-config --version` == expected GDAL version (the base image is the
|
||||
pinned one, and gdal-config is the one rasterio was compiled against).
|
||||
2. `gdalinfo --formats` lists ECW and MrSID.
|
||||
3. `rasterio.__gdal_version__` == expected GDAL version.
|
||||
4. `rasterio.Env().drivers()` includes ECW and MrSID.
|
||||
5. The libgdal mapped into this Python process is the system one, and there
|
||||
is exactly one — i.e. no wheel-bundled libgdal shadowing it.
|
||||
6. rioxarray can open a raster through that stack (in-memory GeoTIFF only;
|
||||
no ECW/MrSID files are touched).
|
||||
|
||||
The expected version comes from --expect-gdal, else $D4C_GDAL_VERSION (baked
|
||||
into the image from the GDAL_VERSION build arg), else `gdal-config --version`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
REQUIRED_DRIVERS = ("ECW", "MrSID")
|
||||
|
||||
|
||||
def fail(msg: str) -> None:
|
||||
print(f"FAIL: {msg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def ok(msg: str) -> None:
|
||||
print(f"ok: {msg}")
|
||||
|
||||
|
||||
def run(*cmd: str) -> str:
|
||||
try:
|
||||
return subprocess.check_output(cmd, text=True, stderr=subprocess.STDOUT).strip()
|
||||
except (OSError, subprocess.CalledProcessError) as exc:
|
||||
fail(f"{' '.join(cmd)}: {exc}")
|
||||
raise # unreachable; keeps type checkers happy
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
|
||||
parser.add_argument("--expect-gdal", metavar="X.Y.Z", help="expected GDAL version")
|
||||
args = parser.parse_args()
|
||||
|
||||
gdal_config = os.environ.get("GDAL_CONFIG", "gdal-config")
|
||||
config_version = run(gdal_config, "--version")
|
||||
expected = args.expect_gdal or os.environ.get("D4C_GDAL_VERSION") or config_version
|
||||
if not re.fullmatch(r"\d+\.\d+\.\d+", expected):
|
||||
fail(f"expected GDAL version {expected!r} is not of the form X.Y.Z")
|
||||
|
||||
# 1. gdal-config is the pinned GDAL
|
||||
if config_version != expected:
|
||||
fail(f"{gdal_config} --version is {config_version!r}, expected {expected!r}")
|
||||
ok(f"gdal-config --version == {expected}")
|
||||
|
||||
# 2. gdalinfo --formats lists both proprietary drivers
|
||||
formats = run("gdalinfo", "--formats")
|
||||
for drv in REQUIRED_DRIVERS:
|
||||
if not re.search(rf"^\s+{re.escape(drv)} -raster-", formats, re.MULTILINE):
|
||||
fail(f"{drv} missing from 'gdalinfo --formats'")
|
||||
ok("gdalinfo --formats lists ECW and MrSID")
|
||||
|
||||
# 3. rasterio links against the pinned GDAL
|
||||
try:
|
||||
import rasterio
|
||||
except ImportError as exc:
|
||||
fail(f"cannot import rasterio: {exc}")
|
||||
if rasterio.__gdal_version__ != expected:
|
||||
fail(
|
||||
f"rasterio.__gdal_version__ is {rasterio.__gdal_version__!r}, expected "
|
||||
f"{expected!r} — rasterio is not using the image's GDAL "
|
||||
"(was a PyPI wheel with bundled libgdal installed?)"
|
||||
)
|
||||
ok(f"rasterio {rasterio.__version__} reports GDAL {rasterio.__gdal_version__}")
|
||||
|
||||
# 4. rasterio sees the drivers (drivers() needs an *entered* Env)
|
||||
with rasterio.Env() as env:
|
||||
drivers = env.drivers()
|
||||
for drv in REQUIRED_DRIVERS:
|
||||
if drv not in drivers:
|
||||
fail(f"{drv} missing from rasterio.Env().drivers() ({len(drivers)} drivers registered)")
|
||||
ok("rasterio.Env().drivers() includes ECW and MrSID")
|
||||
|
||||
# 5. exactly one libgdal in this process, and it is the system one
|
||||
with open("/proc/self/maps") as maps:
|
||||
libgdal = sorted(
|
||||
{line.split()[-1] for line in maps if "/libgdal" in line and line.split()[-1].startswith("/")}
|
||||
)
|
||||
if len(libgdal) != 1:
|
||||
fail(f"expected exactly one libgdal mapped into the process, found {libgdal}")
|
||||
if not libgdal[0].startswith("/usr/lib/") or "site-packages" in libgdal[0]:
|
||||
fail(f"libgdal is loaded from {libgdal[0]}, not from the system GDAL under /usr/lib/")
|
||||
ok(f"single system libgdal in process: {libgdal[0]}")
|
||||
|
||||
# 6. rioxarray works end-to-end on an in-memory GeoTIFF
|
||||
try:
|
||||
import numpy as np
|
||||
import rioxarray
|
||||
from rasterio.transform import from_origin
|
||||
except ImportError as exc:
|
||||
fail(f"cannot import the raster stack: {exc}")
|
||||
path = "/vsimem/verify.tif"
|
||||
with rasterio.Env():
|
||||
data = np.arange(16, dtype="uint8").reshape(1, 4, 4)
|
||||
with rasterio.open(
|
||||
path, "w", driver="GTiff", width=4, height=4, count=1, dtype="uint8",
|
||||
crs="EPSG:3857", transform=from_origin(0, 4, 1, 1),
|
||||
) as dst:
|
||||
dst.write(data)
|
||||
# Context manager: an open dataset left to the garbage collector gets
|
||||
# finalized during interpreter teardown and prints a spurious
|
||||
# "Error in sys.excepthook" at exit.
|
||||
with rioxarray.open_rasterio(path) as src:
|
||||
da = src.load()
|
||||
if da.shape != (1, 4, 4) or int(da.sum()) != int(data.sum()) or da.rio.crs.to_epsg() != 3857:
|
||||
fail(f"rioxarray round-trip mismatch: shape={da.shape} crs={da.rio.crs}")
|
||||
ok(f"rioxarray {rioxarray.__version__} round-trips an in-memory raster")
|
||||
|
||||
print(f"OK: GDAL {expected} with ECW + MrSID, rasterio {rasterio.__version__} linked against it")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user