mirror of
https://github.com/dataforcanada/d4c-datapkg-orthoimagery.git
synced 2026-10-02 04:05:43 +02:00
Changes to Docker image
This commit is contained in:
@@ -0,0 +1,30 @@
|
|||||||
|
# Build context for docker/python/Dockerfile (stage 2), whose context is the
|
||||||
|
# repo root because that is where pyproject.toml and uv.lock live.
|
||||||
|
#
|
||||||
|
# scripts/ and data/ hold hundreds of GB of orthoimagery. Without this file
|
||||||
|
# every build would try to ship all of it to the daemon, so: exclude everything,
|
||||||
|
# then re-admit only the four files the Dockerfile actually COPYs.
|
||||||
|
#
|
||||||
|
# `*` matches top-level entries only, but a path whose *parent* matches is
|
||||||
|
# excluded too, so data/, scripts/ and .git/ are excluded whole. The `!` lines
|
||||||
|
# re-admit paths under the (excluded) docker/ directory; BuildKit resolves those
|
||||||
|
# by descending only where an exception could match, so the giant trees below
|
||||||
|
# are never walked.
|
||||||
|
#
|
||||||
|
# Stage 1 (docker/gdal/build.sh) is unaffected: its contexts are docker/gdal and
|
||||||
|
# the GDAL source checkout, never the repo root.
|
||||||
|
|
||||||
|
*
|
||||||
|
|
||||||
|
!pyproject.toml
|
||||||
|
!uv.lock
|
||||||
|
!docker/python/uv.toml
|
||||||
|
!docker/python/verify_gdal_drivers.py
|
||||||
|
|
||||||
|
# Belt-and-braces. Already covered by `*` above, but stated explicitly and last
|
||||||
|
# (last match wins) so these can never be dragged in by a future edit that
|
||||||
|
# loosens the blanket exclusion.
|
||||||
|
data
|
||||||
|
scripts
|
||||||
|
.git
|
||||||
|
**/__pycache__
|
||||||
+60
-23
@@ -16,7 +16,7 @@ back here instead of a cryptic "pull access denied".
|
|||||||
| Stage | Image | Contents |
|
| Stage | Image | Contents |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| 1 | `dataforcanada/gdal-ecw-mrsid:3.13.3` | OSGeo's `ubuntu-full` GDAL build (Ubuntu 26.04, PROJ 9.8.1, all PROJ grids) plus the **ECW** (libecwj2 3.3) and **MrSID** (DSDK 9.5.5) drivers. Built from GDAL's own `docker/ubuntu-full/Dockerfile`, unmodified. |
|
| 1 | `dataforcanada/gdal-ecw-mrsid:3.13.3` | OSGeo's `ubuntu-full` GDAL build (Ubuntu 26.04, PROJ 9.8.1, all PROJ grids) plus the **ECW** (libecwj2 3.3) and **MrSID** (DSDK 9.5.5) drivers. Built from GDAL's own `docker/ubuntu-full/Dockerfile`, unmodified. |
|
||||||
| 2 | `dataforcanada/gdal-ecw-mrsid-python:3.13.3` | `FROM` stage 1. `uv`-managed venv at `/opt/venv` (system Python 3.14) with `rioxarray`, `rasterio` (compiled against the image's GDAL), `xarray`, `dask[array]`, `numpy`. Non-root user `d4c`. This is what the dev container runs. |
|
| 2 | `dataforcanada/gdal-ecw-mrsid-python:3.13.3` | `FROM` stage 1. `uv`-managed venv at `/opt/venv` (system Python 3.14) holding the dependency set from the repo's [`pyproject.toml`](../pyproject.toml) + [`uv.lock`](../uv.lock): `rioxarray`, `rasterio` and `pyogrio` (both compiled against the image's GDAL), `xarray`, `dask[array]`, `numpy`, `portolan-cli`. Non-root user `d4c`. This is what the dev container runs. |
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
@@ -147,28 +147,43 @@ and then re-runs the assertions against the finished image as the runtime
|
|||||||
user. Environment knobs: `GDAL_VERSION`, `GDAL_IMAGE`, `IMAGE`, `TAG`,
|
user. Environment knobs: `GDAL_VERSION`, `GDAL_IMAGE`, `IMAGE`, `TAG`,
|
||||||
`USER_UID`/`USER_GID` (default 1000/1000).
|
`USER_UID`/`USER_GID` (default 1000/1000).
|
||||||
|
|
||||||
|
The **build context is the repo root**, not `docker/python/`, because
|
||||||
|
`pyproject.toml` and `uv.lock` live there and the image is built from them.
|
||||||
|
[`.dockerignore`](../.dockerignore) excludes everything and re-admits only the
|
||||||
|
four files the Dockerfile copies, so the context stays a few KB instead of the
|
||||||
|
hundreds of GB under `data/` and `scripts/`. (Stage 1 is unaffected — its
|
||||||
|
contexts are `docker/gdal` and the GDAL source checkout.)
|
||||||
|
|
||||||
Key points, all enforced inside the Dockerfile so the image cannot be built
|
Key points, all enforced inside the Dockerfile so the image cannot be built
|
||||||
with them violated:
|
with them violated:
|
||||||
|
|
||||||
- **rasterio is compiled from source** with `uv pip install --no-binary rasterio`.
|
- **rasterio and pyogrio are compiled from source**, via
|
||||||
rasterio's PyPI wheels bundle their own libgdal (the 1.5.1 wheel ships GDAL
|
`uv sync --no-binary-package rasterio --no-binary-package pyogrio`. Their
|
||||||
|
PyPI wheels bundle their own libgdal (the rasterio 1.5.1 wheel ships GDAL
|
||||||
3.12.4, without ECW/MrSID) that would shadow the image's GDAL; the sdist
|
3.12.4, without ECW/MrSID) that would shadow the image's GDAL; the sdist
|
||||||
build links against it via `gdal-config` instead. Flag spelling matters:
|
builds link against it via `gdal-config` instead. pyogrio matters now because
|
||||||
`uv pip install` takes pip-style `--no-binary <pkg>`, while
|
`portolan-cli` depends on it. This is stated in three places, deliberately:
|
||||||
`--no-binary-package <pkg>` belongs to `uv sync`/`uv add`. The image also
|
`[tool.uv] no-binary-package` in [`../pyproject.toml`](../pyproject.toml) (so
|
||||||
installs [`python/uv.toml`](python/uv.toml) as `/etc/uv/uv.toml`, listing
|
`uv sync` is correct on its own terms), `/etc/uv/uv.toml` from
|
||||||
`rasterio fiona pyogrio gdal` for *both* interfaces, so anything you install
|
[`python/uv.toml`](python/uv.toml) (so ad-hoc installs in the container are
|
||||||
later inside the container is forced from source too (fiona/pyogrio wheels
|
forced from source too), and the explicit flags in the Dockerfile. Flag
|
||||||
bundle GDAL as well). Note `UV_NO_BINARY_PACKAGE` is not used: it only
|
spelling matters: `uv pip install` takes pip-style `--no-binary <pkg>`, while
|
||||||
affects `uv sync`/`uv add`, not `uv pip install` (verified on uv 0.12.13).
|
`--no-binary-package <pkg>` belongs to `uv sync`/`uv add`. Note
|
||||||
|
`UV_NO_BINARY_PACKAGE` is not used: it only affects `uv sync`/`uv add`, not
|
||||||
|
`uv pip install` (verified on uv 0.12.13).
|
||||||
- **No `libgdal-dev` from apt.** The base image already ships `gdal-config`,
|
- **No `libgdal-dev` from apt.** The base image already ships `gdal-config`,
|
||||||
the headers and `libgdal.so` for the pinned GDAL; Ubuntu's `libgdal-dev`
|
the headers and `libgdal.so` for the pinned GDAL; Ubuntu's `libgdal-dev`
|
||||||
would add a second, driver-less GDAL next to it — exactly the shadowing this
|
would add a second, driver-less GDAL next to it — exactly the shadowing this
|
||||||
image exists to prevent (GDAL's own `docker/README.md` warns against it).
|
image exists to prevent (GDAL's own `docker/README.md` warns against it).
|
||||||
Only `build-essential` and `python3-dev` are added, and the build refuses to
|
Only `build-essential` and `python3-dev` are added, and the build refuses to
|
||||||
continue if any `libgdal*` apt package is installed.
|
continue if any `libgdal*` apt package is installed.
|
||||||
- Packages are pinned in [`python/requirements.txt`](python/requirements.txt);
|
- Packages come from [`../pyproject.toml`](../pyproject.toml) and
|
||||||
the venv uses the system Python 3.14 (`UV_PYTHON_DOWNLOADS=never`).
|
[`../uv.lock`](../uv.lock), installed with `uv sync --locked` — the build
|
||||||
|
fails if the two have drifted apart, so the image can never be built from an
|
||||||
|
unlocked dependency set. The lock pins the whole transitive tree (119
|
||||||
|
packages), not just the six direct ones. The venv uses the system Python 3.14
|
||||||
|
(`UV_PYTHON_DOWNLOADS=never`), and `UV_PROJECT_ENVIRONMENT=/opt/venv` makes
|
||||||
|
uv's project commands target it rather than creating a `./.venv`.
|
||||||
- The last build step runs `verify-gdal-drivers`
|
- The last build step runs `verify-gdal-drivers`
|
||||||
([`python/verify_gdal_drivers.py`](python/verify_gdal_drivers.py)), which
|
([`python/verify_gdal_drivers.py`](python/verify_gdal_drivers.py)), which
|
||||||
asserts:
|
asserts:
|
||||||
@@ -176,9 +191,11 @@ with them violated:
|
|||||||
2. `gdalinfo --formats` lists ECW and MrSID;
|
2. `gdalinfo --formats` lists ECW and MrSID;
|
||||||
3. `rasterio.__gdal_version__` == the pinned GDAL;
|
3. `rasterio.__gdal_version__` == the pinned GDAL;
|
||||||
4. `rasterio.Env().drivers()` includes ECW and MrSID;
|
4. `rasterio.Env().drivers()` includes ECW and MrSID;
|
||||||
5. exactly one `libgdal` is mapped into the Python process and it is the
|
5. `pyogrio.__gdal_version_string__` == the pinned GDAL;
|
||||||
system one (no wheel-bundled copy);
|
6. exactly one `libgdal` is mapped into the Python process and it is the
|
||||||
6. rioxarray round-trips an in-memory GeoTIFF.
|
system one (no wheel-bundled copy) — checked after both rasterio and
|
||||||
|
pyogrio are imported, so it covers either of them bundling a copy;
|
||||||
|
7. rioxarray round-trips an in-memory GeoTIFF.
|
||||||
|
|
||||||
`verify-gdal-drivers` is on `PATH` in the image; run it any time (the dev
|
`verify-gdal-drivers` is on `PATH` in the image; run it any time (the dev
|
||||||
container runs it as `postCreateCommand`).
|
container runs it as `postCreateCommand`).
|
||||||
@@ -188,9 +205,24 @@ with them violated:
|
|||||||
`.devcontainer/devcontainer.json` points at `dataforcanada/gdal-ecw-mrsid-python:3.13.3`
|
`.devcontainer/devcontainer.json` points at `dataforcanada/gdal-ecw-mrsid-python:3.13.3`
|
||||||
directly — no build on the VS Code side. `remoteUser` is `d4c`; VS Code remaps
|
directly — no build on the VS Code side. `remoteUser` is `d4c`; VS Code remaps
|
||||||
its UID/GID to yours on first start so files in the bind-mounted workspace keep
|
its UID/GID to yours on first start so files in the bind-mounted workspace keep
|
||||||
sane ownership. The venv at `/opt/venv` is owned by that user, so
|
sane ownership. The venv at `/opt/venv` is owned by that user, so installing
|
||||||
`uv pip install <pkg>` works inside the container without root (and honours
|
works inside the container without root (and honours the no-binary rule above).
|
||||||
the no-binary rule above).
|
|
||||||
|
Because the image sets `UV_PROJECT_ENVIRONMENT=/opt/venv`, uv's project commands
|
||||||
|
run from the workspace root operate on that venv directly — no `.venv` is
|
||||||
|
created and nothing needs activating:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv add portolan-cli # edits pyproject.toml + uv.lock, installs to /opt/venv
|
||||||
|
uv remove dask
|
||||||
|
uv sync # bring /opt/venv back in line with uv.lock
|
||||||
|
uv lock --upgrade-package rasterio
|
||||||
|
```
|
||||||
|
|
||||||
|
Commit the resulting `pyproject.toml` and `uv.lock`. They are what the next
|
||||||
|
image build installs, so a dependency added this way survives a rebuild — but
|
||||||
|
only after `docker/python/build.sh` is re-run. `uv pip install <pkg>` still
|
||||||
|
works for a throwaway package you do not want recorded.
|
||||||
|
|
||||||
Headless check with the Dev Containers CLI:
|
Headless check with the Dev Containers CLI:
|
||||||
|
|
||||||
@@ -217,10 +249,15 @@ npx --yes @devcontainers/cli@latest exec --workspace-folder . verify-gdal-driver
|
|||||||
-DMRSID_ROOT=/opt/Raster_DSDK` on the `GDAL_CMAKE_EXTRA_OPTS` echo line
|
-DMRSID_ROOT=/opt/Raster_DSDK` on the `GDAL_CMAKE_EXTRA_OPTS` echo line
|
||||||
and `Found ECW` / `Found MRSID` from CMake.
|
and `Found ECW` / `Found MRSID` from CMake.
|
||||||
- **`rasterio.__gdal_version__` mismatch in stage 2** (e.g. it reports 3.12.4)
|
- **`rasterio.__gdal_version__` mismatch in stage 2** (e.g. it reports 3.12.4)
|
||||||
— a rasterio wheel got installed. Check that `--no-binary rasterio` and
|
— a rasterio wheel got installed. Check that the `--no-binary-package` flags,
|
||||||
`/etc/uv/uv.toml` are in effect (`uv pip install --dry-run -v rasterio`
|
`[tool.uv] no-binary-package` in `pyproject.toml` and `/etc/uv/uv.toml` are in
|
||||||
should log `Selecting: rasterio==… (rasterio-….tar.gz)`) and that no
|
effect: `uv sync -v` should log `Built rasterio==…` (not a wheel download),
|
||||||
`libgdal*` apt package is present.
|
and `uv pip install --dry-run -v rasterio` should log
|
||||||
|
`Selecting: rasterio==… (rasterio-….tar.gz)`. Also confirm no `libgdal*` apt
|
||||||
|
package is present. The same applies to `pyogrio.__gdal_version__`.
|
||||||
|
- **`the lockfile is not up-to-date with pyproject.toml`** during stage 2 —
|
||||||
|
someone edited `pyproject.toml` without re-locking. Run `uv lock` at the repo
|
||||||
|
root, commit the result, and rebuild.
|
||||||
- **"this image is linux/amd64 only"** — you are building on a non-x86_64
|
- **"this image is linux/amd64 only"** — you are building on a non-x86_64
|
||||||
daemon; see Prerequisites.
|
daemon; see Prerequisites.
|
||||||
- **Stale grids / wrong PROJ** — see the PROJ notes under stage 1.
|
- **Stale grids / wrong PROJ** — see the PROJ notes under stage 1.
|
||||||
|
|||||||
+36
-14
@@ -1,18 +1,26 @@
|
|||||||
# syntax=docker/dockerfile:1
|
# syntax=docker/dockerfile:1
|
||||||
#
|
#
|
||||||
# Stage 2: Python environment (uv-managed venv with rioxarray/rasterio/xarray/
|
# Stage 2: Python environment (uv-managed venv holding the repo's
|
||||||
# dask/numpy) on top of the stage-1 GDAL + ECW + MrSID image.
|
# pyproject.toml + uv.lock dependency set — rioxarray, rasterio, xarray, dask,
|
||||||
|
# numpy, portolan-cli) on top of the stage-1 GDAL + ECW + MrSID image.
|
||||||
#
|
#
|
||||||
# To bump GDAL, change GDAL_VERSION (the base image tag) and rebuild. Nothing
|
# To bump GDAL, change GDAL_VERSION (the base image tag) and rebuild. Nothing
|
||||||
# else here encodes the GDAL version: rasterio is compiled against whatever
|
# else here encodes the GDAL version: rasterio is compiled against whatever
|
||||||
# `gdal-config` the base image ships, and the build asserts that it matches.
|
# `gdal-config` the base image ships, and the build asserts that it matches.
|
||||||
|
# To change Python packages, edit ../../pyproject.toml (via `uv add`) and commit
|
||||||
|
# the regenerated ../../uv.lock; `uv sync --locked` below will pick them up.
|
||||||
|
#
|
||||||
|
# The build context is the REPO ROOT, not this directory, because pyproject.toml
|
||||||
|
# and uv.lock live there. .dockerignore keeps that context tiny — data/ and
|
||||||
|
# scripts/ hold hundreds of GB of imagery and are excluded.
|
||||||
#
|
#
|
||||||
# Build (from the repo root):
|
# Build (from the repo root):
|
||||||
# docker/python/build.sh
|
# docker/python/build.sh
|
||||||
# # or by hand:
|
# # or by hand:
|
||||||
# docker buildx build --platform linux/amd64 \
|
# docker buildx build --platform linux/amd64 \
|
||||||
# --build-arg GDAL_VERSION=3.13.3 \
|
# --build-arg GDAL_VERSION=3.13.3 \
|
||||||
# -t dataforcanada/gdal-ecw-mrsid-python:3.13.3 docker/python
|
# --file docker/python/Dockerfile \
|
||||||
|
# -t dataforcanada/gdal-ecw-mrsid-python:3.13.3 .
|
||||||
|
|
||||||
ARG GDAL_VERSION=3.13.3
|
ARG GDAL_VERSION=3.13.3
|
||||||
ARG GDAL_IMAGE=dataforcanada/gdal-ecw-mrsid
|
ARG GDAL_IMAGE=dataforcanada/gdal-ecw-mrsid
|
||||||
@@ -62,7 +70,7 @@ RUN test "$(gdal-config --version)" = "${GDAL_VERSION}" \
|
|||||||
|
|
||||||
COPY --from=uv /uv /uvx /usr/local/bin/
|
COPY --from=uv /uv /uvx /usr/local/bin/
|
||||||
# Image-wide default: never install wheels that bundle libgdal (see file).
|
# Image-wide default: never install wheels that bundle libgdal (see file).
|
||||||
COPY uv.toml /etc/uv/uv.toml
|
COPY docker/python/uv.toml /etc/uv/uv.toml
|
||||||
|
|
||||||
# ---- non-root runtime user ---------------------------------------------------
|
# ---- non-root runtime user ---------------------------------------------------
|
||||||
# Ubuntu 26.04 ships a stock "ubuntu" user at uid 1000; drop it so USER_UID can
|
# Ubuntu 26.04 ships a stock "ubuntu" user at uid 1000; drop it so USER_UID can
|
||||||
@@ -77,18 +85,24 @@ RUN if id ubuntu >/dev/null 2>&1; then userdel -r ubuntu; fi \
|
|||||||
&& chown "${USER_UID}:${USER_GID}" /opt/venv \
|
&& chown "${USER_UID}:${USER_GID}" /opt/venv \
|
||||||
&& echo "source /usr/share/bash-completion/bash_completion" >> "/home/${USERNAME}/.bashrc"
|
&& echo "source /usr/share/bash-completion/bash_completion" >> "/home/${USERNAME}/.bashrc"
|
||||||
|
|
||||||
COPY --chmod=0755 verify_gdal_drivers.py /usr/local/bin/verify-gdal-drivers
|
COPY --chmod=0755 docker/python/verify_gdal_drivers.py /usr/local/bin/verify-gdal-drivers
|
||||||
|
|
||||||
# VIRTUAL_ENV/PATH: the venv is the default python for everything downstream.
|
# VIRTUAL_ENV/PATH: the venv is the default python for everything downstream.
|
||||||
|
# UV_PROJECT_ENVIRONMENT: point uv's *project* interface at that same venv, so
|
||||||
|
# `uv sync`/`uv add`/`uv run` in the repo update /opt/venv instead of creating
|
||||||
|
# a ./.venv next to pyproject.toml — both in this build and, more importantly,
|
||||||
|
# in the dev container, where the workspace is bind-mounted.
|
||||||
# UV_PYTHON_DOWNLOADS=never: always use the system interpreter, never a
|
# UV_PYTHON_DOWNLOADS=never: always use the system interpreter, never a
|
||||||
# uv-managed one.
|
# uv-managed one.
|
||||||
# GDAL_CONFIG: what rasterio's setup.py consults; explicit rather than PATH luck.
|
# GDAL_CONFIG: what rasterio's setup.py consults; explicit rather than PATH luck.
|
||||||
# D4C_GDAL_VERSION: what verify-gdal-drivers checks against at runtime.
|
# D4C_GDAL_VERSION: what verify-gdal-drivers checks against at runtime.
|
||||||
# (Not named GDAL_VERSION: rasterio's setup.py treats that as an override.)
|
# (Not named GDAL_VERSION: rasterio's setup.py treats that as an override.)
|
||||||
# (No UV_NO_BINARY_PACKAGE here on purpose: it does not apply to
|
# (No UV_NO_BINARY_PACKAGE here on purpose: it does not apply to
|
||||||
# `uv pip install`; /etc/uv/uv.toml covers both interfaces.)
|
# `uv pip install`. /etc/uv/uv.toml covers both interfaces image-wide, and
|
||||||
|
# pyproject.toml's [tool.uv] covers the project interface for this repo.)
|
||||||
ENV VIRTUAL_ENV=/opt/venv \
|
ENV VIRTUAL_ENV=/opt/venv \
|
||||||
PATH=/opt/venv/bin:${PATH} \
|
PATH=/opt/venv/bin:${PATH} \
|
||||||
|
UV_PROJECT_ENVIRONMENT=/opt/venv \
|
||||||
UV_PYTHON_DOWNLOADS=never \
|
UV_PYTHON_DOWNLOADS=never \
|
||||||
UV_LINK_MODE=copy \
|
UV_LINK_MODE=copy \
|
||||||
UV_COMPILE_BYTECODE=1 \
|
UV_COMPILE_BYTECODE=1 \
|
||||||
@@ -99,21 +113,29 @@ USER ${USERNAME}
|
|||||||
WORKDIR /home/${USERNAME}
|
WORKDIR /home/${USERNAME}
|
||||||
|
|
||||||
# ---- venv + packages ---------------------------------------------------------
|
# ---- venv + packages ---------------------------------------------------------
|
||||||
# --no-binary rasterio is the critical bit: rasterio's PyPI wheels bundle their
|
# The repo's pyproject.toml + uv.lock are the single source of truth for this
|
||||||
# own libgdal (built without ECW/MrSID). Building the sdist makes it link
|
# environment; `--locked` fails the build if they have drifted apart, so the
|
||||||
# against this image's libgdal via gdal-config. (`uv pip install` takes the
|
# image can never be built from an unlocked (i.e. unreproducible) dependency
|
||||||
# pip-style `--no-binary <pkg>`; `--no-binary-package` is the `uv sync`/`uv add`
|
# set. The lock pins the whole transitive tree, not just the direct deps.
|
||||||
# spelling. /etc/uv/uv.toml already says the same; the flag is belt-and-braces.)
|
#
|
||||||
COPY --chown=${USER_UID}:${USER_GID} requirements.txt /tmp/requirements.txt
|
# --no-binary-package is the critical bit: the PyPI wheels for rasterio and
|
||||||
|
# pyogrio (the latter pulled in by portolan-cli) bundle their own libgdal, built
|
||||||
|
# without ECW/MrSID. Building the sdists makes them link against this image's
|
||||||
|
# libgdal via gdal-config. Both pyproject.toml's [tool.uv] and /etc/uv/uv.toml
|
||||||
|
# already say this; the flags are belt-and-braces. (`uv sync`/`uv add` take
|
||||||
|
# `--no-binary-package <pkg>`; `uv pip install` takes pip-style `--no-binary`.)
|
||||||
|
COPY --chown=${USER_UID}:${USER_GID} pyproject.toml uv.lock /tmp/project/
|
||||||
RUN --mount=type=cache,target=/home/${USERNAME}/.cache/uv,uid=${USER_UID},gid=${USER_GID} \
|
RUN --mount=type=cache,target=/home/${USERNAME}/.cache/uv,uid=${USER_UID},gid=${USER_GID} \
|
||||||
uv venv --python /usr/bin/python3 "${VIRTUAL_ENV}" \
|
uv venv --python /usr/bin/python3 "${VIRTUAL_ENV}" \
|
||||||
&& uv pip install --no-binary rasterio --requirements /tmp/requirements.txt \
|
&& uv sync --project /tmp/project --locked --no-dev \
|
||||||
&& rm /tmp/requirements.txt
|
--no-binary-package rasterio --no-binary-package pyogrio \
|
||||||
|
&& rm -rf /tmp/project
|
||||||
|
|
||||||
# ---- verification: the build fails unless all of these hold ------------------
|
# ---- verification: the build fails unless all of these hold ------------------
|
||||||
# gdalinfo --formats lists ECW and MrSID
|
# gdalinfo --formats lists ECW and MrSID
|
||||||
# rasterio.__gdal_version__ == GDAL_VERSION
|
# rasterio.__gdal_version__ == GDAL_VERSION
|
||||||
# rasterio.Env().drivers() includes ECW and MrSID
|
# rasterio.Env().drivers() includes ECW and MrSID
|
||||||
|
# pyogrio.__gdal_version_string__ == GDAL_VERSION
|
||||||
# (+ exactly one libgdal, the system one, is loaded; rioxarray round-trips)
|
# (+ exactly one libgdal, the system one, is loaded; rioxarray round-trips)
|
||||||
RUN verify-gdal-drivers --expect-gdal "${GDAL_VERSION}"
|
RUN verify-gdal-drivers --expect-gdal "${GDAL_VERSION}"
|
||||||
|
|
||||||
|
|||||||
+16
-3
@@ -1,9 +1,14 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
#
|
#
|
||||||
# Stage 2: Python (uv venv with rioxarray/rasterio/xarray/dask/numpy) on top of
|
# Stage 2: Python (uv venv holding the repo's pyproject.toml + uv.lock set:
|
||||||
# the stage-1 GDAL + ECW + MrSID image. Requires stage 1 to exist locally:
|
# rioxarray, rasterio, xarray, dask, numpy, portolan-cli) on top of the stage-1
|
||||||
|
# GDAL + ECW + MrSID image. Requires stage 1 to exist locally:
|
||||||
# docker/gdal/build.sh
|
# docker/gdal/build.sh
|
||||||
#
|
#
|
||||||
|
# The build context is the repo root (that is where pyproject.toml and uv.lock
|
||||||
|
# live); .dockerignore keeps it to a handful of files rather than the hundreds
|
||||||
|
# of GB of imagery under data/ and scripts/.
|
||||||
|
#
|
||||||
# Result: ${IMAGE}:${TAG} (default dataforcanada/gdal-ecw-mrsid-python:3.13.3)
|
# Result: ${IMAGE}:${TAG} (default dataforcanada/gdal-ecw-mrsid-python:3.13.3)
|
||||||
#
|
#
|
||||||
# Usage:
|
# Usage:
|
||||||
@@ -24,6 +29,7 @@ TAG="${TAG:-${GDAL_VERSION}}"
|
|||||||
USER_UID="${USER_UID:-1000}"
|
USER_UID="${USER_UID:-1000}"
|
||||||
USER_GID="${USER_GID:-${USER_UID}}"
|
USER_GID="${USER_GID:-${USER_UID}}"
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||||
PLATFORM=linux/amd64
|
PLATFORM=linux/amd64
|
||||||
|
|
||||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||||
@@ -31,6 +37,12 @@ die() { echo "ERROR: $*" >&2; exit 1; }
|
|||||||
docker image inspect "${GDAL_IMAGE}:${GDAL_VERSION}" >/dev/null 2>&1 \
|
docker image inspect "${GDAL_IMAGE}:${GDAL_VERSION}" >/dev/null 2>&1 \
|
||||||
|| die "base image ${GDAL_IMAGE}:${GDAL_VERSION} not found locally — run docker/gdal/build.sh first"
|
|| die "base image ${GDAL_IMAGE}:${GDAL_VERSION} not found locally — run docker/gdal/build.sh first"
|
||||||
|
|
||||||
|
# The image is built straight from these two; `uv sync --locked` in the
|
||||||
|
# Dockerfile refuses to build if they have drifted apart.
|
||||||
|
for f in pyproject.toml uv.lock; do
|
||||||
|
[ -f "${REPO_ROOT}/${f}" ] || die "${REPO_ROOT}/${f} not found — the image is built from it"
|
||||||
|
done
|
||||||
|
|
||||||
echo ">>> Building ${IMAGE}:${TAG} FROM ${GDAL_IMAGE}:${GDAL_VERSION}"
|
echo ">>> Building ${IMAGE}:${TAG} FROM ${GDAL_IMAGE}:${GDAL_VERSION}"
|
||||||
docker buildx build \
|
docker buildx build \
|
||||||
--platform "${PLATFORM}" \
|
--platform "${PLATFORM}" \
|
||||||
@@ -39,8 +51,9 @@ docker buildx build \
|
|||||||
--build-arg USER_UID="${USER_UID}" \
|
--build-arg USER_UID="${USER_UID}" \
|
||||||
--build-arg USER_GID="${USER_GID}" \
|
--build-arg USER_GID="${USER_GID}" \
|
||||||
--tag "${IMAGE}:${TAG}" \
|
--tag "${IMAGE}:${TAG}" \
|
||||||
|
--file "${SCRIPT_DIR}/Dockerfile" \
|
||||||
--load \
|
--load \
|
||||||
"${SCRIPT_DIR}"
|
"${REPO_ROOT}"
|
||||||
|
|
||||||
# Re-run the assertions against the finished image, as the runtime user.
|
# Re-run the assertions against the finished image, as the runtime user.
|
||||||
echo ">>> Verifying ${IMAGE}:${TAG}"
|
echo ">>> Verifying ${IMAGE}:${TAG}"
|
||||||
|
|||||||
@@ -13,9 +13,13 @@ Checks
|
|||||||
2. `gdalinfo --formats` lists ECW and MrSID.
|
2. `gdalinfo --formats` lists ECW and MrSID.
|
||||||
3. `rasterio.__gdal_version__` == expected GDAL version.
|
3. `rasterio.__gdal_version__` == expected GDAL version.
|
||||||
4. `rasterio.Env().drivers()` includes ECW and MrSID.
|
4. `rasterio.Env().drivers()` includes ECW and MrSID.
|
||||||
5. The libgdal mapped into this Python process is the system one, and there
|
5. `pyogrio.__gdal_version_string__` == expected GDAL version. pyogrio is the
|
||||||
is exactly one — i.e. no wheel-bundled libgdal shadowing it.
|
other GDAL-linked extension in the image (pulled in by portolan-cli) and
|
||||||
6. rioxarray can open a raster through that stack (in-memory GeoTIFF only;
|
its wheels bundle libgdal too.
|
||||||
|
6. The libgdal mapped into this Python process is the system one, and there
|
||||||
|
is exactly one — i.e. no wheel-bundled libgdal shadowing it. This runs
|
||||||
|
after both extensions are imported, so it covers rasterio and pyogrio.
|
||||||
|
7. rioxarray can open a raster through that stack (in-memory GeoTIFF only;
|
||||||
no ECW/MrSID files are touched).
|
no ECW/MrSID files are touched).
|
||||||
|
|
||||||
The expected version comes from --expect-gdal, else $D4C_GDAL_VERSION (baked
|
The expected version comes from --expect-gdal, else $D4C_GDAL_VERSION (baked
|
||||||
@@ -94,7 +98,21 @@ def main() -> None:
|
|||||||
fail(f"{drv} missing from rasterio.Env().drivers() ({len(drivers)} drivers registered)")
|
fail(f"{drv} missing from rasterio.Env().drivers() ({len(drivers)} drivers registered)")
|
||||||
ok("rasterio.Env().drivers() includes ECW and MrSID")
|
ok("rasterio.Env().drivers() includes ECW and MrSID")
|
||||||
|
|
||||||
# 5. exactly one libgdal in this process, and it is the system one
|
# 5. pyogrio links against the pinned GDAL too. Imported here, before the
|
||||||
|
# libgdal count below, so a wheel-bundled libgdal of its own is caught.
|
||||||
|
try:
|
||||||
|
import pyogrio
|
||||||
|
except ImportError as exc:
|
||||||
|
fail(f"cannot import pyogrio: {exc}")
|
||||||
|
if pyogrio.__gdal_version_string__ != expected:
|
||||||
|
fail(
|
||||||
|
f"pyogrio.__gdal_version_string__ is {pyogrio.__gdal_version_string__!r}, "
|
||||||
|
f"expected {expected!r} — pyogrio is not using the image's GDAL "
|
||||||
|
"(was a PyPI wheel with bundled libgdal installed?)"
|
||||||
|
)
|
||||||
|
ok(f"pyogrio {pyogrio.__version__} reports GDAL {pyogrio.__gdal_version_string__}")
|
||||||
|
|
||||||
|
# 6. exactly one libgdal in this process, and it is the system one
|
||||||
with open("/proc/self/maps") as maps:
|
with open("/proc/self/maps") as maps:
|
||||||
libgdal = sorted(
|
libgdal = sorted(
|
||||||
{line.split()[-1] for line in maps if "/libgdal" in line and line.split()[-1].startswith("/")}
|
{line.split()[-1] for line in maps if "/libgdal" in line and line.split()[-1].startswith("/")}
|
||||||
@@ -105,7 +123,7 @@ def main() -> None:
|
|||||||
fail(f"libgdal is loaded from {libgdal[0]}, not from the system GDAL under /usr/lib/")
|
fail(f"libgdal is loaded from {libgdal[0]}, not from the system GDAL under /usr/lib/")
|
||||||
ok(f"single system libgdal in process: {libgdal[0]}")
|
ok(f"single system libgdal in process: {libgdal[0]}")
|
||||||
|
|
||||||
# 6. rioxarray works end-to-end on an in-memory GeoTIFF
|
# 7. rioxarray works end-to-end on an in-memory GeoTIFF
|
||||||
try:
|
try:
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import rioxarray
|
import rioxarray
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
# Python environment for this repo, and for the dev-container image that runs it
|
||||||
|
# (docker/python/Dockerfile installs exactly this with `uv sync --locked`).
|
||||||
|
#
|
||||||
|
# There is no Python source here — the repo is shell scripts, VRTs and metadata —
|
||||||
|
# so `package = false` below: uv installs the dependencies and never tries to
|
||||||
|
# build or install the project itself.
|
||||||
|
#
|
||||||
|
# Versions: this file carries lower bounds, `uv.lock` carries the exact pins for
|
||||||
|
# the *whole* transitive tree. Both are committed, and the image build runs
|
||||||
|
# `uv sync --locked`, so a rebuild resolves to byte-identical versions. Bump
|
||||||
|
# deliberately, then rebuild + re-verify:
|
||||||
|
# uv lock --upgrade-package rasterio # one package
|
||||||
|
# uv lock --upgrade # everything
|
||||||
|
# (The bounds below are the versions pinned 2026-09-14, all supporting Python
|
||||||
|
# 3.14 — the Ubuntu 26.04 system interpreter the image uses.)
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "d4c-datapkg-orthoimagery"
|
||||||
|
version = "0.0.0"
|
||||||
|
description = "Orthoimagery data packages for Data for Canada"
|
||||||
|
requires-python = ">=3.14"
|
||||||
|
dependencies = [
|
||||||
|
"dask[array]>=2026.8.0",
|
||||||
|
"numpy>=2.5.3",
|
||||||
|
"portolan-cli>=0.8.0",
|
||||||
|
"rasterio>=1.5.1",
|
||||||
|
"rioxarray>=0.23.0",
|
||||||
|
"xarray>=2026.7.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.uv]
|
||||||
|
# No Python package of our own to build/install — dependencies only.
|
||||||
|
package = false
|
||||||
|
|
||||||
|
# These wheels bundle their own libgdal, which would shadow the image's GDAL
|
||||||
|
# (built with ECW + MrSID) and silently lose both drivers. Building them from
|
||||||
|
# source makes them link against the image's libgdal via `gdal-config`.
|
||||||
|
# portolan-cli pulls in both rasterio and pyogrio, so this is load-bearing.
|
||||||
|
#
|
||||||
|
# docker/python/uv.toml says the same thing image-wide (for ad-hoc
|
||||||
|
# `uv pip install` inside the container); this key is the project's own copy so
|
||||||
|
# `uv sync` is correct on its own terms, independent of that file.
|
||||||
|
no-binary-package = ["rasterio", "fiona", "pyogrio", "gdal"]
|
||||||
Reference in New Issue
Block a user