diff --git a/scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/download.sh b/scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/download.sh deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/README.md b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/README.md new file mode 100644 index 0000000..54c0729 --- /dev/null +++ b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/README.md @@ -0,0 +1,34 @@ +# Vancouver 2022 Orthoimagery — Download Script + +This directory contains the automation script for acquiring the **City of Vancouver 2022 Orthophoto Imagery** dataset (7.5 cm resolution) from [Vancouver Open Data](https://opendata.vancouver.ca/explore/dataset/orthophoto-imagery-2022/). + +## What the Script Does + +`download.sh` performs four sequential steps: + +1. **Download Index** — Uses `aria2c` to fetch the dataset catalogue as a Parquet file from the Vancouver Open Data API. +2. **Extract URLs** — Queries the Parquet file with `duckdb` to extract all MrSID image URLs into a plain-text file suitable for batch downloading. +3. **Create Output Directory** — Ensures the data input directory exists at `../../data/input/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/` (relative to this script). +4. **Download Images** — Uses `aria2c` to download all images in parallel (12 concurrent connections, 4 connections per server) into the data input directory. + +## Dependencies + +The following command-line tools must be installed and available on your `PATH`: + +| Tool | Purpose | Install | +|---|---|---| +| [aria2c](https://aria2.github.io/) | High-speed parallel downloads | `sudo apt install aria2` | +| [duckdb](https://duckdb.org/) | Query Parquet files from the CLI | [Install guide](https://duckdb.org/docs/installation/) | + +## Usage + +```bash +cd scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm +bash download.sh +``` + +The script will print progress for each step. Once complete, the downloaded MrSID image files will be located in: + +``` +data/input/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ +``` diff --git a/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_index_2022_075mm_2026-03-09.parquet b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_index_2022_075mm_2026-03-09.parquet new file mode 100644 index 0000000..d05464a Binary files /dev/null and b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_index_2022_075mm_2026-03-09.parquet differ diff --git a/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_index_2022_075mm_2026-03-09.txt b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_index_2022_075mm_2026-03-09.txt new file mode 100644 index 0000000..98115ee --- /dev/null +++ b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_index_2022_075mm_2026-03-09.txt @@ -0,0 +1,470 @@ +https://webtransfer.vancouver.ca/opendata/2022sid/B12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/B14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/C14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H08.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I08.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K2.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M1.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/C11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G08.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M4.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/B11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/B13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/B15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/B16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/C10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M2.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N4.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O4.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/C16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M3.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N2.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/B17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/C12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/C13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O02.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P03.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/B10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/C17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F08.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K3.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L1.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L3.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N1.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/C9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/B9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J08.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K07.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L2.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N3.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q04.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R27.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/C15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/C7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/C8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/E13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/F16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/G9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K06.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K08.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K1.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K4.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L4.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O3.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P14.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U18.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U21.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V23.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V7.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/X20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/D19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/H20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/I9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J10.zip +https://webtransfer.vancouver.ca/opendata/2022sid/J20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/K16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L17.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/L6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/M20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N11.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N25.zip +https://webtransfer.vancouver.ca/opendata/2022sid/N5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O24.zip +https://webtransfer.vancouver.ca/opendata/2022sid/O6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/P04.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q19.zip +https://webtransfer.vancouver.ca/opendata/2022sid/Q26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R5.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R6.zip +https://webtransfer.vancouver.ca/opendata/2022sid/R9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/S13.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T16.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T22.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T8.zip +https://webtransfer.vancouver.ca/opendata/2022sid/T9.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U12.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U20.zip +https://webtransfer.vancouver.ca/opendata/2022sid/U26.zip +https://webtransfer.vancouver.ca/opendata/2022sid/V15.zip +https://webtransfer.vancouver.ca/opendata/2022sid/W16.zip diff --git a/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/download.sh b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/download.sh new file mode 100755 index 0000000..090b3c7 --- /dev/null +++ b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/download.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# +# download.sh — Download Vancouver 2022 orthoimagery dataset +# +# This script: +# 1. Downloads the dataset index as a Parquet file from Vancouver Open Data. +# 2. Extracts image URLs from the Parquet file using DuckDB. +# 3. Creates the data input directory. +# 4. Downloads all images in parallel using aria2c. +# +# Usage: +# cd scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm +# bash download.sh +# + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DATASET_ID="ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm" +INDEX_BASE="ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_index_2022_075mm_2026-03-09" +PARQUET_FILE="${SCRIPT_DIR}/${INDEX_BASE}.parquet" +URL_FILE="${SCRIPT_DIR}/${INDEX_BASE}.txt" +DATA_INPUT_DIR="${SCRIPT_DIR}/../../data/input/${DATASET_ID}" + +PARQUET_URL="https://opendata.vancouver.ca/api/explore/v2.1/catalog/datasets/orthophoto-imagery-2022/exports/parquet" + +# --------------------------------------------------------------------------- +# Step 1 — Download the Parquet index file +# --------------------------------------------------------------------------- +echo "==> Step 1: Downloading Parquet index file..." +aria2c \ + "${PARQUET_URL}" \ + -d "${SCRIPT_DIR}" \ + -o "${INDEX_BASE}.parquet" + +echo " Saved to: ${PARQUET_FILE}" + +# --------------------------------------------------------------------------- +# Step 2 — Extract image URLs from the Parquet file using DuckDB +# --------------------------------------------------------------------------- +echo "==> Step 2: Extracting image URLs with DuckDB..." +duckdb -noheader -csv -c \ + "SELECT mrsid_url FROM '${PARQUET_FILE}';" \ + | tr -d '"' \ + > "${URL_FILE}" + +echo " URL list written to: ${URL_FILE}" + +# --------------------------------------------------------------------------- +# Step 3 — Create the data input directory +# --------------------------------------------------------------------------- +echo "==> Step 3: Creating data input directory..." +mkdir -p "${DATA_INPUT_DIR}" +# Resolve to an absolute path to avoid any relative-path surprises +DATA_INPUT_DIR="$(cd "${DATA_INPUT_DIR}" && pwd)" +echo " Directory ready: ${DATA_INPUT_DIR}" + +# --------------------------------------------------------------------------- +# Step 4 — Download all images into the data input directory +# --------------------------------------------------------------------------- +echo "==> Step 4: Downloading images with aria2c..." +aria2c \ + -i "${URL_FILE}" \ + -j 12 \ + -x 4 \ + -d "${DATA_INPUT_DIR}" + +echo "==> Done. All files downloaded to: ${DATA_INPUT_DIR}" diff --git a/scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/generate_tiles.sh b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/generate_tiles.sh similarity index 97% rename from scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/generate_tiles.sh rename to scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/generate_tiles.sh index 2449eb5..5f5bffd 100755 --- a/scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/generate_tiles.sh +++ b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/generate_tiles.sh @@ -1,5 +1,6 @@ #!/bin/bash +# TODO: Update paths in this script PROJECT_DIR="~/Documents/Personal/Projects/dataforcanada/process-orthoimagery-dev" DATA_DIR="${PROJECT_DIR}/data" DATA_INPUT_DIR="${DATA_DIR}/input/maptiler_engine_experiments" diff --git a/scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/tune.sh b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/tune.sh similarity index 100% rename from scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/tune.sh rename to scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/tune.sh