Add various Open Database datasets, change URLs to Source Cooperative

main.sh

@@ -53,3 +53,6 @@ census_of_agriculture/process.sh
 national_address_register/download.sh
 ## 8.2 Load National Address Register ##
 national_address_register/process.sh
+#### 9.0 Open Databases ####
+## 9.1 Download Open Databases ##
+open_databases/download.sh

open_databases/download.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+if [ ! -d "${DATA_FOLDER}/open_databases" ]
+then
+    echo "Making directory ${DATA_FOLDER}/open_databases/"
+    mkdir -p ${DATA_FOLDER}/open_databases/{input,extracted,output}/{addresses,buildings,businesses,cultural,educational,greenhouses,healthcare,infrastructure,recreation}
+fi
+
+INPUT_FOLDER="${DATA_FOLDER}/open_databases/input"
+
+echo "Downloading open databases addresses"
+aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_addresses.txt" --dir=${INPUT_FOLDER}/addresses --auto-file-renaming=false
+echo "Downloading open databases buildings"
+aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_buildings.txt" --dir=${INPUT_FOLDER}/buildings --auto-file-renaming=false
+echo "Downloading open databases businesses"
+aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_businesses.txt" --dir=${INPUT_FOLDER}/businesses --auto-file-renaming=false
+echo "Downloading open databases cultural"
+aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_cultural.txt" --dir=${INPUT_FOLDER}/cultural --auto-file-renaming=false
+echo "Downloading open databases educational"
+aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_educational.txt" --dir=${INPUT_FOLDER}/educational --auto-file-renaming=false
+echo "Downloading open databases greenhouses"
+aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_greenhouses.txt" --dir=${INPUT_FOLDER}/greenhouses --auto-file-renaming=false
+echo "Downloading open databases healthcare"
+aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_healthcare.txt" --dir=${INPUT_FOLDER}/healthcare --auto-file-renaming=false
+## Infrastructure
+echo "Downloading open databases infrastructure"
+aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_infrastructure.txt" --dir=${INPUT_FOLDER}/infrastructure --auto-file-renaming=false
+# Canadian Pedestrian Network Database
+# https://web.archive.org/web/20251111023745/https://www150.statcan.gc.ca/n1/pub/34-26-0004/342600042025001-eng.htm
+aria2c -x16 https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/canadian_pedestrian_network_2025-03-19.zip --dir=${INPUT_FOLDER}/infrastructure
+# Canadian Cycling Network Database
+# https://web.archive.org/web/20260807081428/https://www150.statcan.gc.ca/n1/pub/23-26-0004/232600042024001-eng.htm
+aria2c -x16 https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/canadian_cycling_network_database_2025-01-30.zip --dir=${INPUT_FOLDER}/infrastructure
+#  Canadian Public Transit Network Database 
+# https://web.archive.org/web/20260611033413/https://www150.statcan.gc.ca/n1/pub/23-26-0003/232600032025001-eng.htm
+aria2c -x16 https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/canadian_public_transit_network_database_2025-05-07.zip --dir=${INPUT_FOLDER}/infrastructure
+## Infrastructure
+echo "Downloading open databases recreation"
+aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_recreation.txt" --dir=${INPUT_FOLDER}/recreation --auto-file-renaming=false

open_databases/files_opendatabase_addresses.txt

@@ -1,10 +0,0 @@
-https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_AB_v1.zip
-https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_BC_v1.zip
-https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_MB_v1.zip
-https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_NB_v1.zip
-https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_NT_v1.zip
-https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_NS_v1.zip
-https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_ON_v1.zip
-https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_PE_v1.zip
-https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_QC_v1.zip
-https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_SK_v1.zip

open_databases/files_opendatabase_buildings.txt

@@ -1,8 +0,0 @@
-https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Alberta.zip?st=0J_AsIyy
-https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_BritishColumbia.zip
-https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NewBrunswick.zip?st=k35-Ygwr
-https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NorthwestTerritories.zip?st=SXozU436
-https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NovaScotia.zip
-https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Ontario.zip
-https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Quebec.zip
-https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Saskatchewan.zip

open_databases/files_opendatabase_cultural.txt

@@ -1 +0,0 @@
-https://www150.statcan.gc.ca/n1/en/pub/21-26-0001/2020001/ODCAF_V1.0.zip

open_databases/files_opendatabase_educational.txt

@@ -1 +0,0 @@
-https://www150.statcan.gc.ca/n1/en/pub/37-26-0001/2022001/ODEF_v2.1.zip

open_databases/files_opendatabase_greenhouses.txt

@@ -1 +0,0 @@
-https://www150.statcan.gc.ca/n1/en/pub/32-26-0005/2023001/ODG_V1.zip

open_databases/files_opendatabase_healthcare.txt

@@ -1 +0,0 @@
-https://www150.statcan.gc.ca/n1/en/pub/13-26-0001/2020001/ODHF_v1.1.zip

open_databases/files_opendatabase_recreation.txt

@@ -1 +0,0 @@
-https://www150.statcan.gc.ca/n1/en/pub/21-26-0002/2021001/ODRSF_v1.0.zip

open_databases/opendatabases_addresses.txt

@@ -0,0 +1,11 @@
+# https://web.archive.org/web/20260426164254/https://www.statcan.gc.ca/en/lode/databases/oda
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_AB_v1.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_BC_v1.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_MB_v1.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_NB_v1.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_NT_v1.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_NS_v1.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_ON_v1.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_PE_v1.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_QC_v1.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_SK_v1.zip

open_databases/opendatabases_buildings.txt

@@ -0,0 +1,29 @@
+# https://web.archive.org/web/20260313042557/https://www150.statcan.gc.ca/n1/pub/34-26-0001/342600012018001-eng.htm
+
+# Version 2.0
+# Looks like the version 2 links are now dead
+https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Alberta.zip?st=0J_AsIyy
+https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_BritishColumbia.zip
+https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NewBrunswick.zip?st=k35-Ygwr
+https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NorthwestTerritories.zip?st=SXozU436
+https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NovaScotia.zip
+https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Ontario.zip
+https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Quebec.zip
+https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Saskatchewan.zip
+
+# Version 3.0
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_NL.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_PE.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_NS.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_NB.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_QC_1.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_QC_2.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_ON_1.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_ON_2.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_ON_3.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_MB.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_SK.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_AB.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_BC.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_YT.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_NT.zip

open_databases/opendatabases_businesses.txt

@@ -0,0 +1,2 @@
+# https://web.archive.org/web/20260425232705/https://www150.statcan.gc.ca/n1/pub/21-26-0003/212600032023001-eng.htm
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/businesses/ODBus_2023.zip

open_databases/opendatabases_cultural.txt

@@ -0,0 +1,2 @@
+# https://web.archive.org/web/20260410204622/https://www.statcan.gc.ca/en/lode/databases/odcaf
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/cultural/ODCAF_V1.0.zip

open_databases/opendatabases_educational.txt

@@ -0,0 +1,7 @@
+# https://web.archive.org/web/20251114214413/https://www150.statcan.gc.ca/n1/pub/37-26-0001/372600012022001-eng.htm
+
+# Version 2.1
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/educational/ODEF_v2.1.zip
+
+# Version 3.0
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/educational/ODEF_v3.0.zip

open_databases/opendatabases_greenhouses.txt

@@ -0,0 +1,7 @@
+# https://web.archive.org/web/20260417003905/https://www150.statcan.gc.ca/n1/pub/32-26-0005/322600052023001-eng.htm
+
+# Version 1.0
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/greenhouses/ODG_V1-eng.zip
+
+# Version 2.0
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/greenhouses/ODG_V2-eng.zip

open_databases/opendatabases_healthcare.txt

@@ -0,0 +1,2 @@
+# https://web.archive.org/web/20260418105420/https://www.statcan.gc.ca/en/lode/databases/odhf
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/healthcare/ODHF_v1.1.zip

open_databases/opendatabases_infrastructure.txt

@@ -0,0 +1,12 @@
+# https://web.archive.org/web/20260120190207/https://www150.statcan.gc.ca/n1/pub/34-26-0003/342600032023001-eng.htm
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_electric_grid.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_low_carbon.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_oil_gas.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_airports.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_bridges_tunnels.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_ports_marinas.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_railways.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_potable_water.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_solid_waste.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_telecom.zip
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_wastewater_stormwater.zip

open_databases/opendatabases_recreation.txt

@@ -0,0 +1,2 @@
+# https://web.archive.org/web/20260418100230/https://www.statcan.gc.ca/en/lode/databases/odrsf
+https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/recreation/ODRSF_v1.0.zip