diff --git a/main.sh b/main.sh index 366fc5b..b43f17c 100755 --- a/main.sh +++ b/main.sh @@ -53,3 +53,6 @@ census_of_agriculture/process.sh national_address_register/download.sh ## 8.2 Load National Address Register ## national_address_register/process.sh +#### 9.0 Open Databases #### +## 9.1 Download Open Databases ## +open_databases/download.sh diff --git a/open_databases/download.sh b/open_databases/download.sh new file mode 100755 index 0000000..e4665d8 --- /dev/null +++ b/open_databases/download.sh @@ -0,0 +1,38 @@ +#!/bin/bash +if [ ! -d "${DATA_FOLDER}/open_databases" ] +then + echo "Making directory ${DATA_FOLDER}/open_databases/" + mkdir -p ${DATA_FOLDER}/open_databases/{input,extracted,output}/{addresses,buildings,businesses,cultural,educational,greenhouses,healthcare,infrastructure,recreation} +fi + +INPUT_FOLDER="${DATA_FOLDER}/open_databases/input" + +echo "Downloading open databases addresses" +aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_addresses.txt" --dir=${INPUT_FOLDER}/addresses --auto-file-renaming=false +echo "Downloading open databases buildings" +aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_buildings.txt" --dir=${INPUT_FOLDER}/buildings --auto-file-renaming=false +echo "Downloading open databases businesses" +aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_businesses.txt" --dir=${INPUT_FOLDER}/businesses --auto-file-renaming=false +echo "Downloading open databases cultural" +aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_cultural.txt" --dir=${INPUT_FOLDER}/cultural --auto-file-renaming=false +echo "Downloading open databases educational" +aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_educational.txt" --dir=${INPUT_FOLDER}/educational --auto-file-renaming=false +echo "Downloading open databases greenhouses" +aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_greenhouses.txt" --dir=${INPUT_FOLDER}/greenhouses --auto-file-renaming=false +echo "Downloading open databases healthcare" +aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_healthcare.txt" --dir=${INPUT_FOLDER}/healthcare --auto-file-renaming=false +## Infrastructure +echo "Downloading open databases infrastructure" +aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_infrastructure.txt" --dir=${INPUT_FOLDER}/infrastructure --auto-file-renaming=false +# Canadian Pedestrian Network Database +# https://web.archive.org/web/20251111023745/https://www150.statcan.gc.ca/n1/pub/34-26-0004/342600042025001-eng.htm +aria2c -x16 https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/canadian_pedestrian_network_2025-03-19.zip --dir=${INPUT_FOLDER}/infrastructure +# Canadian Cycling Network Database +# https://web.archive.org/web/20260807081428/https://www150.statcan.gc.ca/n1/pub/23-26-0004/232600042024001-eng.htm +aria2c -x16 https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/canadian_cycling_network_database_2025-01-30.zip --dir=${INPUT_FOLDER}/infrastructure +# Canadian Public Transit Network Database +# https://web.archive.org/web/20260611033413/https://www150.statcan.gc.ca/n1/pub/23-26-0003/232600032025001-eng.htm +aria2c -x16 https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/canadian_public_transit_network_database_2025-05-07.zip --dir=${INPUT_FOLDER}/infrastructure +## Infrastructure +echo "Downloading open databases recreation" +aria2c -x16 -i "${SCRIPT_DIR}/open_databases/opendatabases_recreation.txt" --dir=${INPUT_FOLDER}/recreation --auto-file-renaming=false \ No newline at end of file diff --git a/open_databases/files_opendatabase_addresses.txt b/open_databases/files_opendatabase_addresses.txt deleted file mode 100644 index 9661319..0000000 --- a/open_databases/files_opendatabase_addresses.txt +++ /dev/null @@ -1,10 +0,0 @@ -https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_AB_v1.zip -https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_BC_v1.zip -https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_MB_v1.zip -https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_NB_v1.zip -https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_NT_v1.zip -https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_NS_v1.zip -https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_ON_v1.zip -https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_PE_v1.zip -https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_QC_v1.zip -https://www150.statcan.gc.ca/n1/en/pub/46-26-0001/2021001/ODA_SK_v1.zip \ No newline at end of file diff --git a/open_databases/files_opendatabase_buildings.txt b/open_databases/files_opendatabase_buildings.txt deleted file mode 100644 index 4fd116e..0000000 --- a/open_databases/files_opendatabase_buildings.txt +++ /dev/null @@ -1,8 +0,0 @@ -https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Alberta.zip?st=0J_AsIyy -https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_BritishColumbia.zip -https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NewBrunswick.zip?st=k35-Ygwr -https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NorthwestTerritories.zip?st=SXozU436 -https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NovaScotia.zip -https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Ontario.zip -https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Quebec.zip -https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Saskatchewan.zip \ No newline at end of file diff --git a/open_databases/files_opendatabase_cultural.txt b/open_databases/files_opendatabase_cultural.txt deleted file mode 100644 index 51a262e..0000000 --- a/open_databases/files_opendatabase_cultural.txt +++ /dev/null @@ -1 +0,0 @@ -https://www150.statcan.gc.ca/n1/en/pub/21-26-0001/2020001/ODCAF_V1.0.zip \ No newline at end of file diff --git a/open_databases/files_opendatabase_educational.txt b/open_databases/files_opendatabase_educational.txt deleted file mode 100644 index 3161708..0000000 --- a/open_databases/files_opendatabase_educational.txt +++ /dev/null @@ -1 +0,0 @@ -https://www150.statcan.gc.ca/n1/en/pub/37-26-0001/2022001/ODEF_v2.1.zip \ No newline at end of file diff --git a/open_databases/files_opendatabase_greenhouses.txt b/open_databases/files_opendatabase_greenhouses.txt deleted file mode 100644 index 70fb308..0000000 --- a/open_databases/files_opendatabase_greenhouses.txt +++ /dev/null @@ -1 +0,0 @@ -https://www150.statcan.gc.ca/n1/en/pub/32-26-0005/2023001/ODG_V1.zip \ No newline at end of file diff --git a/open_databases/files_opendatabase_healthcare.txt b/open_databases/files_opendatabase_healthcare.txt deleted file mode 100644 index f8e5052..0000000 --- a/open_databases/files_opendatabase_healthcare.txt +++ /dev/null @@ -1 +0,0 @@ -https://www150.statcan.gc.ca/n1/en/pub/13-26-0001/2020001/ODHF_v1.1.zip \ No newline at end of file diff --git a/open_databases/files_opendatabase_recreation.txt b/open_databases/files_opendatabase_recreation.txt deleted file mode 100644 index 711ca77..0000000 --- a/open_databases/files_opendatabase_recreation.txt +++ /dev/null @@ -1 +0,0 @@ -https://www150.statcan.gc.ca/n1/en/pub/21-26-0002/2021001/ODRSF_v1.0.zip \ No newline at end of file diff --git a/open_databases/opendatabases_addresses.txt b/open_databases/opendatabases_addresses.txt new file mode 100644 index 0000000..6d27e46 --- /dev/null +++ b/open_databases/opendatabases_addresses.txt @@ -0,0 +1,11 @@ +# https://web.archive.org/web/20260426164254/https://www.statcan.gc.ca/en/lode/databases/oda +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_AB_v1.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_BC_v1.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_MB_v1.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_NB_v1.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_NT_v1.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_NS_v1.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_ON_v1.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_PE_v1.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_QC_v1.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/addresses/ODA_SK_v1.zip \ No newline at end of file diff --git a/open_databases/opendatabases_buildings.txt b/open_databases/opendatabases_buildings.txt new file mode 100644 index 0000000..cf2e84d --- /dev/null +++ b/open_databases/opendatabases_buildings.txt @@ -0,0 +1,29 @@ +# https://web.archive.org/web/20260313042557/https://www150.statcan.gc.ca/n1/pub/34-26-0001/342600012018001-eng.htm + +# Version 2.0 +# Looks like the version 2 links are now dead +https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Alberta.zip?st=0J_AsIyy +https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_BritishColumbia.zip +https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NewBrunswick.zip?st=k35-Ygwr +https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NorthwestTerritories.zip?st=SXozU436 +https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_NovaScotia.zip +https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Ontario.zip +https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Quebec.zip +https://www150.statcan.gc.ca/n1/en/pub/34-26-0001/2018001/ODB_v2_Saskatchewan.zip + +# Version 3.0 +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_NL.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_PE.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_NS.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_NB.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_QC_1.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_QC_2.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_ON_1.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_ON_2.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_ON_3.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_MB.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_SK.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_AB.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_BC.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_YT.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/buildings/ODB_v3_NT.zip \ No newline at end of file diff --git a/open_databases/opendatabases_businesses.txt b/open_databases/opendatabases_businesses.txt new file mode 100644 index 0000000..54923ef --- /dev/null +++ b/open_databases/opendatabases_businesses.txt @@ -0,0 +1,2 @@ +# https://web.archive.org/web/20260425232705/https://www150.statcan.gc.ca/n1/pub/21-26-0003/212600032023001-eng.htm +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/businesses/ODBus_2023.zip \ No newline at end of file diff --git a/open_databases/opendatabases_cultural.txt b/open_databases/opendatabases_cultural.txt new file mode 100644 index 0000000..9a54eb0 --- /dev/null +++ b/open_databases/opendatabases_cultural.txt @@ -0,0 +1,2 @@ +# https://web.archive.org/web/20260410204622/https://www.statcan.gc.ca/en/lode/databases/odcaf +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/cultural/ODCAF_V1.0.zip \ No newline at end of file diff --git a/open_databases/opendatabases_educational.txt b/open_databases/opendatabases_educational.txt new file mode 100644 index 0000000..db80dde --- /dev/null +++ b/open_databases/opendatabases_educational.txt @@ -0,0 +1,7 @@ +# https://web.archive.org/web/20251114214413/https://www150.statcan.gc.ca/n1/pub/37-26-0001/372600012022001-eng.htm + +# Version 2.1 +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/educational/ODEF_v2.1.zip + +# Version 3.0 +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/educational/ODEF_v3.0.zip \ No newline at end of file diff --git a/open_databases/opendatabases_greenhouses.txt b/open_databases/opendatabases_greenhouses.txt new file mode 100644 index 0000000..897a2e1 --- /dev/null +++ b/open_databases/opendatabases_greenhouses.txt @@ -0,0 +1,7 @@ +# https://web.archive.org/web/20260417003905/https://www150.statcan.gc.ca/n1/pub/32-26-0005/322600052023001-eng.htm + +# Version 1.0 +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/greenhouses/ODG_V1-eng.zip + +# Version 2.0 +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/greenhouses/ODG_V2-eng.zip \ No newline at end of file diff --git a/open_databases/opendatabases_healthcare.txt b/open_databases/opendatabases_healthcare.txt new file mode 100644 index 0000000..c4da830 --- /dev/null +++ b/open_databases/opendatabases_healthcare.txt @@ -0,0 +1,2 @@ +# https://web.archive.org/web/20260418105420/https://www.statcan.gc.ca/en/lode/databases/odhf +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/healthcare/ODHF_v1.1.zip \ No newline at end of file diff --git a/open_databases/opendatabases_infrastructure.txt b/open_databases/opendatabases_infrastructure.txt new file mode 100644 index 0000000..a460018 --- /dev/null +++ b/open_databases/opendatabases_infrastructure.txt @@ -0,0 +1,12 @@ +# https://web.archive.org/web/20260120190207/https://www150.statcan.gc.ca/n1/pub/34-26-0003/342600032023001-eng.htm +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_electric_grid.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_low_carbon.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_oil_gas.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_airports.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_bridges_tunnels.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_ports_marinas.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_railways.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_potable_water.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_solid_waste.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_telecom.zip +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/infrastructure/ODI_v2_wastewater_stormwater.zip \ No newline at end of file diff --git a/open_databases/opendatabases_recreation.txt b/open_databases/opendatabases_recreation.txt new file mode 100644 index 0000000..f9518fa --- /dev/null +++ b/open_databases/opendatabases_recreation.txt @@ -0,0 +1,2 @@ +# https://web.archive.org/web/20260418100230/https://www.statcan.gc.ca/en/lode/databases/odrsf +https://data.source.coop/dataforcanada/d4c-datapkg-foundation/archive/statcan/open_databases/recreation/ODRSF_v1.0.zip \ No newline at end of file