mirror of
https://github.com/dataforcanada/d4c-datapkg-statistical.git
synced 2026-06-13 14:10:55 +02:00
Initial commit
This commit is contained in:
Diego Ripley
@@ -0,0 +1,2 @@
|
||||
# TODO
|
||||
- Process 2021 hydro
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
if [ ! -d "${DATA_FOLDER}/hydro" ]
|
||||
then
|
||||
echo "Making directory ${DATA_FOLDER}/hydro/"
|
||||
mkdir -p ${DATA_FOLDER}/hydro/{input,output}/{2021,2016,2011,2006}
|
||||
fi
|
||||
|
||||
INPUT_FOLDER="${DATA_FOLDER}/hydro/input"
|
||||
|
||||
#echo "Downloading 2021 hydro"
|
||||
#aria2c -x16 -i "${SCRIPT_DIR}/hydro/hydro_2021.txt" --dir=${INPUT_FOLDER}/2021 --auto-file-renaming=false
|
||||
echo "Downloading 2016 hydro"
|
||||
aria2c -x16 -i "${SCRIPT_DIR}/hydro/hydro_2016.txt" --dir=${INPUT_FOLDER}/2016 --auto-file-renaming=false
|
||||
echo "Downloading 2011 boundaries"
|
||||
aria2c -x16 -i "${SCRIPT_DIR}/hydro/hydro_2011.txt" --dir=${INPUT_FOLDER}/2011 --auto-file-renaming=false
|
||||
echo "Downloading 2006 boundaries"
|
||||
aria2c -x16 -i "${SCRIPT_DIR}/hydro/hydro_2006.txt" --dir=${INPUT_FOLDER}/2006 --auto-file-renaming=false
|
||||
@@ -0,0 +1,4 @@
|
||||
# https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-2006-eng.cfm
|
||||
https://data.dataforcanada.org/archive/statistics_canada/hydro/2006/ghy_000c06a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/files-fichiers/ghy_000c06a_e.zip
|
||||
https://data.dataforcanada.org/archive/statistics_canada/hydro/2006/ghy_000d06a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/files-fichiers/ghy_000d06a_e.zip
|
||||
https://data.dataforcanada.org/archive/statistics_canada/hydro/2006/ghy_000f06a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/files-fichiers/ghy_000f06a_e.zip
|
||||
@@ -0,0 +1,4 @@
|
||||
# https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-2011-eng.cfm
|
||||
https://data.dataforcanada.org/archive/statistics_canada/hydro/2011/ghy_000c11a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/files-fichiers/ghy_000c11a_e.zip
|
||||
https://data.dataforcanada.org/archive/statistics_canada/hydro/2011/ghy_000d11a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/files-fichiers/ghy_000d11a_e.zip
|
||||
https://data.dataforcanada.org/archive/statistics_canada/hydro/2011/ghy_000h11a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/files-fichiers/ghy_000h11a_e.zip
|
||||
@@ -0,0 +1,4 @@
|
||||
# https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-2016-eng.cfm
|
||||
https://data.dataforcanada.org/archive/statistics_canada/hydro/2016/lhy_000c16a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/files-fichiers/2016/lhy_000c16a_e.zip
|
||||
https://data.dataforcanada.org/archive/statistics_canada/hydro/2016/lhy_000d16a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/files-fichiers/2016/lhy_000d16a_e.zip
|
||||
https://data.dataforcanada.org/archive/statistics_canada/hydro/2016/lhy_000h16a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/files-fichiers/2016/lhy_000h16a_e.zip
|
||||
@@ -0,0 +1,15 @@
|
||||
# https://open.canada.ca/data/en/dataset/9d96e8c9-22fe-4ad2-b5e8-94a6991b744b
|
||||
# They are the "Prepackaged FGDB files (download directory)", 50K Hydro
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_AB_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_AB_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_AB_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_BC_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_BC_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_BC_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_MB_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_MB_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_MB_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_NB_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_NB_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_NB_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_NL_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_NL_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_NL_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_NS_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_NS_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_NS_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_NT_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_NT_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_NT_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_NU_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_NU_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_NU_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_ON_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_ON_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_ON_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_PE_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_PE_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_PE_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_QC_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_QC_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_QC_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_SK_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_SK_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_SK_Hydro_fgdb.zip
|
||||
https://zenodo.org/records/15350060/files/canvec_50K_YT_Hydro_fgdb.zip?download=1 https://data.dataforcanada.org/archive/natural_resources_canada/hydro/2019/canvec_50K_YT_Hydro_fgdb.zip https://archive.org/download/canvec-hydro-50k-2019-07-24/canvec_50K_YT_Hydro_fgdb.zip
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
|
||||
import_to_postgis() {
|
||||
local filepath="$1"
|
||||
local table_name="$2"
|
||||
local extra_parameters="${@:3}"
|
||||
|
||||
# Handle zip files using GDAL's virtual file system
|
||||
if [[ "${filepath: -4}" == ".zip" ]]; then
|
||||
filepath="/vsizip/${filepath}"
|
||||
fi
|
||||
|
||||
echo "Importing ${filepath} into table ${table_name}"
|
||||
ogr2ogr \
|
||||
--config PG_USE_COPY YES \
|
||||
-lco "OVERWRITE=YES" \
|
||||
-f "PostgreSQL" \
|
||||
"PG:host=db dbname=${POSTGRES_DB} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} port=5432" \
|
||||
-lco GEOMETRY_NAME=geom \
|
||||
-progress \
|
||||
-gt 500000 \
|
||||
-t_srs EPSG:4326 \
|
||||
-nln "${table_name}" \
|
||||
${extra_parameters} \
|
||||
"${filepath}"
|
||||
}
|
||||
|
||||
# Define input folders
|
||||
INPUT_FOLDER="${DATA_FOLDER}/hydro/input"
|
||||
EXTRACTED_FOLDER="${DATA_FOLDER}/hydro/extracted"
|
||||
|
||||
# Import 2016 hydro data
|
||||
import_data_2016() {
|
||||
# Source: https://web.archive.org/web/20230120140926/https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-2016-eng.cfm
|
||||
# Lakes and rivers (polygons)
|
||||
import_to_postgis "${INPUT_FOLDER}/2016/lhy_000c16a_e.zip" lhy_000c16a_e "-nlt PROMOTE_TO_MULTI -lco SCHEMA=bronze"
|
||||
# Rivers (lines)
|
||||
import_to_postgis "${INPUT_FOLDER}/2016/lhy_000d16a_e.zip" lhy_000d16a_e "-lco SCHEMA=bronze"
|
||||
# Coastal waters (polygons)
|
||||
import_to_postgis "${INPUT_FOLDER}/2016/lhy_000h16a_e.zip" lhy_000h16a_e "-lco SCHEMA=bronze"
|
||||
}
|
||||
|
||||
# Import 2011 hydro data
|
||||
import_data_2011() {
|
||||
# Source: https://web.archive.org/web/20230110163150/https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-2011-eng.cfm
|
||||
export PGCLIENTENCODING=LATIN-1;
|
||||
# Lakes and rivers (polygons)
|
||||
import_to_postgis "${INPUT_FOLDER}/2011/ghy_000c11a_e.zip" ghy_000c11a_e "-nlt PROMOTE_TO_MULTI -lco SCHEMA=bronze"
|
||||
# Rivers (lines)
|
||||
import_to_postgis "${INPUT_FOLDER}/2011/ghy_000d11a_e.zip" ghy_000d11a_e "-lco SCHEMA=bronze"
|
||||
# Coastal waters (polygons)
|
||||
import_to_postgis "${INPUT_FOLDER}/2011/ghy_000h11a_e.zip" ghy_000h11a_e "-nlt PROMOTE_TO_MULTI -lco SCHEMA=bronze"
|
||||
}
|
||||
|
||||
# Import 2006 hydro data
|
||||
import_data_2006() {
|
||||
# Source: https://web.archive.org/web/20221218043125/https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-2006-eng.cfm
|
||||
export PGCLIENTENCODING=LATIN-1;
|
||||
# Lakes and rivers (polygons)
|
||||
import_to_postgis "${INPUT_FOLDER}/2006/ghy_000c06a_e.zip" ghy_000c06a_e "-nlt PROMOTE_TO_MULTI -lco SCHEMA=bronze"
|
||||
# Rivers (lines)
|
||||
import_to_postgis "${INPUT_FOLDER}/2006/ghy_000d06a_e.zip" ghy_000d06a_e "-lco SCHEMA=bronze"
|
||||
# Coastal waters (polygons)
|
||||
import_to_postgis "${INPUT_FOLDER}/2006/ghy_000f06a_e.zip" ghy_000f06a_e "-nlt PROMOTE_TO_MULTI -lco SCHEMA=bronze"
|
||||
}
|
||||
|
||||
# Execute all import functions
|
||||
import_data_2016
|
||||
import_data_2011
|
||||
import_data_2006
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
# Standardizes field names, builds up hierarchy for datasets, create Canada and Geographic Regions of Canada
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/country.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/geographic_regions_of_canada.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/provinces_and_territories.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/census_divisions.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/economic_regions.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/census_agricultural_regions.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/forward_sortation_areas.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/federal_electoral_districts.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/census_consolidated_subdivisions.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/census_metropolitan_areas.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/census_subdivisions.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/population_centres.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/designated_places.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/census_tracts.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/aggregate_dissemination_areas.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/dissemination_areas.sql
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f boundaries/dissemination_blocks.sql
|
||||
Reference in New Issue
Block a user