mirror of
https://github.com/dataforcanada/d4c-datapkg-statistical.git
synced 2026-06-13 14:10:55 +02:00
Initial commit
This commit is contained in:
Diego Ripley
@@ -0,0 +1,4 @@
|
||||
# TODO
|
||||
- Process 2006 Geographic Attribute File Road Network
|
||||
- Process 2001 road network
|
||||
- Change loading of Census Road Network files as a function
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
if [ ! -d "${DATA_FOLDER}/road_network_files" ]
|
||||
then
|
||||
echo "Making directory ${DATA_FOLDER}/road_network_files/"
|
||||
mkdir -p ${DATA_FOLDER}/road_network_files/{input,extracted,output}/{2021,2016,2011,2006}
|
||||
fi
|
||||
|
||||
INPUT_FOLDER="${DATA_FOLDER}/road_network_files/input"
|
||||
|
||||
echo "Downloading 2021 road network file"
|
||||
aria2c -x16 -i "${SCRIPT_DIR}/road_network_files/road_network_file_2021.txt" --dir="${INPUT_FOLDER}/2021" --auto-file-renaming=false
|
||||
echo "Downloading 2016 road network file"
|
||||
aria2c -x16 -i "${SCRIPT_DIR}/road_network_files/road_network_file_2016.txt" --dir="${INPUT_FOLDER}/2016" --auto-file-renaming=false
|
||||
echo "Downloading 2011 road network file"
|
||||
aria2c -x16 -i "${SCRIPT_DIR}/road_network_files/road_network_file_2011.txt" --dir="${INPUT_FOLDER}/2011" --auto-file-renaming=false
|
||||
echo "Downloading 2006 road network file"
|
||||
aria2c -x16 -i "${SCRIPT_DIR}/road_network_files/road_network_file_2006.txt" --dir="${INPUT_FOLDER}/2006" --auto-file-renaming=false
|
||||
echo "Downloading 2001 road network file"
|
||||
aria2c -x16 -i "${SCRIPT_DIR}/road_network_files/road_network_file_2001.txt" --dir="${INPUT_FOLDER}/2001" --auto-file-renaming=false
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/bash
|
||||
|
||||
import_to_postgis() {
|
||||
local filepath="$1"
|
||||
local table_name="$2"
|
||||
local extra_parameters="${@:3}"
|
||||
|
||||
# Handle zip files using GDAL's virtual file system
|
||||
if [[ "${filepath: -4}" == ".zip" ]]; then
|
||||
filepath="/vsizip/${filepath}"
|
||||
fi
|
||||
|
||||
echo "Importing ${filepath} into table bronze.${table_name}"
|
||||
ogr2ogr \
|
||||
--config PG_USE_COPY YES \
|
||||
-lco "OVERWRITE=YES" \
|
||||
-f "PostgreSQL" \
|
||||
"PG:host=db dbname=${POSTGRES_DB} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} port=5432" \
|
||||
-lco GEOMETRY_NAME=geom \
|
||||
-progress \
|
||||
-gt 500000 \
|
||||
-t_srs EPSG:4326 \
|
||||
-nln "${table_name}" \
|
||||
${extra_parameters} \
|
||||
"${filepath}"
|
||||
}
|
||||
|
||||
# Define input folders
|
||||
INPUT_FOLDER="${DATA_FOLDER}/road_network_files/input"
|
||||
EXTRACTED_FOLDER="${DATA_FOLDER}/road_network_files/extracted"
|
||||
|
||||
### 2021 Road Network File ###
|
||||
# https://web.archive.org/web/20230307163203/https://www12.statcan.gc.ca/census-recensement/2021/geo/sip-pis/boundary-limites/index2021-eng.cfm?year=21
|
||||
export PGCLIENTENCODING=LATIN-1;
|
||||
import_to_postgis "${INPUT_FOLDER}/2021/lrnf000r21a_e.zip" lrnf000r21a_e "-lco COLUMN_TYPES=afl_val=integer,atl_val=integer,afr_val=integer,atr_val=integer -lco SCHEMA=bronze"
|
||||
unset PGCLIENTENCODING
|
||||
|
||||
### 2016 Road Network File ###
|
||||
# https://web.archive.org/web/20230120140926/https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-2016-eng.cfm
|
||||
import_to_postgis "${INPUT_FOLDER}/2016/lrnf000r16a_e.zip" lrnf000r16a_e "-lco COLUMN_TYPES=afl_val=integer,atl_val=integer,afr_val=integer,atr_val=integer -lco SCHEMA=bronze"
|
||||
|
||||
### 2011 Road Network File ###
|
||||
# https://web.archive.org/web/20230110163150/https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-2011-eng.cfm
|
||||
export PGCLIENTENCODING=LATIN-1;
|
||||
import_to_postgis "${INPUT_FOLDER}/2011/grnf000r11a_e.zip" grnf000r11a_e "-lco COLUMN_TYPES=afl_val=integer,atl_val=integer,afr_val=integer,atr_val=integer -lco SCHEMA=bronze"
|
||||
unset PGCLIENTENCODING
|
||||
|
||||
### 2006 Road Network File ###
|
||||
# https://web.archive.org/web/20221218043125/https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-2006-eng.cfm
|
||||
export PGCLIENTENCODING=LATIN-1;
|
||||
import_to_postgis "${INPUT_FOLDER}/2006/grnf000r06a_e.zip" grnf000r06a_e "-lco COLUMN_TYPES=addr_fm_le=integer,addr_to_le=integer,addr_fm_rg=integer,addr_to_rg=integer -lco SCHEMA=bronze"
|
||||
unset PGCLIENTENCODING
|
||||
|
||||
# TODO - get working
|
||||
### 2001 Road Network Files ###
|
||||
# https://web.archive.org/web/20221218043135/https://www12.statcan.gc.ca/census-recensement/2011/geo/bound-limit/bound-limit-2001-eng.cfm
|
||||
# Census Road Network file
|
||||
#unzip -n ${INPUT_FOLDER}/grnf000r01m_e.zip -d ${EXTRACTED_FOLDER}
|
||||
#import_to_postgis ${EXTRACTED_FOLDER}/grnf000r02ml_e.MID statcan_grnf000r02ml_e_tmp
|
||||
## Census Skeletal Road Network File
|
||||
#unzip -n ${INPUT_FOLDER}/gsrn000r01m_e.zip -d ${EXTRACTED_FOLDER}
|
||||
#import_to_postgis ${DATA_FOLDER}/gsrn000r02m_e.MID statcan_gsrn000r02m_e_tmp
|
||||
@@ -0,0 +1,48 @@
|
||||
/* Road Network */
|
||||
|
||||
--- 2016 Road Network;
|
||||
drop table if exists statcan_rn_2016;
|
||||
create table statcan_rn_2016 as
|
||||
select ngduid as ngd_uid, "name", type, dir, afl_val, atl_val, afr_val, atr_val,
|
||||
csduid_l, csduid_r, concat(pruid_l, cmauid_l) as cmapuid_l, concat(pruid_r, cmauid_r) as cmapuid_r, pruid_l, pruid_r, rank, class, geom
|
||||
from statcan_lrnf000r16a_e;
|
||||
|
||||
create index statcan_rn_2016_geom_idx on statcan_rn_2016 using GIST(geom) with (FILLFACTOR=100);
|
||||
drop table if exists statcan_lrnf000r16a_e;
|
||||
|
||||
--- 2011 Road Network;
|
||||
drop table if exists statcan_rn_2011;
|
||||
create table statcan_rn_2011 as
|
||||
select ngd_uid, "name", type, dir, afl_val, atl_val, afr_val, atr_val,
|
||||
csduid_l, csduid_r, concat(pruid_l, cmauid_l) as cmapuid_l, concat(pruid_r, cmauid_r) as cmapuid_r, pruid_l, pruid_r, rank, class, geom
|
||||
from statcan_grnf000r11a_e;
|
||||
|
||||
create index statcan_rn_2011_geom_idx on statcan_rn_2011 using GIST(geom) with (FILLFACTOR=100);
|
||||
drop table if exists statcan_grnf000r11a_e;
|
||||
|
||||
--- 2006 Road Network;
|
||||
drop table if exists statcan_rn_2006;
|
||||
create table statcan_rn_2006 as
|
||||
select rb_uid, "name", type, direction as dir, addr_fm_le as afl_val, addr_to_le as atl_val, addr_fm_rg as afr_val, addr_to_rg as atr_val,
|
||||
geom
|
||||
from statcan_grnf000r06a_e;
|
||||
|
||||
create index statcan_rn_2006_geom_idx on statcan_rn_2006 using GIST(geom) with (FILLFACTOR=100);
|
||||
drop table if exists statcan_grnf000r06a_e;
|
||||
|
||||
--- TODO;
|
||||
--- 2001 Road Network;
|
||||
create table statcan_rn_2001_tmp as
|
||||
select arc_id, "name", type, direction as dir, addr_fm_left as afl_val,
|
||||
addr_to_left as atl_val, addr_fm_rght as afr_val, addr_to_rght as atr_val,
|
||||
geom
|
||||
from statcan_grnf000r02ml_e_tmp
|
||||
where class = 'U'
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Executable
+1
@@ -0,0 +1 @@
|
||||
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB" -f road_network_files/road_network_files.sql
|
||||
@@ -0,0 +1,4 @@
|
||||
# 2001 Census Road Network File
|
||||
https://data.dataforcanada.org/archive/statistics_canada/road_network_files/2001/grnf000r01m_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/RNF-FRR/files-fichiers/grnf000r01m_e.zip
|
||||
# 2001 Census Skeletal Road Network File
|
||||
https://data.dataforcanada.org/archive/statistics_canada/road_network_files/2001/gsrn000r01m_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/RNF-FRR/files-fichiers/gsrn000r01m_e.zip
|
||||
@@ -0,0 +1,2 @@
|
||||
# 2006 Census Road Network and Geographic Attribute File Road Network
|
||||
https://data.dataforcanada.org/archive/statistics_canada/road_network_files/2006/grgf000r06a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/RNF-FRR/files-fichiers/grgf000r06a_e.zip
|
||||
@@ -0,0 +1,2 @@
|
||||
# 2011 Census Road Network File
|
||||
https://data.dataforcanada.org/archive/statistics_canada/road_network_files/2011/grnf000r11a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/RNF-FRR/files-fichiers/grnf000r11a_e.zip
|
||||
@@ -0,0 +1,2 @@
|
||||
# 2016 Census Road Network File
|
||||
https://data.dataforcanada.org/archive/statistics_canada/road_network_files/2016/lrnf000r16a_e.zip https://www12.statcan.gc.ca/census-recensement/2011/geo/RNF-FRR/files-fichiers/2016/lrnf000r16a_e.zip
|
||||
@@ -0,0 +1,2 @@
|
||||
# 2021 Census Road Network File. Geodatabase was not working so fall back to shapefile
|
||||
https://data.dataforcanada.org/archive/statistics_canada/road_network_files/2021/lrnf000r21a_e.zip https://www12.statcan.gc.ca/census-recensement/2021/geo/sip-pis/RNF-FRR/files-fichiers/lrnf000r21a_e.zip
|
||||
@@ -0,0 +1,89 @@
|
||||
/*
|
||||
Road Network Files
|
||||
*/
|
||||
|
||||
/* 2021
|
||||
Definition here: https://web.archive.org/web/20240215050313/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/definition-eng.cfm?ID=geo041
|
||||
*/
|
||||
|
||||
DROP TABLE IF EXISTS silver.road_2021;
|
||||
CREATE TABLE silver.road_2021 AS
|
||||
SELECT
|
||||
left_side.country_dguid,
|
||||
left_side.country_en_name,
|
||||
left_side.country_fr_name,
|
||||
left_side.country_en_abbreviation,
|
||||
left_side.country_fr_abbreviation,
|
||||
left_side.grc_dguid AS grc_dguid_left,
|
||||
right_side.grc_dguid AS grc_dguid_right,
|
||||
left_side.grc_en_name AS grc_en_name_left,
|
||||
right_side.grc_en_name AS grc_en_name_right,
|
||||
left_side.grc_fr_name AS grc_fr_name_left,
|
||||
right_side.grc_fr_name AS grc_fr_name_right,
|
||||
left_side.pr_dguid AS pr_dguid_left,
|
||||
right_side.pr_dguid AS pr_dguid_right,
|
||||
left_side.pr_en_name AS pr_en_name_left,
|
||||
right_side.pr_en_name AS pr_en_name_right,
|
||||
left_side.pr_fr_name AS pr_fr_name_left,
|
||||
right_side.pr_fr_name AS pr_fr_name_right,
|
||||
left_side.pr_iso_code AS pr_iso_code_left,
|
||||
right_side.pr_iso_code AS pr_iso_code_right,
|
||||
left_side.car_dguid AS car_dguid_left,
|
||||
right_side.car_dguid AS car_dguid_right,
|
||||
left_side.car_en_name AS car_en_name_left,
|
||||
right_side.car_en_name AS car_en_name_right,
|
||||
left_side.car_fr_name AS car_fr_name_left,
|
||||
right_side.car_fr_name AS car_fr_name_right,
|
||||
left_side.er_dguid AS er_dguid_left,
|
||||
right_side.er_dguid AS er_dguid_right,
|
||||
left_side.er_name AS er_name_left,
|
||||
right_side.er_name AS er_name_right,
|
||||
left_side.cd_dguid AS cd_dguid_left,
|
||||
right_side.cd_dguid AS cd_dguid_right,
|
||||
left_side.cd_name AS cd_name_left,
|
||||
right_side.cd_name AS cd_name_right,
|
||||
left_side.cd_type AS cd_type_left,
|
||||
right_side.cd_type AS cd_type_right,
|
||||
left_side.ccs_dguid AS ccs_dguid_left,
|
||||
right_side.ccs_dguid AS ccs_dguid_right,
|
||||
left_side.ccs_name AS ccs_name_left,
|
||||
right_side.ccs_name AS ccs_name_right,
|
||||
left_side.cma_dguid AS cma_dguid_left,
|
||||
right_side.cma_dguid AS cma_dguid_right,
|
||||
left_side.cma_p_dguid AS cma_p_dguid_left,
|
||||
right_side.cma_p_dguid AS cma_p_dguid_right,
|
||||
left_side.cma_name AS cma_name_left,
|
||||
right_side.cma_name AS cma_name_right,
|
||||
left_side.cma_type AS cma_type_left,
|
||||
right_side.cma_type AS cma_type_right,
|
||||
left_side.csd_dguid AS csd_dguid_left,
|
||||
right_side.csd_dguid AS csd_dguid_right,
|
||||
left_side.csd_name AS csd_name_left,
|
||||
right_side.csd_name AS csd_name_right,
|
||||
left_side.csd_type AS csd_type_left,
|
||||
right_side.csd_type AS csd_type_right,
|
||||
left_side.sac_type AS sac_type_left,
|
||||
right_side.sac_type AS sac_type_right,
|
||||
left_side.sac_code AS sac_code_left,
|
||||
right_side.sac_code AS sac_code_right,
|
||||
road.ngd_uid AS road_ngd_uid,
|
||||
road.name AS road_name,
|
||||
road.type AS road_type,
|
||||
road.dir AS road_direction,
|
||||
road.afl_val AS road_address_from_left_value,
|
||||
road.atl_val AS road_address_to_left_value,
|
||||
road.afr_val AS road_address_from_right_value,
|
||||
road.atr_val AS road_address_to_right_value,
|
||||
road.rank AS road_rank,
|
||||
road.class AS road_class,
|
||||
road.geom
|
||||
FROM bronze.lrnf000r21a_e AS road
|
||||
LEFT JOIN silver.csd_2021 AS left_side
|
||||
ON road.csddguid_l = left_side.csd_dguid
|
||||
LEFT JOIN silver.csd_2021 AS right_side
|
||||
ON road.csddguid_r = right_side.csd_dguid;
|
||||
|
||||
-- Create spatial index
|
||||
CREATE INDEX road_2021_geom_idx ON
|
||||
silver.road_2021
|
||||
USING gist (geom) WITH (fillfactor = 100);
|
||||
Reference in New Issue
Block a user