d4c-datapkg-statistical/boundaries/export_2021.ipynb

In [8]:

DATA_FOLDER=/data

source ../.env

1. Export Digital Boundary Files

In [9]:

output_folder="${DATA_FOLDER}/boundaries/output/2021/digital_boundary_files"

In [10]:

files=('country_2021_digital'
       'grc_2021_digital'
       'pr_2021_digital'
       'er_2021_digital'
       'car_2021_digital'
       'cd_2021_digital'
       'ccs_2021_digital'
       'cma_2021_digital'
       'csd_2021_digital'
       'fed_2021_2013_digital'
       'dpl_2021_digital'
       'fsa_2021_digital'
       'pop_ctr_2021_digital'
       'ct_2021_digital'
       'da_2021_digital'
       'db_2021_digital'
       'ada_2021_digital'
       )

Export as Parquet

In [11]:

for file in ${files[@]}
do
    filename="${file/_digital/}.parquet"
    output_file="${output_folder}/${filename}"
    echo "Exporting silver.${file} table to ${output_file}" 
    ogr2ogr \
        -lco COMPRESSION="ZSTD" \
        -lco CREATOR="www.dataforcanada.org" \
        -lco WRITE_COVERING_BBOX="YES" \
        -lco SORT_BY_BBOX="YES" \
        -f Parquet \
        ${output_file} \
        "PG:host=db dbname=${POSTGRES_DB} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} port=5432" \
        "silver.${file}"
done

Exporting silver.country_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/country_2021.parquet
Exporting silver.grc_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/grc_2021.parquet
Exporting silver.pr_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/pr_2021.parquet
Exporting silver.er_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/er_2021.parquet
Exporting silver.car_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/car_2021.parquet
Exporting silver.cd_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/cd_2021.parquet
Exporting silver.ccs_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/ccs_2021.parquet
Exporting silver.cma_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/cma_2021.parquet
Exporting silver.csd_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/csd_2021.parquet
Exporting silver.fed_2021_2013_digital table to /data/boundaries/output/2021/digital_boundary_files/fed_2021_2013.parquet
Exporting silver.dpl_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/dpl_2021.parquet
Exporting silver.fsa_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/fsa_2021.parquet
Exporting silver.pop_ctr_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/pop_ctr_2021.parquet
Exporting silver.ct_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/ct_2021.parquet
Exporting silver.da_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/da_2021.parquet
Exporting silver.db_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/db_2021.parquet
Exporting silver.ada_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/ada_2021.parquet

2. Export Cartographic Boundary Files

In [13]:

output_folder="${DATA_FOLDER}/boundaries/output/2021/cartographic_boundary_files"

In [14]:

files=('country_2021_cartographic'
       'grc_2021_cartographic'
       'pr_2021_cartographic'
       'er_2021_cartographic'
       'car_2021_cartographic'
       'cd_2021_cartographic'
       'ccs_2021_cartographic'
       'cma_2021_cartographic'
       'csd_2021_cartographic'
       'fed_2021_2013_cartographic'
       'dpl_2021_cartographic'
       'fsa_2021_cartographic'
       'pop_ctr_2021_cartographic'
       'ct_2021_cartographic'
       'da_2021_cartographic'
       'db_2021_cartographic'
       'ada_2021_cartographic'
       )

In [15]:

for file in ${files[@]}
do
    filename="${file/_cartographic/}.parquet"
    output_file="${output_folder}/${filename}"
    echo "Exporting silver.${file} table to ${output_file}" 
    ogr2ogr \
        -lco COMPRESSION="ZSTD" \
        -lco CREATOR="www.dataforcanada.org" \
        -lco WRITE_COVERING_BBOX="YES" \
        -lco SORT_BY_BBOX="YES" \
        -f Parquet \
        ${output_file} \
        "PG:host=db dbname=${POSTGRES_DB} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} port=5432" \
        "silver.${file}"
done

Exporting silver.country_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/country_2021.parquet
Exporting silver.grc_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/grc_2021.parquet
Exporting silver.pr_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/pr_2021.parquet
Exporting silver.er_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/er_2021.parquet
Exporting silver.car_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/car_2021.parquet
Exporting silver.cd_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/cd_2021.parquet
Exporting silver.ccs_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/ccs_2021.parquet
Exporting silver.cma_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/cma_2021.parquet
Exporting silver.csd_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/csd_2021.parquet
Exporting silver.fed_2021_2013_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/fed_2021_2013.parquet
Exporting silver.dpl_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/dpl_2021.parquet
Exporting silver.fsa_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/fsa_2021.parquet
Exporting silver.pop_ctr_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/pop_ctr_2021.parquet
Exporting silver.ct_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/ct_2021.parquet
Exporting silver.da_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/da_2021.parquet
Exporting silver.db_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/db_2021.parquet
Exporting silver.ada_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/ada_2021.parquet