From fad884efc87a93becebf67e08ee6ee382da7d4cc Mon Sep 17 00:00:00 2001 From: Diego Ripley Date: Wed, 4 Jun 2025 16:36:45 +0000 Subject: [PATCH] Create 2021 cartographic boundary files --- boundaries/aggregate_dissemination_areas.sql | 61 +- boundaries/census_agricultural_regions.sql | 52 +- .../census_consolidated_subdivisions.sql | 57 +- boundaries/census_divisions.sql | 56 +- boundaries/census_metropolitan_areas.sql | 58 +- boundaries/census_subdivisions.sql | 73 +- boundaries/census_tracts.sql | 53 +- boundaries/country.sql | 164 ++- boundaries/designated_places.sql | 51 +- boundaries/dissemination_areas.sql | 70 +- boundaries/dissemination_blocks.sql | 77 +- boundaries/economic_regions.sql | 51 +- boundaries/federal_electoral_districts.sql | 53 +- boundaries/forward_sortation_areas.sql | 49 +- boundaries/geographic_regions_of_canada.sql | 286 +++++- boundaries/population_centres.sql | 53 +- boundaries/provinces_and_territories.sql | 94 +- experiments/export_2021_boundaries.ipynb | 937 +++--------------- 18 files changed, 1324 insertions(+), 971 deletions(-) diff --git a/boundaries/aggregate_dissemination_areas.sql b/boundaries/aggregate_dissemination_areas.sql index 61cead5..49a1f12 100644 --- a/boundaries/aggregate_dissemination_areas.sql +++ b/boundaries/aggregate_dissemination_areas.sql @@ -6,8 +6,9 @@ Aggregate Dissemination Areas Definition here: https://web.archive.org/web/20240731061904/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/Definition-eng.cfm?ID=geo053 */ -DROP TABLE IF EXISTS silver.ada_2021; -CREATE TABLE silver.ada_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.ada_2021_digital; +CREATE TABLE silver.ada_2021_digital AS SELECT DISTINCT cd.country_dguid, cd.country_en_name, @@ -36,9 +37,9 @@ SELECT DISTINCT dgr.ada_dguid, ada.geom FROM silver.dissemination_geographies_relationship_2021 AS dgr -LEFT JOIN silver.cd_2021 AS cd +LEFT JOIN silver.cd_2021_digital AS cd ON dgr.cd_dguid = cd.cd_dguid -LEFT JOIN silver.cma_2021 AS cma +LEFT JOIN silver.cma_2021_digital AS cma ON concat(dgr.cma_dguid, dgr.cma_p_dguid) = concat(cma.cma_dguid, cma.cma_p_dguid) @@ -47,13 +48,61 @@ LEFT JOIN bronze.lada000a21a_e AS ada -- Make geometries valid UPDATE - silver.ada_2021 + silver.ada_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX ada_2021_geom_idx ON silver.ada_2021 USING gist (geom) WITH ( +CREATE INDEX ada_2021_digital_geom_idx ON silver.ada_2021_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.ada_2021_cartographic; +CREATE TABLE silver.ada_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.car_dguid, + b.car_en_name, + b.car_fr_name, + b.cd_dguid, + b.cd_name, + b.cd_type, + b.cma_dguid, + b.cma_p_dguid, + b.cma_name, + b.cma_type, + b.ada_dguid, + a.geom +FROM + bronze.lada000b21a_e AS a, + silver.ada_2021_digital AS b +WHERE a.dguid = b.ada_dguid; + +-- Make geometries valid +UPDATE + silver.ada_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX ada_2021_cartographic_geom_idx ON silver.ada_2021_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/census_agricultural_regions.sql b/boundaries/census_agricultural_regions.sql index 0f4ec33..11b9d71 100644 --- a/boundaries/census_agricultural_regions.sql +++ b/boundaries/census_agricultural_regions.sql @@ -6,8 +6,9 @@ Census Agricultural Regions Definition here: https://web.archive.org/web/20250401192328/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/definition-eng.cfm?ID=geo006 */ -DROP TABLE IF EXISTS silver.car_2021; -CREATE TABLE silver.car_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.car_2021_digital; +CREATE TABLE silver.car_2021_digital AS SELECT DISTINCT pr.country_dguid, pr.country_en_name, @@ -28,7 +29,7 @@ SELECT DISTINCT car.carfname AS car_fr_name, car.geom FROM - silver.pr_2021 AS pr, + silver.pr_2021_digital AS pr, silver.dissemination_geographies_relationship_2021 AS dgr, bronze.lcar000a21a_e AS car WHERE @@ -37,13 +38,54 @@ WHERE -- Make geometries valid UPDATE - silver.car_2021 + silver.car_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX car_2021_geom_idx ON silver.car_2021 USING gist (geom) WITH ( +CREATE INDEX car_2021_digital_geom_idx ON silver.car_2021_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.car_2021_cartographic; +CREATE TABLE silver.car_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.car_dguid, + b.car_en_name, + b.car_fr_name, + a.geom +FROM + bronze.lcar000b21a_e AS a, + silver.car_2021_digital AS b +WHERE + a.dguid = b.car_dguid; + +-- Make geometries valid +UPDATE + silver.car_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX car_2021_cartographic_geom_idx ON silver.car_2021_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/census_consolidated_subdivisions.sql b/boundaries/census_consolidated_subdivisions.sql index 3bb8d63..09d5514 100644 --- a/boundaries/census_consolidated_subdivisions.sql +++ b/boundaries/census_consolidated_subdivisions.sql @@ -6,8 +6,9 @@ Census Consolidated Subdivisions Definition here: https://web.archive.org/web/20250401192303/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/Definition-eng.cfm?ID=geo007 */ -DROP TABLE IF EXISTS silver.ccs_2021; -CREATE TABLE silver.ccs_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.ccs_2021_digital; +CREATE TABLE silver.ccs_2021_digital AS SELECT DISTINCT cd.country_dguid, cd.country_en_name, @@ -33,7 +34,7 @@ SELECT DISTINCT ccs.ccsname AS ccs_name, ccs.geom FROM - silver.cd_2021 AS cd, + silver.cd_2021_digital AS cd, silver.dissemination_geographies_relationship_2021 AS dgr, bronze.lccs000a21a_e AS ccs WHERE @@ -43,13 +44,59 @@ WHERE -- Make geometries valid UPDATE - silver.ccs_2021 + silver.ccs_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX ccs_2021_geom_idx ON silver.ccs_2021 USING gist (geom) WITH ( +CREATE INDEX ccs_2021_digital_geom_idx ON silver.ccs_2021_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.ccs_2021_cartographic; +CREATE TABLE silver.ccs_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.car_dguid, + b.car_en_name, + b.car_fr_name, + b.cd_dguid, + b.cd_name, + b.cd_type, + b.ccs_dguid, + b.ccs_name, + a.geom +FROM + bronze.lccs000b21a_e AS a, + silver.ccs_2021_digital AS b +WHERE + a.dguid = b.ccs_dguid; + +-- Make geometries valid +UPDATE + silver.ccs_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX ccs_2021_cartographic_geom_idx ON silver.ccs_2021_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/census_divisions.sql b/boundaries/census_divisions.sql index 795a5a9..ace7c75 100644 --- a/boundaries/census_divisions.sql +++ b/boundaries/census_divisions.sql @@ -6,8 +6,9 @@ Census Divisions Definition here: https://web.archive.org/web/20250131082459/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/definition-eng.cfm?ID=geo008#moreinfo */ -DROP TABLE IF EXISTS silver.cd_2021; -CREATE TABLE silver.cd_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.cd_2021_digital; +CREATE TABLE silver.cd_2021_digital AS SELECT DISTINCT pr.country_dguid, pr.country_en_name, @@ -31,7 +32,7 @@ SELECT DISTINCT cd.cdtype AS cd_type, cd.geom FROM - silver.pr_2021 AS pr, + silver.pr_2021_digital AS pr, bronze.lcd_000a21a_e AS cd, silver.dissemination_geographies_relationship_2021 AS dgr, bronze.lcar000a21a_e AS car @@ -42,16 +43,59 @@ WHERE -- Make geometries valid UPDATE - silver.cd_2021 + silver.cd_2021_digital SET geom = ST_MAKEVALID(geom) WHERE ST_ISVALID(geom) = 'f'; -- Create spatial index -CREATE INDEX cd_2021_geom_idx ON silver.cd_2021 +CREATE INDEX cd_2021_digital_geom_idx ON silver.cd_2021_digital +USING gist (geom) WITH (fillfactor = 100); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.cd_2021_cartographic; +CREATE TABLE silver.cd_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.car_dguid, + b.car_en_name, + b.car_fr_name, + b.cd_dguid, + b.cd_name, + b.cd_type, + a.geom +FROM + bronze.lcd_000b21a_e AS a, + silver.cd_2021_digital AS b +WHERE + a.dguid = b.cd_dguid; + +-- Make geometries valid +UPDATE + silver.cd_2021_cartographic +SET + geom = ST_MAKEVALID(geom) +WHERE + ST_ISVALID(geom) = 'f'; + +-- Create spatial index +CREATE INDEX cd_2021_cartographic_geom_idx ON silver.cd_2021_cartographic USING gist (geom) WITH (fillfactor = 100); /* 2016 Definition here: https://web.archive.org/web/20250304001456/https://www12.statcan.gc.ca/census-recensement/2016/ref/dict/geo008-eng.cfm -*/ +*/ \ No newline at end of file diff --git a/boundaries/census_metropolitan_areas.sql b/boundaries/census_metropolitan_areas.sql index d0c9672..2ad835c 100644 --- a/boundaries/census_metropolitan_areas.sql +++ b/boundaries/census_metropolitan_areas.sql @@ -6,8 +6,9 @@ Census Metropolitan Areas Definition here: https://web.archive.org/web/20250518133322/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/Definition-eng.cfm?ID=geo009 */ -DROP TABLE IF EXISTS silver.cma_2021; -CREATE TABLE silver.cma_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.cma_2021_digital; +CREATE TABLE silver.cma_2021_digital AS SELECT DISTINCT pr.country_dguid, pr.country_en_name, @@ -29,7 +30,7 @@ SELECT DISTINCT cma.cmatype AS cma_type, cma.geom FROM - silver.pr_2021 AS pr, + silver.pr_2021_digital AS pr, bronze.lcma000a21a_e AS cma, silver.dissemination_geographies_relationship_2021 AS dgr WHERE @@ -44,13 +45,60 @@ WHERE -- Make geometries valid UPDATE - silver.cma_2021 + silver.cma_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX cma_2021_geom_idx ON silver.cma_2021 USING gist (geom) WITH ( +CREATE INDEX cma_2021_digital_geom_idx ON silver.cma_2021_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.cma_2021_cartographic; +CREATE TABLE silver.cma_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.cma_dguid, + b.cma_p_dguid, + b.cma_name, + b.cma_type, + a.geom +FROM + bronze.lcma000b21a_e AS a, + silver.cma_2021_digital AS b +WHERE + concat( + a.dguid, + a.dguidp) = concat( + b.cma_dguid, + b.cma_p_dguid + ); + +-- Make geometries valid +UPDATE + silver.cma_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX cma_2021_cartographic_geom_idx ON silver.cma_2021_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/census_subdivisions.sql b/boundaries/census_subdivisions.sql index 7319a87..9af2dc5 100644 --- a/boundaries/census_subdivisions.sql +++ b/boundaries/census_subdivisions.sql @@ -6,8 +6,9 @@ Census Subdivisions Definition here: https://web.archive.org/web/20240526213705/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/Definition-eng.cfm?ID=geo012 */ -DROP TABLE IF EXISTS silver.csd_2021; -CREATE TABLE silver.csd_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.csd_2021_digital; +CREATE TABLE silver.csd_2021_digital AS SELECT DISTINCT cd.country_dguid, cd.country_en_name, @@ -44,28 +45,84 @@ SELECT DISTINCT gaf.sac_code, csd.geom FROM silver.gaf_2021 AS gaf -LEFT JOIN silver.cma_2021 AS cma +LEFT JOIN silver.cma_2021_digital AS cma ON concat(gaf.cma_dguid, gaf.cma_p_dguid) = concat(cma.cma_dguid, cma.cma_p_dguid) -LEFT JOIN silver.cd_2021 AS cd +LEFT JOIN silver.cd_2021_digital AS cd ON gaf.cd_dguid = cd.cd_dguid -LEFT JOIN silver.er_2021 AS er +LEFT JOIN silver.er_2021_digital AS er ON gaf.er_dguid = er.er_dguid -LEFT JOIN silver.ccs_2021 AS ccs +LEFT JOIN silver.ccs_2021_digital AS ccs ON gaf.ccs_dguid = ccs.ccs_dguid LEFT JOIN bronze.lcsd000a21a_e AS csd ON gaf.csd_dguid = csd.dguid; -- Make geometries valid UPDATE - silver.csd_2021 + silver.csd_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX csd_2021_geom_idx ON silver.csd_2021 USING gist (geom) WITH ( +CREATE INDEX csd_2021_digital_geom_idx ON silver.csd_2021_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.csd_2021_cartographic; +CREATE TABLE silver.csd_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.car_dguid, + b.car_en_name, + b.car_fr_name, + b.er_dguid, + b.er_name, + b.cd_dguid, + b.cd_name, + b.cd_type, + b.ccs_dguid, + b.ccs_name, + b.cma_dguid, + b.cma_p_dguid, + b.cma_name, + b.cma_type, + b.csd_dguid, + b.csd_name, + b.csd_type, + b.sac_type, + b.sac_code, + a.geom +FROM + bronze.lcsd000b21a_e AS a, + silver.csd_2021_digital AS b +WHERE a.dguid = b.csd_dguid; + +-- Make geometries valid +UPDATE + silver.csd_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX csd_2021_cartographic_geom_idx ON silver.csd_2021_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/census_tracts.sql b/boundaries/census_tracts.sql index 61ce0fd..d7b0192 100644 --- a/boundaries/census_tracts.sql +++ b/boundaries/census_tracts.sql @@ -6,8 +6,9 @@ Census Tracts Definition here: https://web.archive.org/web/20241013011815/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/Definition-eng.cfm?ID=geo013 */ -DROP TABLE IF EXISTS silver.ct_2021; -CREATE TABLE silver.ct_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.ct_2021_digital; +CREATE TABLE silver.ct_2021_digital AS SELECT DISTINCT cma.country_dguid, cma.country_en_name, @@ -32,7 +33,7 @@ SELECT DISTINCT FROM silver.dissemination_geographies_relationship_2021 AS dgr LEFT JOIN bronze.lct_000a21a_e AS ct ON dgr.ct_dguid = ct.dguid -LEFT JOIN silver.cma_2021 AS cma +LEFT JOIN silver.cma_2021_digital AS cma ON concat(dgr.cma_dguid, dgr.cma_p_dguid) = concat(cma.cma_dguid, cma.cma_p_dguid) @@ -40,13 +41,55 @@ WHERE dgr.ct_dguid IS NOT null; -- Make geometries valid UPDATE - silver.ct_2021 + silver.ct_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX ct_2021_geom_idx ON silver.ct_2021 USING gist (geom) WITH ( +CREATE INDEX ct_2021_digital_geom_idx ON silver.ct_2021_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.ct_2021_cartographic; +CREATE TABLE silver.ct_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.cma_dguid, + b.cma_p_dguid, + b.cma_name, + b.cma_type, + b.ct_dguid, + a.geom +FROM + bronze.lct_000b21a_e AS a, + silver.ct_2021_digital AS b +WHERE a.dguid = b.ct_dguid; + +-- Make geometries valid +UPDATE + silver.ct_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX ct_2021_cartographic_geom_idx ON silver.ct_2021_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/country.sql b/boundaries/country.sql index 5b67631..45b3e16 100644 --- a/boundaries/country.sql +++ b/boundaries/country.sql @@ -2,9 +2,9 @@ Canada */ --- 2021 Canada; -DROP TABLE IF EXISTS silver.country_2021; -CREATE TABLE silver.country_2021 AS +-- 2021 Canada digital boundary; +DROP TABLE IF EXISTS silver.country_2021_digital; +CREATE TABLE silver.country_2021_digital AS SELECT DISTINCT '2021A000011124' AS country_dguid, 'Canada' AS country_en_name, @@ -17,19 +17,44 @@ FROM -- Make geometries valid UPDATE - silver.country_2021 + silver.country_2021_digital SET geom = ST_MAKEVALID(geom) WHERE ST_ISVALID(geom) = 'f'; -- Create spatial index -CREATE INDEX country_2021_geom_idx ON silver.country_2021 +CREATE INDEX country_2021_digital_geom_idx ON silver.country_2021_digital USING gist (geom) WITH (fillfactor = 100); --- 2016 Canada; -DROP TABLE IF EXISTS silver.country_2016; -CREATE TABLE silver.country_2016 AS +-- 2021 Canada cartographic boundary; +DROP TABLE IF EXISTS silver.country_2021_cartographic; +CREATE TABLE silver.country_2021_cartographic AS +SELECT DISTINCT + '2021A000011124' AS country_dguid, + 'Canada' AS country_en_name, + 'Canada' AS country_fr_name, + 'CAN' AS country_en_abbreviation, + 'CAN' AS country_fr_abbreviation, + ST_UNION(geom) AS geom +FROM + bronze.lpr_000b21a_e; + +-- Make geometries valid +UPDATE + silver.country_2021_cartographic +SET + geom = ST_MAKEVALID(geom) +WHERE + ST_ISVALID(geom) = 'f'; + +-- Create spatial index +CREATE INDEX country_2021_cartographic_geom_idx ON silver.country_2021_cartographic +USING gist (geom) WITH (fillfactor = 100); + +-- 2016 Canada digital boundary; +DROP TABLE IF EXISTS silver.country_2016_digital; +CREATE TABLE silver.country_2016_digital AS SELECT DISTINCT '2016A000011124' AS country_dguid, 'Canada' AS country_en_name, @@ -42,44 +67,94 @@ FROM -- Make geometries valid UPDATE - silver.country_2016 + silver.country_2016_digital SET geom = ST_MAKEVALID(geom) WHERE ST_ISVALID(geom) = 'f'; -- Create spatial index -CREATE INDEX country_2016_geom_idx ON silver.country_2016 +CREATE INDEX country_2016_digital_geom_idx ON silver.country_2016_digital USING gist (geom) WITH (fillfactor = 100); --- 2011 Canada; -DROP TABLE IF EXISTS silver.country_2011; -CREATE TABLE silver.country_2011 AS +-- 2016 Canada cartographic boundary; +DROP TABLE IF EXISTS silver.country_2016_cartographic; +CREATE TABLE silver.country_2016_cartographic AS +SELECT DISTINCT + '2016A000011124' AS country_dguid, + 'Canada' AS country_en_name, + 'Canada' AS country_fr_name, + 'CAN' AS country_en_abbreviation, + 'CAN' AS country_fr_abbreviation, + ST_UNION(geom) AS geom +FROM + bronze.lpr_000b16a_e; + +-- Make geometries valid +UPDATE + silver.country_2016_cartographic +SET + geom = ST_MAKEVALID(geom) +WHERE + ST_ISVALID(geom) = 'f'; + +-- Create spatial index +CREATE INDEX country_2016_cartographic_geom_idx ON silver.country_2016_cartographic +USING gist (geom) WITH (fillfactor = 100); + +-- 2011 Canada digital boundary; +DROP TABLE IF EXISTS silver.country_2011_digital; +CREATE TABLE silver.country_2011_digital AS SELECT DISTINCT '2011A000011124' AS country_dguid, 'Canada' AS country_en_name, 'Canada' AS country_fr_name, 'CAN' AS country_en_abbreviation, 'CAN' AS country_fr_abbreviation, - st_union(geom) AS geom + ST_UNION(geom) AS geom FROM bronze.gpr_000a11a_e; -- Make geometries valid UPDATE - silver.country_2011 + silver.country_2011_digital SET geom = ST_MAKEVALID(geom) WHERE ST_ISVALID(geom) = 'f'; -- Create spatial index -CREATE INDEX country_2011_geom_idx ON silver.country_2011 +CREATE INDEX country_2011_digital_geom_idx ON silver.country_2011_digital USING gist (geom) WITH (fillfactor = 100); --- 2006 Canada; -DROP TABLE IF EXISTS silver.country_2006; -CREATE TABLE silver.country_2006 AS +-- 2011 Canada cartographic boundary; +DROP TABLE IF EXISTS silver.country_2011_cartographic; +CREATE TABLE silver.country_2011_cartographic AS +SELECT DISTINCT + '2011A000011124' AS country_dguid, + 'Canada' AS country_en_name, + 'Canada' AS country_fr_name, + 'CAN' AS country_en_abbreviation, + 'CAN' AS country_fr_abbreviation, + ST_UNION(geom) AS geom +FROM + bronze.gpr_000b11a_e; + +-- Make geometries valid +UPDATE + silver.country_2011_cartographic +SET + geom = ST_MAKEVALID(geom) +WHERE + ST_ISVALID(geom) = 'f'; + +-- Create spatial index +CREATE INDEX country_2011_cartographic_geom_idx ON silver.country_2011_cartographic +USING gist (geom) WITH (fillfactor = 100); + +-- 2006 Canada digital boundary; +DROP TABLE IF EXISTS silver.country_2006_digital; +CREATE TABLE silver.country_2006_digital AS SELECT DISTINCT '2006A000011124' AS country_dguid, 'Canada' AS country_en_name, @@ -92,40 +167,37 @@ FROM -- Make geometries valid UPDATE - silver.country_2006 + silver.country_2006_digital SET geom = ST_MAKEVALID(geom) WHERE ST_ISVALID(geom) = 'f'; -- Create spatial index -CREATE INDEX country_2006_geom_idx ON silver.country_2006 +CREATE INDEX country_2006_digital_geom_idx ON silver.country_2006_digital USING gist (geom) WITH (fillfactor = 100); --- 2001 Canada; --- TODO -/* --- Clean Provinces and Territories layer; -UPDATE - bronze.gpr_000a06a_e -SET - geom = st_makevalid(geom) -WHERE - st_isvalid(geom) = 'f'; -DROP TABLE IF EXISTS country_2001; - -CREATE TABLE country_2001 AS - SELECT - DISTINCT '2001A000011124' AS country_dguid, - 'Canada' AS country_en_name, - 'Canada' AS country_fr_name, - 'CAN' AS country_en_abbreviation, - 'CAN' AS country_fr_abbreviation, - st_union(geom) AS geom +-- 2006 Canada cartographic boundary; +DROP TABLE IF EXISTS silver.country_2006_cartographic; +CREATE TABLE silver.country_2006_cartographic AS +SELECT DISTINCT + '2006A000011124' AS country_dguid, + 'Canada' AS country_en_name, + 'Canada' AS country_fr_name, + 'CAN' AS country_en_abbreviation, + 'CAN' AS country_fr_abbreviation, + ST_UNION(geom) AS geom FROM - lpr_000a21a_e; + bronze.gpr_000b06a_e; -CREATE INDEX country_2001_geom_idx ON -country_2001 - USING GIST(geom) WITH (FILLFACTOR = 100); -*/ +-- Make geometries valid +UPDATE + silver.country_2006_cartographic +SET + geom = ST_MAKEVALID(geom) +WHERE + ST_ISVALID(geom) = 'f'; + +-- Create spatial index +CREATE INDEX country_2006_cartographic_geom_idx ON silver.country_2006_cartographic +USING gist (geom) WITH (fillfactor = 100); \ No newline at end of file diff --git a/boundaries/designated_places.sql b/boundaries/designated_places.sql index acf0ab7..bd98239 100644 --- a/boundaries/designated_places.sql +++ b/boundaries/designated_places.sql @@ -6,8 +6,9 @@ Designated Places Definition here: https://web.archive.org/web/20240731061904/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/Definition-eng.cfm?ID=geo018 */ -DROP TABLE IF EXISTS silver.dpl_2021; -CREATE TABLE silver.dpl_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.dpl_2021_digital; +CREATE TABLE silver.dpl_2021_digital AS SELECT DISTINCT pr.country_dguid, pr.country_en_name, @@ -28,7 +29,7 @@ SELECT DISTINCT dpl.dpltype AS dpl_type, dpl.geom FROM silver.dissemination_geographies_relationship_2021 AS dgr -LEFT JOIN silver.pr_2021 AS pr +LEFT JOIN silver.pr_2021_digital AS pr ON dgr.pr_dguid = pr.pr_dguid LEFT JOIN bronze.ldpl000a21a_e AS dpl ON dgr.dpl_dguid = dpl.dguid @@ -36,13 +37,53 @@ WHERE dgr.dpl_dguid IS NOT null; -- Make geometries valid UPDATE - silver.dpl_2021 + silver.dpl_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX dpl_2021_geom_idx ON silver.dpl_2021 USING gist (geom) WITH ( +CREATE INDEX dpl_2021_digital_geom_idx ON silver.dpl_2021_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.dpl_2021_cartographic; +CREATE TABLE silver.dpl_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.dpl_dguid, + b.dpl_name, + b.dpl_type, + a.geom +FROM + bronze.ldpl000b21a_e AS a, + silver.dpl_2021_digital AS b +WHERE b.dpl_dguid = a.dguid; + +-- Make geometries valid +UPDATE + silver.dpl_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX dpl_2021_cartographic_geom_idx ON silver.dpl_2021_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/dissemination_areas.sql b/boundaries/dissemination_areas.sql index 62b2377..8847d94 100644 --- a/boundaries/dissemination_areas.sql +++ b/boundaries/dissemination_areas.sql @@ -6,8 +6,9 @@ Dissemination Areas Definition here: https://web.archive.org/web/20240731061905/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/definition-eng.cfm?ID=geo021 */ -DROP TABLE IF EXISTS silver.da_2021; -CREATE TABLE silver.da_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.da_2021_digital; +CREATE TABLE silver.da_2021_digital AS SELECT DISTINCT csd.country_dguid, csd.country_en_name, @@ -47,20 +48,79 @@ SELECT DISTINCT dgr.da_dguid, da.geom FROM silver.dissemination_geographies_relationship_2021 AS dgr -LEFT JOIN silver.csd_2021 AS csd +LEFT JOIN silver.csd_2021_digital AS csd ON dgr.csd_dguid = csd.csd_dguid LEFT JOIN bronze.lda_000a21a_e AS da ON dgr.da_dguid = da.dguid; -- Make geometries valid UPDATE - bronze.lda_000a21a_e + silver.da_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX da_2021_geom_idx ON silver.da_2021 USING gist (geom) WITH ( +CREATE INDEX da_2021_digital_geom_idx ON silver.da_2021_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.da_2021_cartographic; +CREATE TABLE silver.da_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.car_dguid, + b.car_en_name, + b.car_fr_name, + b.er_dguid, + b.er_name, + b.cd_dguid, + b.cd_name, + b.cd_type, + b.ccs_dguid, + b.ccs_name, + b.cma_dguid, + b.cma_p_dguid, + b.cma_name, + b.cma_type, + b.csd_dguid, + b.csd_name, + b.csd_type, + b.sac_type, + b.sac_code, + b.ct_dguid, + b.ada_dguid, + b.da_dguid, + a.geom +FROM + bronze.lda_000b21a_e AS a, + silver.da_2021_digital AS b +WHERE a.dguid = b.da_dguid; + +-- Make geometries valid +UPDATE + silver.da_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX da_2021_cartographic_geom_idx ON silver.da_2021_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/dissemination_blocks.sql b/boundaries/dissemination_blocks.sql index 7ba3d4e..2e790ca 100644 --- a/boundaries/dissemination_blocks.sql +++ b/boundaries/dissemination_blocks.sql @@ -6,8 +6,9 @@ Dissemination Blocks Definition here: https://web.archive.org/web/20250212081621/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/definition-eng.cfm?ID=geo014 */ -DROP TABLE IF EXISTS silver.db_2021; -CREATE TABLE silver.db_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.db_2021_digital; +CREATE TABLE silver.db_2021_digital AS SELECT DISTINCT csd.country_dguid, csd.country_en_name, @@ -52,22 +53,86 @@ SELECT DISTINCT dgr.db_dguid, db.geom FROM silver.dissemination_geographies_relationship_2021 AS dgr -LEFT JOIN silver.csd_2021 AS csd +LEFT JOIN silver.csd_2021_digital AS csd ON dgr.csd_dguid = csd.csd_dguid -LEFT JOIN silver.fed_2021 AS fed +LEFT JOIN silver.fed_2021_2013_digital AS fed ON dgr.fed_dguid = fed.fed_dguid LEFT JOIN bronze.ldb_000a21a_e AS db ON dgr.db_dguid = db.dguid; -- Make geometries valid UPDATE - silver.db_2021 + silver.db_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX db_2021_geom_idx ON silver.db_2021 USING gist (geom) WITH ( +CREATE INDEX db_2021_digital_geom_idx ON silver.db_2021_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.db_2021_cartographic; +CREATE TABLE silver.db_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.car_dguid, + b.car_en_name, + b.car_fr_name, + b.er_dguid, + b.er_name, + b.cd_dguid, + b.cd_name, + b.cd_type, + b.ccs_dguid, + b.ccs_name, + b.cma_dguid, + b.cma_p_dguid, + b.cma_name, + b.cma_type, + b.csd_dguid, + b.csd_name, + b.csd_type, + b.sac_type, + b.sac_code, + b.fed_dguid, + b.fed_name, + b.fed_en_name, + b.fed_fr_name, + b.ct_dguid, + b.ada_dguid, + b.da_dguid, + b.db_dguid, + a.geom +FROM + bronze.ldb_000b21a_e AS a, + silver.db_2021_digital AS b +WHERE b.db_dguid = a.dguid; + +-- Make geometries valid +UPDATE + silver.db_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX db_2021_cartographic_geom_idx ON silver.db_2021_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/economic_regions.sql b/boundaries/economic_regions.sql index 27a16fc..b4535d6 100644 --- a/boundaries/economic_regions.sql +++ b/boundaries/economic_regions.sql @@ -6,8 +6,9 @@ Economic Regions Definition here: https://web.archive.org/web/20250518132130/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/Definition-eng.cfm?ID=geo022 */ -DROP TABLE IF EXISTS silver.er_2021; -CREATE TABLE silver.er_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.er_2021_digital; +CREATE TABLE silver.er_2021_digital AS SELECT DISTINCT pr.country_dguid, pr.country_en_name, @@ -27,7 +28,7 @@ SELECT DISTINCT er.ername AS er_name, er.geom FROM - silver.pr_2021 AS pr, + silver.pr_2021_digital AS pr, silver.dissemination_geographies_relationship_2021 AS dgr, bronze.ler_000a21a_e AS er WHERE @@ -37,13 +38,53 @@ WHERE -- Make geometries valid UPDATE - silver.er_2021 + silver.er_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX er_2021_geom_idx ON silver.er_2021 USING gist (geom) WITH ( +CREATE INDEX er_2021_digital_geom_idx ON silver.er_2021_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.er_2021_cartographic; +CREATE TABLE silver.er_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.er_dguid, + b.er_name, + a.geom +FROM + bronze.ler_000b21a_e AS a, + silver.er_2021_digital AS b +WHERE + a.dguid = b.er_dguid; + +-- Make geometries valid +UPDATE + silver.er_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX er_2021_cartographic_geom_idx ON silver.er_2021_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/federal_electoral_districts.sql b/boundaries/federal_electoral_districts.sql index 12cf509..18f9c29 100644 --- a/boundaries/federal_electoral_districts.sql +++ b/boundaries/federal_electoral_districts.sql @@ -6,9 +6,9 @@ Federal Electoral Districts Definition here: https://web.archive.org/web/20240731061905/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/Definition-eng.cfm?ID=geo025 */ --- 2021 vintage, 2013 representation order; -DROP TABLE IF EXISTS silver.fed_2021_2013; -CREATE TABLE silver.fed_2021_2013 AS +-- 2021 vintage, 2013 representation order. Digital boundary; +DROP TABLE IF EXISTS silver.fed_2021_2013_digital; +CREATE TABLE silver.fed_2021_2013_digital AS SELECT DISTINCT pr.country_dguid, pr.country_en_name, @@ -30,7 +30,7 @@ SELECT DISTINCT fed.fedfname AS fed_fr_name, fed.geom FROM silver.dissemination_geographies_relationship_2021 AS dgr -LEFT JOIN silver.pr_2021 AS pr +LEFT JOIN silver.pr_2021_digital AS pr ON dgr.pr_dguid = pr.pr_dguid LEFT JOIN bronze.lfed000a21a_e AS fed ON dgr.fed_dguid = fed.dguid @@ -38,13 +38,54 @@ WHERE dgr.fed_dguid IS NOT null; -- Make geometries valid UPDATE - silver.fed_2021_2013 + silver.fed_2021_2013_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX fed_2021_2013_geom_idx ON silver.fed_2021_2013 USING gist (geom) WITH ( +CREATE INDEX fed_2021_2013_digital_geom_idx ON silver.fed_2021_2013_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- 2021 vintage, 2013 representation order. Cartographic boundary; +DROP TABLE IF EXISTS silver.fed_2021_2013_cartographic; +CREATE TABLE silver.fed_2021_2013_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.fed_dguid, + b.fed_name, + b.fed_en_name, + b.fed_fr_name, + a.geom +FROM + bronze.lfed000b21a_e AS a, + silver.fed_2021_2013_digital AS b +WHERE a.dguid = b.fed_dguid; + +-- Make geometries valid +UPDATE + silver.fed_2021_2013_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX fed_2021_2013_cartographic_geom_idx ON silver.fed_2021_2013_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/forward_sortation_areas.sql b/boundaries/forward_sortation_areas.sql index ef05844..bfa012f 100644 --- a/boundaries/forward_sortation_areas.sql +++ b/boundaries/forward_sortation_areas.sql @@ -6,8 +6,9 @@ Forward Sortation Areas Definition here: https://web.archive.org/web/20241102112247/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/Definition-eng.cfm?ID=geo036 */ -DROP TABLE IF EXISTS silver.fsa_2021; -CREATE TABLE silver.fsa_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.fsa_2021_digital; +CREATE TABLE silver.fsa_2021_digital AS SELECT DISTINCT pr.country_dguid, pr.country_en_name, @@ -26,18 +27,56 @@ SELECT DISTINCT fsa.dguid AS fsa_dguid, fsa.geom FROM bronze.lfsa000a21a_e AS fsa, - silver.pr_2021 AS pr + silver.pr_2021_digital AS pr WHERE concat('2021A0002', fsa.pruid) = pr.pr_dguid; -- Make geometries valid UPDATE - silver.fsa_2021 + silver.fsa_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX fsa_2021_geom_idx ON silver.fsa_2021 USING gist (geom) WITH ( +CREATE INDEX fsa_2021_digital_geom_idx ON silver.fsa_2021_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.fsa_2021_cartographic; +CREATE TABLE silver.fsa_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.fsa_dguid, + a.geom +FROM + bronze.lfsa000b21a_e AS a, + silver.fsa_2021_digital AS b +WHERE a.dguid = b.fsa_dguid; + +-- Make geometries valid +UPDATE + silver.fsa_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX fsa_2021_cartographic_geom_idx ON silver.fsa_2021_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/boundaries/geographic_regions_of_canada.sql b/boundaries/geographic_regions_of_canada.sql index c31fb0d..a4e21aa 100644 --- a/boundaries/geographic_regions_of_canada.sql +++ b/boundaries/geographic_regions_of_canada.sql @@ -6,9 +6,9 @@ Geographic Regions of Canada Definition here: https://web.archive.org/web/20240624230708/https://www150.statcan.gc.ca/n1/pub/92-195-x/2021001/geo/region/region-eng.htm */ --- With geometries; -DROP TABLE IF EXISTS silver.grc_2021; -CREATE TABLE silver.grc_2021 AS +-- With geometries. Digital boundary; +DROP TABLE IF EXISTS silver.grc_2021_digital; +CREATE TABLE silver.grc_2021_digital AS WITH territories AS ( SELECT '2021A00016' AS grc_dguid, @@ -71,7 +71,7 @@ final AS ( territories.* FROM territories, - silver.country_2021 AS country + silver.country_2021_digital AS country UNION SELECT country.country_dguid, @@ -82,7 +82,7 @@ final AS ( prairies.* FROM prairies, - silver.country_2021 AS country + silver.country_2021_digital AS country UNION SELECT country.country_dguid, @@ -93,7 +93,7 @@ final AS ( atlantic.* FROM atlantic, - silver.country_2021 AS country + silver.country_2021_digital AS country UNION SELECT country.country_dguid, @@ -104,7 +104,7 @@ final AS ( the_rest.* FROM the_rest, - silver.country_2021 AS country + silver.country_2021_digital AS country ) SELECT * @@ -113,14 +113,131 @@ FROM -- Make geometries valid UPDATE - silver.grc_2021 + silver.grc_2021_digital SET geom = ST_MAKEVALID(geom) WHERE ST_ISVALID(geom) = 'f'; -- Create spatial index -CREATE INDEX grc_2021_geom_idx ON silver.grc_2021 +CREATE INDEX grc_2021_digital_geom_idx ON silver.grc_2021_digital +USING gist (geom) WITH (fillfactor = 100); + +-- With geometries. Cartographic boundary; +DROP TABLE IF EXISTS silver.grc_2021_cartographic; +CREATE TABLE silver.grc_2021_cartographic AS +WITH territories AS ( + SELECT + '2021A00016' AS grc_dguid, + 'Territories' AS grc_en_name, + 'Territoires' AS grc_fr_name, + ST_UNION(geom) AS geom + FROM + bronze.lpr_000b21a_e + WHERE + pruid IN ('60', '61', '62') +), + +atlantic AS ( + SELECT + '2021A00011' AS grc_dguid, + 'Atlantic' AS grc_en_name, + 'Atlantique' AS grc_fr_name, + ST_UNION(geom) AS geom + FROM + bronze.lpr_000b21a_e + WHERE + pruid IN ('10', '11', '12', '13') +), + +prairies AS ( + SELECT + '2021A00014' AS grc_dguid, + 'Prairies' AS grc_en_name, + 'Prairies' AS grc_fr_name, + ST_UNION(geom) AS geom + FROM + bronze.lpr_000b21a_e + WHERE + pruid IN ('48', '47', '46') +), + +the_rest AS ( + SELECT + CASE + WHEN pruid = '59' THEN '2021A00015' + WHEN pruid = '35' THEN '2021A00013' + WHEN pruid = '24' THEN '2021A00012' + END AS grc_dguid, + prename AS grc_en_name, + prfname AS grc_fr_name, + geom + FROM + bronze.lpr_000b21a_e + WHERE + pruid IN ('59', '35', '24') +), + +final AS ( + SELECT + country.country_dguid, + country.country_en_name, + country.country_fr_name, + country.country_en_abbreviation, + country.country_fr_abbreviation, + territories.* + FROM + territories, + silver.country_2021_digital AS country + UNION + SELECT + country.country_dguid, + country.country_en_name, + country.country_fr_name, + country.country_en_abbreviation, + country.country_fr_abbreviation, + prairies.* + FROM + prairies, + silver.country_2021_digital AS country + UNION + SELECT + country.country_dguid, + country.country_en_name, + country.country_fr_name, + country.country_en_abbreviation, + country.country_fr_abbreviation, + atlantic.* + FROM + atlantic, + silver.country_2021_digital AS country + UNION + SELECT + country.country_dguid, + country.country_en_name, + country.country_fr_name, + country.country_en_abbreviation, + country.country_fr_abbreviation, + the_rest.* + FROM + the_rest, + silver.country_2021_digital AS country +) + +SELECT * +FROM + final; + +-- Make geometries valid +UPDATE + silver.grc_2021_cartographic +SET + geom = ST_MAKEVALID(geom) +WHERE + ST_ISVALID(geom) = 'f'; + +-- Create spatial index +CREATE INDEX grc_2021_cartographic_geom_idx ON silver.grc_2021_cartographic USING gist (geom) WITH (fillfactor = 100); -- 2021 without geometries, and with pr_dguid; @@ -188,7 +305,7 @@ final AS ( territories.* FROM territories, - silver.country_2021 AS country + silver.country_2021_digital AS country UNION SELECT country.country_dguid, @@ -199,7 +316,7 @@ final AS ( prairies.* FROM prairies, - silver.country_2021 AS country + silver.country_2021_digital AS country UNION SELECT country.country_dguid, @@ -210,7 +327,7 @@ final AS ( atlantic.* FROM atlantic, - silver.country_2021 AS country + silver.country_2021_digital AS country UNION SELECT country.country_dguid, @@ -221,7 +338,7 @@ final AS ( the_rest.* FROM the_rest, - silver.country_2021 AS country + silver.country_2021_digital AS country ) SELECT * @@ -232,9 +349,9 @@ FROM Definition here: https://web.archive.org/web/20240224030001/https://www12.statcan.gc.ca/census-recensement/2016/ref/dict/geo027a-eng.cfm */ --- With geometries; -DROP TABLE IF EXISTS silver.grc_2016; -CREATE TABLE silver.grc_2016 AS +-- With geometries. Digital boundary; +DROP TABLE IF EXISTS silver.grc_2016_digital; +CREATE TABLE silver.grc_2016_digital AS WITH territories AS ( SELECT '2016A00016' AS grc_dguid, @@ -297,7 +414,7 @@ final AS ( territories.* FROM territories, - silver.country_2016 AS country + silver.country_2016_digital AS country UNION SELECT country.country_dguid, @@ -308,7 +425,7 @@ final AS ( prairies.* FROM prairies, - silver.country_2016 AS country + silver.country_2016_digital AS country UNION SELECT country.country_dguid, @@ -319,7 +436,7 @@ final AS ( atlantic.* FROM atlantic, - silver.country_2016 AS country + silver.country_2016_digital AS country UNION SELECT country.country_dguid, @@ -330,7 +447,7 @@ final AS ( the_rest.* FROM the_rest, - silver.country_2016 AS country + silver.country_2016_digital AS country ) SELECT * @@ -339,14 +456,131 @@ FROM -- Make geometries valid UPDATE - silver.grc_2016 + silver.grc_2016_digital SET geom = ST_MAKEVALID(geom) WHERE ST_ISVALID(geom) = 'f'; -- Create spatial index -CREATE INDEX grc_2016_geom_idx ON silver.grc_2016 +CREATE INDEX grc_2016_digital_geom_idx ON silver.grc_2016_digital +USING gist (geom) WITH (fillfactor = 100); + +-- With geometries. Cartographic boundary; +DROP TABLE IF EXISTS silver.grc_2016_cartographic; +CREATE TABLE silver.grc_2016_cartographic AS +WITH territories AS ( + SELECT + '2016A00016' AS grc_dguid, + 'Territories' AS grc_en_name, + 'Territoires' AS grc_fr_name, + ST_UNION(geom) AS geom + FROM + bronze.lpr_000b16a_e + WHERE + pruid IN ('60', '61', '62') +), + +atlantic AS ( + SELECT + '2016A00011' AS grc_dguid, + 'Atlantic' AS grc_en_name, + 'Atlantique' AS grc_fr_name, + ST_UNION(geom) AS geom + FROM + bronze.lpr_000b16a_e + WHERE + pruid IN ('10', '11', '12', '13') +), + +prairies AS ( + SELECT + '2016A00014' AS grc_dguid, + 'Prairies' AS grc_en_name, + 'Prairies' AS grc_fr_name, + ST_UNION(geom) AS geom + FROM + bronze.lpr_000b16a_e + WHERE + pruid IN ('48', '47', '46') +), + +the_rest AS ( + SELECT + CASE + WHEN pruid = '59' THEN '2016A00015' + WHEN pruid = '35' THEN '2016A00013' + WHEN pruid = '24' THEN '2016A00012' + END AS grc_dguid, + prename AS grc_en_name, + prfname AS grc_fr_name, + geom + FROM + bronze.lpr_000b16a_e + WHERE + pruid IN ('59', '35', '24') +), + +final AS ( + SELECT + country.country_dguid, + country.country_en_name, + country.country_fr_name, + country.country_en_abbreviation, + country.country_fr_abbreviation, + territories.* + FROM + territories, + silver.country_2016_digital AS country + UNION + SELECT + country.country_dguid, + country.country_en_name, + country.country_fr_name, + country.country_en_abbreviation, + country.country_fr_abbreviation, + prairies.* + FROM + prairies, + silver.country_2016_digital AS country + UNION + SELECT + country.country_dguid, + country.country_en_name, + country.country_fr_name, + country.country_en_abbreviation, + country.country_fr_abbreviation, + atlantic.* + FROM + atlantic, + silver.country_2016_digital AS country + UNION + SELECT + country.country_dguid, + country.country_en_name, + country.country_fr_name, + country.country_en_abbreviation, + country.country_fr_abbreviation, + the_rest.* + FROM + the_rest, + silver.country_2016_digital AS country +) + +SELECT * +FROM + final; + +-- Make geometries valid +UPDATE + silver.grc_2016_cartographic +SET + geom = ST_MAKEVALID(geom) +WHERE + ST_ISVALID(geom) = 'f'; + +-- Create spatial index +CREATE INDEX grc_2016_cartographic_geom_idx ON silver.grc_2016_cartographic USING gist (geom) WITH (fillfactor = 100); -- 2016 without geometries, and with pr_dguid; @@ -415,7 +649,7 @@ final AS ( territories.* FROM territories, - silver.country_2016 AS country + silver.country_2016_digital AS country UNION SELECT country.country_dguid, @@ -426,7 +660,7 @@ final AS ( prairies.* FROM prairies, - silver.country_2016 AS country + silver.country_2016_digital AS country UNION SELECT country.country_dguid, @@ -437,7 +671,7 @@ final AS ( atlantic.* FROM atlantic, - silver.country_2016 AS country + silver.country_2016_digital AS country UNION SELECT country.country_dguid, @@ -448,7 +682,7 @@ final AS ( the_rest.* FROM the_rest, - silver.country_2016 AS country + silver.country_2016_digital AS country ) SELECT * diff --git a/boundaries/population_centres.sql b/boundaries/population_centres.sql index ae42742..b953888 100644 --- a/boundaries/population_centres.sql +++ b/boundaries/population_centres.sql @@ -6,8 +6,9 @@ Population Centres Definition here: https://web.archive.org/web/20250324222733/https://www12.statcan.gc.ca/census-recensement/2021/ref/dict/az/definition-eng.cfm?ID=geo049a */ -DROP TABLE IF EXISTS silver.pop_ctr_2021; -CREATE TABLE silver.pop_ctr_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.pop_ctr_2021_digital; +CREATE TABLE silver.pop_ctr_2021_digital AS SELECT DISTINCT pr.country_dguid, pr.country_en_name, @@ -34,19 +35,61 @@ LEFT JOIN bronze.lpc_000a21a_e AS pop_ctr ON concat(gaf.pop_ctr_dguid, gaf.pop_ctr_p_dguid) = concat(pop_ctr.dguid, pop_ctr.dguidp) -LEFT JOIN silver.pr_2021 AS pr +LEFT JOIN silver.pr_2021_digital AS pr ON gaf.pr_dguid = pr.pr_dguid WHERE gaf.pop_ctr_dguid IS NOT null OR gaf.pop_ctr_p_dguid IS NOT null; -- Make geometries valid UPDATE - silver.pop_ctr_2021 + silver.pop_ctr_2021_digital SET geom = st_makevalid(geom) WHERE st_isvalid(geom) = 'f'; -- Create spatial index -CREATE INDEX pop_ctr_2021_geom_idx ON silver.pop_ctr_2021 USING gist ( +CREATE INDEX pop_ctr_2021_digital_geom_idx ON silver.pop_ctr_2021_digital USING gist ( geom ) WITH (fillfactor = 100); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.pop_ctr_2021_cartographic; +CREATE TABLE silver.pop_ctr_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + b.pop_ctr_dguid, + b.pop_ctr_p_dguid, + b.pop_ctr_name, + b.pop_ctr_type, + b.pop_ctr_class, + a.geom +FROM + bronze.lpc_000b21a_e AS a, + silver.pop_ctr_2021_digital AS b +WHERE concat(b.pop_ctr_dguid, b.pop_ctr_p_dguid) = concat(a.dguid, a.dguidp); + +-- Make geometries valid +UPDATE + silver.pop_ctr_2021_cartographic +SET + geom = st_makevalid(geom) +WHERE + st_isvalid(geom) = 'f'; + +-- Create spatial index +CREATE INDEX pop_ctr_2021_cartographic_geom_idx ON silver.pop_ctr_2021_cartographic USING gist ( + geom +) WITH (fillfactor = 100); \ No newline at end of file diff --git a/boundaries/provinces_and_territories.sql b/boundaries/provinces_and_territories.sql index fbe3d02..51d29ef 100644 --- a/boundaries/provinces_and_territories.sql +++ b/boundaries/provinces_and_territories.sql @@ -6,8 +6,9 @@ Provinces and territories Definition here: https://web.archive.org/web/20240402175445/https://www150.statcan.gc.ca/n1/pub/92-195-x/2021001/geo/prov/prov-eng.htm */ -DROP TABLE IF EXISTS silver.pr_2021; -CREATE TABLE silver.pr_2021 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.pr_2021_digital; +CREATE TABLE silver.pr_2021_digital AS SELECT DISTINCT grc.country_dguid, grc.country_en_name, @@ -46,14 +47,52 @@ WHERE -- Make geometries valid UPDATE - silver.pr_2021 + silver.pr_2021_digital SET geom = ST_MAKEVALID(geom) WHERE ST_ISVALID(geom) = 'f'; -- Create spatial index -CREATE INDEX pr_2021_geom_idx ON pr_2021 USING gist (geom) WITH ( +CREATE INDEX pr_2021_digital_geom_idx ON pr_2021_digital USING gist (geom) WITH ( + fillfactor = 100 +); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.pr_2021_cartographic; +CREATE TABLE silver.pr_2021_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + a.geom +FROM + bronze.lpr_000b21a_e AS a, + silver.pr_2021_digital AS b +WHERE + b.pr_dguid = a.dguid; + +-- Make geometries valid +UPDATE + silver.pr_2021_cartographic +SET + geom = ST_MAKEVALID(geom) +WHERE + ST_ISVALID(geom) = 'f'; + +-- Create spatial index +CREATE INDEX pr_2021_cartographic_geom_idx ON pr_2021_cartographic USING gist (geom) WITH ( fillfactor = 100 ); @@ -61,8 +100,9 @@ CREATE INDEX pr_2021_geom_idx ON pr_2021 USING gist (geom) WITH ( Definition here: https://web.archive.org/web/20241104061057/https://www12.statcan.gc.ca/census-recensement/2016/ref/dict/geo038-eng.cfm */ -DROP TABLE IF EXISTS silver.pr_2016; -CREATE TABLE silver.pr_2016 AS +-- Digital boundary; +DROP TABLE IF EXISTS silver.pr_2016_digital; +CREATE TABLE silver.pr_2016_digital AS SELECT DISTINCT grc.country_dguid, grc.country_en_name, @@ -101,13 +141,51 @@ WHERE -- Make geometries valid UPDATE - silver.pr_2016 + silver.pr_2016_digital SET geom = ST_MAKEVALID(geom) WHERE ST_ISVALID(geom) = 'f'; -- Create spatial index -CREATE INDEX pr_2016_geom_idx ON pr_2016 USING gist (geom) WITH ( +CREATE INDEX pr_2016_digital_geom_idx ON pr_2016_digital USING gist (geom) WITH ( fillfactor = 100 ); + +-- Cartographic boundary; +DROP TABLE IF EXISTS silver.pr_2016_cartographic; +CREATE TABLE silver.pr_2016_cartographic AS +SELECT DISTINCT + b.country_dguid, + b.country_en_name, + b.country_fr_name, + b.country_en_abbreviation, + b.country_fr_abbreviation, + b.grc_dguid, + b.grc_en_name, + b.grc_fr_name, + b.pr_dguid, + b.pr_en_name, + b.pr_fr_name, + b.pr_en_abbreviation, + b.pr_fr_abbreviation, + b.pr_iso_code, + a.geom +FROM + bronze.lpr_000b16a_e AS a, + silver.pr_2016_digital AS b +WHERE + b.pr_dguid = CONCAT('2016A0002', a.pruid); + +-- Make geometries valid +UPDATE + silver.pr_2016_cartographic +SET + geom = ST_MAKEVALID(geom) +WHERE + ST_ISVALID(geom) = 'f'; + +-- Create spatial index +CREATE INDEX pr_2016_cartographic_geom_idx ON pr_2016_cartographic USING gist (geom) WITH ( + fillfactor = 100 +); \ No newline at end of file diff --git a/experiments/export_2021_boundaries.ipynb b/experiments/export_2021_boundaries.ipynb index 561dc88..82e220a 100644 --- a/experiments/export_2021_boundaries.ipynb +++ b/experiments/export_2021_boundaries.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 8, "id": "b6e053ec", "metadata": {}, "outputs": [], @@ -22,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 9, "id": "22183463", "metadata": { "vscode": { @@ -37,28 +37,28 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 10, "id": "a0372dac", "metadata": {}, "outputs": [], "source": [ - "files=('country_2021'\n", - " 'grc_2021'\n", - " 'pr_2021'\n", - " 'er_2021'\n", - " 'car_2021'\n", - " 'cd_2021'\n", - " 'ccs_2021'\n", - " 'cma_2021'\n", - " 'csd_2021'\n", - " 'fed_2021_2013'\n", - " 'dpl_2021'\n", - " 'fsa_2021'\n", - " 'pop_ctr_2021'\n", - " 'ct_2021'\n", - " 'da_2021'\n", - " 'db_2021'\n", - " 'ada_2021'\n", + "files=('country_2021_digital'\n", + " 'grc_2021_digital'\n", + " 'pr_2021_digital'\n", + " 'er_2021_digital'\n", + " 'car_2021_digital'\n", + " 'cd_2021_digital'\n", + " 'ccs_2021_digital'\n", + " 'cma_2021_digital'\n", + " 'csd_2021_digital'\n", + " 'fed_2021_2013_digital'\n", + " 'dpl_2021_digital'\n", + " 'fsa_2021_digital'\n", + " 'pop_ctr_2021_digital'\n", + " 'ct_2021_digital'\n", + " 'da_2021_digital'\n", + " 'db_2021_digital'\n", + " 'ada_2021_digital'\n", " )" ] }, @@ -72,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 11, "id": "4c5bb532", "metadata": {}, "outputs": [ @@ -80,30 +80,31 @@ "name": "stdout", "output_type": "stream", "text": [ - "Exporting silver.country_2021 table to /data/boundaries/output/2021/digital_boundary_files/country_2021.parquet\n", - "Exporting silver.grc_2021 table to /data/boundaries/output/2021/digital_boundary_files/grc_2021.parquet\n", - "Exporting silver.pr_2021 table to /data/boundaries/output/2021/digital_boundary_files/pr_2021.parquet\n", - "Exporting silver.er_2021 table to /data/boundaries/output/2021/digital_boundary_files/er_2021.parquet\n", - "Exporting silver.car_2021 table to /data/boundaries/output/2021/digital_boundary_files/car_2021.parquet\n", - "Exporting silver.cd_2021 table to /data/boundaries/output/2021/digital_boundary_files/cd_2021.parquet\n", - "Exporting silver.ccs_2021 table to /data/boundaries/output/2021/digital_boundary_files/ccs_2021.parquet\n", - "Exporting silver.cma_2021 table to /data/boundaries/output/2021/digital_boundary_files/cma_2021.parquet\n", - "Exporting silver.csd_2021 table to /data/boundaries/output/2021/digital_boundary_files/csd_2021.parquet\n", - "Exporting silver.fed_2021_2013 table to /data/boundaries/output/2021/digital_boundary_files/fed_2021_2013.parquet\n", - "Exporting silver.dpl_2021 table to /data/boundaries/output/2021/digital_boundary_files/dpl_2021.parquet\n", - "Exporting silver.fsa_2021 table to /data/boundaries/output/2021/digital_boundary_files/fsa_2021.parquet\n", - "Exporting silver.pop_ctr_2021 table to /data/boundaries/output/2021/digital_boundary_files/pop_ctr_2021.parquet\n", - "Exporting silver.ct_2021 table to /data/boundaries/output/2021/digital_boundary_files/ct_2021.parquet\n", - "Exporting silver.da_2021 table to /data/boundaries/output/2021/digital_boundary_files/da_2021.parquet\n", - "Exporting silver.db_2021 table to /data/boundaries/output/2021/digital_boundary_files/db_2021.parquet\n", - "Exporting silver.ada_2021 table to /data/boundaries/output/2021/digital_boundary_files/ada_2021.parquet\n" + "Exporting silver.country_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/country_2021.parquet\n", + "Exporting silver.grc_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/grc_2021.parquet\n", + "Exporting silver.pr_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/pr_2021.parquet\n", + "Exporting silver.er_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/er_2021.parquet\n", + "Exporting silver.car_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/car_2021.parquet\n", + "Exporting silver.cd_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/cd_2021.parquet\n", + "Exporting silver.ccs_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/ccs_2021.parquet\n", + "Exporting silver.cma_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/cma_2021.parquet\n", + "Exporting silver.csd_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/csd_2021.parquet\n", + "Exporting silver.fed_2021_2013_digital table to /data/boundaries/output/2021/digital_boundary_files/fed_2021_2013.parquet\n", + "Exporting silver.dpl_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/dpl_2021.parquet\n", + "Exporting silver.fsa_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/fsa_2021.parquet\n", + "Exporting silver.pop_ctr_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/pop_ctr_2021.parquet\n", + "Exporting silver.ct_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/ct_2021.parquet\n", + "Exporting silver.da_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/da_2021.parquet\n", + "Exporting silver.db_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/db_2021.parquet\n", + "Exporting silver.ada_2021_digital table to /data/boundaries/output/2021/digital_boundary_files/ada_2021.parquet\n" ] } ], "source": [ "for file in ${files[@]}\n", "do\n", - " output_file=\"${output_folder}/${file}.parquet\"\n", + " filename=\"${file/_digital/}.parquet\"\n", + " output_file=\"${output_folder}/${filename}\"\n", " echo \"Exporting silver.${file} table to ${output_file}\" \n", " ogr2ogr \\\n", " -lco COMPRESSION=\"ZSTD\" \\\n", @@ -119,789 +120,97 @@ }, { "cell_type": "markdown", - "id": "77af4081-b070-4c78-bbd9-ab2bcbb28881", + "id": "7d62468a-f950-4e63-8bc5-4e3324f6ed65", "metadata": {}, "source": [ - "## Export as FlatGeobuf" + "# 2. Export Cartographic Boundary Files" ] }, { "cell_type": "code", - "execution_count": 6, - "id": "c012df55", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exporting country_2021 table to /data/boundaries/output/2021/digital_boundary_files/country_2021.fgb\n", - "Exporting grc_2021 table to /data/boundaries/output/2021/digital_boundary_files/grc_2021.fgb\n", - "Exporting pr_2021 table to /data/boundaries/output/2021/digital_boundary_files/pr_2021.fgb\n", - "Exporting er_2021 table to /data/boundaries/output/2021/digital_boundary_files/er_2021.fgb\n", - "Exporting car_2021 table to /data/boundaries/output/2021/digital_boundary_files/car_2021.fgb\n", - "Exporting cd_2021 table to /data/boundaries/output/2021/digital_boundary_files/cd_2021.fgb\n", - "Exporting ccs_2021 table to /data/boundaries/output/2021/digital_boundary_files/ccs_2021.fgb\n", - "Exporting cma_2021 table to /data/boundaries/output/2021/digital_boundary_files/cma_2021.fgb\n", - "Exporting csd_2021 table to /data/boundaries/output/2021/digital_boundary_files/csd_2021.fgb\n", - "Exporting fed_2021_2013 table to /data/boundaries/output/2021/digital_boundary_files/fed_2021_2013.fgb\n", - "Exporting dpl_2021 table to /data/boundaries/output/2021/digital_boundary_files/dpl_2021.fgb\n", - "Exporting fsa_2021 table to /data/boundaries/output/2021/digital_boundary_files/fsa_2021.fgb\n", - "Exporting pop_ctr_2021 table to /data/boundaries/output/2021/digital_boundary_files/pop_ctr_2021.fgb\n", - "Exporting ct_2021 table to /data/boundaries/output/2021/digital_boundary_files/ct_2021.fgb\n", - "Exporting da_2021 table to /data/boundaries/output/2021/digital_boundary_files/da_2021.fgb\n", - "Exporting db_2021 table to /data/boundaries/output/2021/digital_boundary_files/db_2021.fgb\n", - "Exporting ada_2021 table to /data/boundaries/output/2021/digital_boundary_files/ada_2021.fgb\n" - ] - } - ], - "source": [ - "for file in ${files[@]}\n", - "do\n", - " output_file=\"${output_folder}/${file}.fgb\"\n", - " echo \"Exporting ${file} table to ${output_file}\" \n", - " ogr2ogr \\\n", - " -f FlatGeobuf \\\n", - " -lco TITLE=\"${file}\" \\\n", - " ${output_file} \\\n", - " \"PG:host=db dbname=${POSTGRES_DB} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} port=5432\" \\\n", - " \"silver.${file}\" \\\n", - " -nln ${file}\n", - "done" - ] - }, - { - "cell_type": "markdown", - "id": "b90b3919", - "metadata": {}, - "source": [ - "## Export as File Geodatabase" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "64780573", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exporting country_2021 table to /data/boundaries/output/2021/digital_boundary_files/country_2021.gdb\n", - "ERROR 6: Unsupported geometry type\n", - "ERROR 1: Terminating translation prematurely after failed\n", - "translation of layer silver.country_2021 (use -skipfailures to skip errors)\n", - "Exporting grc_2021 table to /data/boundaries/output/2021/digital_boundary_files/grc_2021.gdb\n", - "ERROR 6: Unsupported geometry type\n", - "ERROR 1: Terminating translation prematurely after failed\n", - "translation of layer silver.grc_2021 (use -skipfailures to skip errors)\n", - "Exporting pr_2021 table to /data/boundaries/output/2021/digital_boundary_files/pr_2021.gdb\n", - "Exporting er_2021 table to /data/boundaries/output/2021/digital_boundary_files/er_2021.gdb\n", - "Exporting car_2021 table to /data/boundaries/output/2021/digital_boundary_files/car_2021.gdb\n", - "Exporting cd_2021 table to /data/boundaries/output/2021/digital_boundary_files/cd_2021.gdb\n", - "Exporting ccs_2021 table to /data/boundaries/output/2021/digital_boundary_files/ccs_2021.gdb\n", - "Exporting cma_2021 table to /data/boundaries/output/2021/digital_boundary_files/cma_2021.gdb\n", - "Exporting csd_2021 table to /data/boundaries/output/2021/digital_boundary_files/csd_2021.gdb\n", - "Exporting fed_2021_2013 table to /data/boundaries/output/2021/digital_boundary_files/fed_2021_2013.gdb\n", - "Exporting dpl_2021 table to /data/boundaries/output/2021/digital_boundary_files/dpl_2021.gdb\n", - "Exporting fsa_2021 table to /data/boundaries/output/2021/digital_boundary_files/fsa_2021.gdb\n", - "Exporting pop_ctr_2021 table to /data/boundaries/output/2021/digital_boundary_files/pop_ctr_2021.gdb\n", - "Exporting ct_2021 table to /data/boundaries/output/2021/digital_boundary_files/ct_2021.gdb\n", - "Exporting da_2021 table to /data/boundaries/output/2021/digital_boundary_files/da_2021.gdb\n", - "Exporting db_2021 table to /data/boundaries/output/2021/digital_boundary_files/db_2021.gdb\n", - "Exporting ada_2021 table to /data/boundaries/output/2021/digital_boundary_files/ada_2021.gdb\n" - ] - } - ], - "source": [ - "for file in ${files[@]}\n", - "do\n", - " output_file=\"${output_folder}/${file}.gdb\"\n", - " echo \"Exporting ${file} table to ${output_file}\" \n", - " ogr2ogr \\\n", - " -f OpenFileGDB \\\n", - " -lco TARGET_ARCGIS_VERSION=\"ARCGIS_PRO_3_2_OR_LATER\" \\\n", - " ${output_file} \\\n", - " \"PG:host=db dbname=${POSTGRES_DB} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} port=5432\" \\\n", - " \"silver.${file}\" \\\n", - " -nln ${file}\n", - "done" - ] - }, - { - "cell_type": "markdown", - "id": "69d3a9be-6fa5-437d-b851-f53457d49333", - "metadata": {}, - "source": [ - "### Zip File Geodatabases" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "69c6203b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Zipping country_2021.gdb\n", - " adding: country_2021.gdb/ (stored 0%)\n", - " adding: country_2021.gdb/gdb (stored 0%)\n", - " adding: country_2021.gdb/timestamps (deflated 98%)\n", - " adding: country_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: country_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: country_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: country_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: country_2021.gdb/a00000003.gdbtable (deflated 42%)\n", - " adding: country_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: country_2021.gdb/a00000004.gdbtable (deflated 44%)\n", - " adding: country_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: country_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: country_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: country_2021.gdb/a00000006.gdbtable (deflated 33%)\n", - " adding: country_2021.gdb/a00000006.gdbtablx (deflated 72%)\n", - " adding: country_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: country_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - "Zipping grc_2021.gdb\n", - " adding: grc_2021.gdb/ (stored 0%)\n", - " adding: grc_2021.gdb/gdb (stored 0%)\n", - " adding: grc_2021.gdb/timestamps (deflated 98%)\n", - " adding: grc_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: grc_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: grc_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: grc_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: grc_2021.gdb/a00000003.gdbtable (deflated 42%)\n", - " adding: grc_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: grc_2021.gdb/a00000004.gdbtable (deflated 44%)\n", - " adding: grc_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: grc_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: grc_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: grc_2021.gdb/a00000006.gdbtable (deflated 33%)\n", - " adding: grc_2021.gdb/a00000006.gdbtablx (deflated 72%)\n", - " adding: grc_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: grc_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - "Zipping pr_2021.gdb\n", - " adding: pr_2021.gdb/ (stored 0%)\n", - " adding: pr_2021.gdb/gdb (stored 0%)\n", - " adding: pr_2021.gdb/timestamps (deflated 98%)\n", - " adding: pr_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: pr_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: pr_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: pr_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: pr_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: pr_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: pr_2021.gdb/a00000004.gdbtable (deflated 77%)\n", - " adding: pr_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: pr_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: pr_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: pr_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: pr_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: pr_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: pr_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: pr_2021.gdb/a00000009.gdbtable (deflated 7%)\n", - " adding: pr_2021.gdb/a00000009.gdbtablx (deflated 98%)\n", - " adding: pr_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: pr_2021.gdb/a00000009.spx (deflated 97%)\n", - "Zipping er_2021.gdb\n", - " adding: er_2021.gdb/ (stored 0%)\n", - " adding: er_2021.gdb/gdb (stored 0%)\n", - " adding: er_2021.gdb/timestamps (deflated 98%)\n", - " adding: er_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: er_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: er_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: er_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: er_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: er_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: er_2021.gdb/a00000004.gdbtable (deflated 78%)\n", - " adding: er_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: er_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: er_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: er_2021.gdb/a00000006.gdbtable (deflated 22%)\n", - " adding: er_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: er_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: er_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: er_2021.gdb/a00000009.gdbtable (deflated 8%)\n", - " adding: er_2021.gdb/a00000009.gdbtablx (deflated 94%)\n", - " adding: er_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: er_2021.gdb/a00000009.spx (deflated 95%)\n", - "Zipping car_2021.gdb\n", - " adding: car_2021.gdb/ (stored 0%)\n", - " adding: car_2021.gdb/gdb (stored 0%)\n", - " adding: car_2021.gdb/timestamps (deflated 98%)\n", - " adding: car_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: car_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: car_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: car_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: car_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: car_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: car_2021.gdb/a00000004.gdbtable (deflated 78%)\n", - " adding: car_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: car_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: car_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: car_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: car_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: car_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: car_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: car_2021.gdb/a00000009.gdbtable (deflated 9%)\n", - " adding: car_2021.gdb/a00000009.gdbtablx (deflated 94%)\n", - " adding: car_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: car_2021.gdb/a00000009.spx (deflated 95%)\n", - "Zipping cd_2021.gdb\n", - " adding: cd_2021.gdb/ (stored 0%)\n", - " adding: cd_2021.gdb/gdb (stored 0%)\n", - " adding: cd_2021.gdb/timestamps (deflated 98%)\n", - " adding: cd_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: cd_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: cd_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: cd_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: cd_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: cd_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: cd_2021.gdb/a00000004.gdbtable (deflated 80%)\n", - " adding: cd_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: cd_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: cd_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: cd_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: cd_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: cd_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: cd_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: cd_2021.gdb/a00000009.gdbtable (deflated 9%)\n", - " adding: cd_2021.gdb/a00000009.gdbtablx (deflated 81%)\n", - " adding: cd_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: cd_2021.gdb/a00000009.spx (deflated 86%)\n", - "Zipping ccs_2021.gdb\n", - " adding: ccs_2021.gdb/ (stored 0%)\n", - " adding: ccs_2021.gdb/gdb (stored 0%)\n", - " adding: ccs_2021.gdb/timestamps (deflated 98%)\n", - " adding: ccs_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: ccs_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: ccs_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: ccs_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: ccs_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: ccs_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: ccs_2021.gdb/a00000004.gdbtable (deflated 81%)\n", - " adding: ccs_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: ccs_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: ccs_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: ccs_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: ccs_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: ccs_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: ccs_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: ccs_2021.gdb/a00000009.gdbtable (deflated 12%)\n", - " adding: ccs_2021.gdb/a00000009.gdbtablx (deflated 52%)\n", - " adding: ccs_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: ccs_2021.gdb/a00000009.spx (deflated 90%)\n", - "Zipping cma_2021.gdb\n", - " adding: cma_2021.gdb/ (stored 0%)\n", - " adding: cma_2021.gdb/gdb (stored 0%)\n", - " adding: cma_2021.gdb/timestamps (deflated 98%)\n", - " adding: cma_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: cma_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: cma_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: cma_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: cma_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: cma_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: cma_2021.gdb/a00000004.gdbtable (deflated 79%)\n", - " adding: cma_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: cma_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: cma_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: cma_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: cma_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: cma_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: cma_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: cma_2021.gdb/a00000009.gdbtable (deflated 9%)\n", - " adding: cma_2021.gdb/a00000009.gdbtablx (deflated 90%)\n", - " adding: cma_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: cma_2021.gdb/a00000009.spx (deflated 88%)\n", - "Zipping csd_2021.gdb\n", - " adding: csd_2021.gdb/ (stored 0%)\n", - " adding: csd_2021.gdb/gdb (stored 0%)\n", - " adding: csd_2021.gdb/timestamps (deflated 98%)\n", - " adding: csd_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: csd_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: csd_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: csd_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: csd_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: csd_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: csd_2021.gdb/a00000004.gdbtable (deflated 85%)\n", - " adding: csd_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: csd_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: csd_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: csd_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: csd_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: csd_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: csd_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: csd_2021.gdb/a00000009.gdbtable (deflated 14%)\n", - " adding: csd_2021.gdb/a00000009.gdbtablx (deflated 53%)\n", - " adding: csd_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: csd_2021.gdb/a00000009.spx (deflated 89%)\n", - "Zipping fed_2021_2013.gdb\n", - " adding: fed_2021_2013.gdb/ (stored 0%)\n", - " adding: fed_2021_2013.gdb/gdb (stored 0%)\n", - " adding: fed_2021_2013.gdb/timestamps (deflated 98%)\n", - " adding: fed_2021_2013.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: fed_2021_2013.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: fed_2021_2013.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: fed_2021_2013.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: fed_2021_2013.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: fed_2021_2013.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: fed_2021_2013.gdb/a00000004.gdbtable (deflated 79%)\n", - " adding: fed_2021_2013.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: fed_2021_2013.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: fed_2021_2013.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: fed_2021_2013.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: fed_2021_2013.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: fed_2021_2013.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: fed_2021_2013.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: fed_2021_2013.gdb/a00000009.gdbtable (deflated 9%)\n", - " adding: fed_2021_2013.gdb/a00000009.gdbtablx (deflated 79%)\n", - " adding: fed_2021_2013.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: fed_2021_2013.gdb/a00000009.spx (deflated 94%)\n", - "Zipping dpl_2021.gdb\n", - " adding: dpl_2021.gdb/ (stored 0%)\n", - " adding: dpl_2021.gdb/gdb (stored 0%)\n", - " adding: dpl_2021.gdb/timestamps (deflated 98%)\n", - " adding: dpl_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: dpl_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: dpl_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: dpl_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: dpl_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: dpl_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: dpl_2021.gdb/a00000004.gdbtable (deflated 78%)\n", - " adding: dpl_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: dpl_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: dpl_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: dpl_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: dpl_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: dpl_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: dpl_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: dpl_2021.gdb/a00000009.gdbtable (deflated 15%)\n", - " adding: dpl_2021.gdb/a00000009.gdbtablx (deflated 55%)\n", - " adding: dpl_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: dpl_2021.gdb/a00000009.spx (deflated 85%)\n", - "Zipping fsa_2021.gdb\n", - " adding: fsa_2021.gdb/ (stored 0%)\n", - " adding: fsa_2021.gdb/gdb (stored 0%)\n", - " adding: fsa_2021.gdb/timestamps (deflated 98%)\n", - " adding: fsa_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: fsa_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: fsa_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: fsa_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: fsa_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: fsa_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: fsa_2021.gdb/a00000004.gdbtable (deflated 77%)\n", - " adding: fsa_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: fsa_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: fsa_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: fsa_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: fsa_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: fsa_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: fsa_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: fsa_2021.gdb/a00000009.gdbtable (deflated 8%)\n", - " adding: fsa_2021.gdb/a00000009.gdbtablx (deflated 54%)\n", - " adding: fsa_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: fsa_2021.gdb/a00000009.spx (deflated 89%)\n", - "Zipping pop_ctr_2021.gdb\n", - " adding: pop_ctr_2021.gdb/ (stored 0%)\n", - " adding: pop_ctr_2021.gdb/gdb (stored 0%)\n", - " adding: pop_ctr_2021.gdb/timestamps (deflated 98%)\n", - " adding: pop_ctr_2021.gdb/a00000001.gdbtable (deflated 33%)\n", - " adding: pop_ctr_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: pop_ctr_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: pop_ctr_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: pop_ctr_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: pop_ctr_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: pop_ctr_2021.gdb/a00000004.gdbtable (deflated 80%)\n", - " adding: pop_ctr_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: pop_ctr_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: pop_ctr_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: pop_ctr_2021.gdb/a00000006.gdbtable (deflated 22%)\n", - " adding: pop_ctr_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: pop_ctr_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: pop_ctr_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: pop_ctr_2021.gdb/a00000009.gdbtable (deflated 10%)\n", - " adding: pop_ctr_2021.gdb/a00000009.gdbtablx (deflated 72%)\n", - " adding: pop_ctr_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: pop_ctr_2021.gdb/a00000009.spx (deflated 85%)\n", - "Zipping ct_2021.gdb\n", - " adding: ct_2021.gdb/ (stored 0%)\n", - " adding: ct_2021.gdb/gdb (stored 0%)\n", - " adding: ct_2021.gdb/timestamps (deflated 98%)\n", - " adding: ct_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: ct_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: ct_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: ct_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: ct_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: ct_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: ct_2021.gdb/a00000004.gdbtable (deflated 80%)\n", - " adding: ct_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: ct_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: ct_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: ct_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: ct_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: ct_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: ct_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: ct_2021.gdb/a00000009.gdbtable (deflated 17%)\n", - " adding: ct_2021.gdb/a00000009.gdbtablx (deflated 52%)\n", - " adding: ct_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: ct_2021.gdb/a00000009.spx (deflated 88%)\n", - "Zipping da_2021.gdb\n", - " adding: da_2021.gdb/ (stored 0%)\n", - " adding: da_2021.gdb/gdb (stored 0%)\n", - " adding: da_2021.gdb/timestamps (deflated 98%)\n", - " adding: da_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: da_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: da_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: da_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: da_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: da_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: da_2021.gdb/a00000004.gdbtable (deflated 85%)\n", - " adding: da_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: da_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: da_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: da_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: da_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: da_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: da_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: da_2021.gdb/a00000009.gdbtable (deflated 31%)\n", - " adding: da_2021.gdb/a00000009.gdbtablx (deflated 45%)\n", - " adding: da_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: da_2021.gdb/a00000009.spx (deflated 88%)\n", - "Zipping db_2021.gdb\n", - " adding: db_2021.gdb/ (stored 0%)\n", - " adding: db_2021.gdb/gdb (stored 0%)\n", - " adding: db_2021.gdb/timestamps (deflated 98%)\n", - " adding: db_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: db_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: db_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: db_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: db_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: db_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: db_2021.gdb/a00000004.gdbtable (deflated 86%)\n", - " adding: db_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: db_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: db_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: db_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: db_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: db_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: db_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: db_2021.gdb/a00000009.gdbtable (deflated 59%)\n", - " adding: db_2021.gdb/a00000009.gdbtablx (deflated 45%)\n", - " adding: db_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: db_2021.gdb/a00000009.spx (deflated 88%)\n", - "Zipping ada_2021.gdb\n", - " adding: ada_2021.gdb/ (stored 0%)\n", - " adding: ada_2021.gdb/gdb (stored 0%)\n", - " adding: ada_2021.gdb/timestamps (deflated 98%)\n", - " adding: ada_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: ada_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: ada_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: ada_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: ada_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: ada_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: ada_2021.gdb/a00000004.gdbtable (deflated 82%)\n", - " adding: ada_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: ada_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: ada_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: ada_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: ada_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: ada_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: ada_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: ada_2021.gdb/a00000009.gdbtable (deflated 13%)\n", - " adding: ada_2021.gdb/a00000009.gdbtablx (deflated 51%)\n", - " adding: ada_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: ada_2021.gdb/a00000009.spx (deflated 89%)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " adding: fed_2021.gdb/a00000004.gdbtable (deflated 79%)\n", - " adding: fed_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: fed_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: fed_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: fed_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: fed_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: fed_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: fed_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: fed_2021.gdb/a00000009.gdbtable (deflated 9%)\n", - " adding: fed_2021.gdb/a00000009.gdbtablx (deflated 79%)\n", - " adding: fed_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: fed_2021.gdb/a00000009.spx (deflated 94%)\n", - "Zipping car_2021.gdb\n", - " adding: car_2021.gdb/ (stored 0%)\n", - " adding: car_2021.gdb/gdb (stored 0%)\n", - " adding: car_2021.gdb/timestamps (deflated 98%)\n", - " adding: car_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: car_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: car_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: car_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: car_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: car_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: car_2021.gdb/a00000004.gdbtable (deflated 78%)\n", - " adding: car_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: car_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: car_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: car_2021.gdb/a00000006.gdbtable (deflated 22%)\n", - " adding: car_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: car_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: car_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: car_2021.gdb/a00000009.gdbtable (deflated 9%)\n", - " adding: car_2021.gdb/a00000009.gdbtablx (deflated 94%)\n", - " adding: car_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: car_2021.gdb/a00000009.spx (deflated 95%)\n", - "Zipping dpl_2021.gdb\n", - " adding: dpl_2021.gdb/ (stored 0%)\n", - " adding: dpl_2021.gdb/gdb (stored 0%)\n", - " adding: dpl_2021.gdb/timestamps (deflated 98%)\n", - " adding: dpl_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: dpl_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: dpl_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: dpl_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: dpl_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: dpl_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: dpl_2021.gdb/a00000004.gdbtable (deflated 78%)\n", - " adding: dpl_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: dpl_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: dpl_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: dpl_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: dpl_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: dpl_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: dpl_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: dpl_2021.gdb/a00000009.gdbtable (deflated 14%)\n", - " adding: dpl_2021.gdb/a00000009.gdbtablx (deflated 55%)\n", - " adding: dpl_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: dpl_2021.gdb/a00000009.spx (deflated 83%)\n", - "Zipping fsa_2021.gdb\n", - " adding: fsa_2021.gdb/ (stored 0%)\n", - " adding: fsa_2021.gdb/gdb (stored 0%)\n", - " adding: fsa_2021.gdb/timestamps (deflated 98%)\n", - " adding: fsa_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: fsa_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: fsa_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: fsa_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: fsa_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: fsa_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: fsa_2021.gdb/a00000004.gdbtable (deflated 77%)\n", - " adding: fsa_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: fsa_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: fsa_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: fsa_2021.gdb/a00000006.gdbtable (deflated 22%)\n", - " adding: fsa_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: fsa_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: fsa_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: fsa_2021.gdb/a00000009.gdbtable (deflated 8%)\n", - " adding: fsa_2021.gdb/a00000009.gdbtablx (deflated 54%)\n", - " adding: fsa_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: fsa_2021.gdb/a00000009.spx (deflated 89%)\n", - "Zipping cma_2021.gdb\n", - " adding: cma_2021.gdb/ (stored 0%)\n", - " adding: cma_2021.gdb/gdb (stored 0%)\n", - " adding: cma_2021.gdb/timestamps (deflated 98%)\n", - " adding: cma_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: cma_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: cma_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: cma_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: cma_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: cma_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: cma_2021.gdb/a00000004.gdbtable (deflated 79%)\n", - " adding: cma_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: cma_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: cma_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: cma_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: cma_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: cma_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: cma_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: cma_2021.gdb/a00000009.gdbtable (deflated 9%)\n", - " adding: cma_2021.gdb/a00000009.gdbtablx (deflated 89%)\n", - " adding: cma_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: cma_2021.gdb/a00000009.spx (deflated 88%)\n", - "Zipping pc_2021.gdb\n", - " adding: pc_2021.gdb/ (stored 0%)\n", - " adding: pc_2021.gdb/gdb (stored 0%)\n", - " adding: pc_2021.gdb/timestamps (deflated 98%)\n", - " adding: pc_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: pc_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: pc_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: pc_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: pc_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: pc_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: pc_2021.gdb/a00000004.gdbtable (deflated 80%)\n", - " adding: pc_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: pc_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: pc_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: pc_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: pc_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: pc_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: pc_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: pc_2021.gdb/a00000009.gdbtable (deflated 10%)\n", - " adding: pc_2021.gdb/a00000009.gdbtablx (deflated 72%)\n", - " adding: pc_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: pc_2021.gdb/a00000009.spx (deflated 85%)\n", - "Zipping ct_2021.gdb\n", - " adding: ct_2021.gdb/ (stored 0%)\n", - " adding: ct_2021.gdb/gdb (stored 0%)\n", - " adding: ct_2021.gdb/timestamps (deflated 98%)\n", - " adding: ct_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: ct_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: ct_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: ct_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: ct_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: ct_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: ct_2021.gdb/a00000004.gdbtable (deflated 80%)\n", - " adding: ct_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: ct_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: ct_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: ct_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: ct_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: ct_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: ct_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: ct_2021.gdb/a00000009.gdbtable (deflated 17%)\n", - " adding: ct_2021.gdb/a00000009.gdbtablx (deflated 52%)\n", - " adding: ct_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: ct_2021.gdb/a00000009.spx (deflated 88%)\n", - "Zipping da_2021.gdb\n", - " adding: da_2021.gdb/ (stored 0%)\n", - " adding: da_2021.gdb/gdb (stored 0%)\n", - " adding: da_2021.gdb/timestamps (deflated 98%)\n", - " adding: da_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: da_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: da_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: da_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: da_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: da_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: da_2021.gdb/a00000004.gdbtable (deflated 85%)\n", - " adding: da_2021.gdb/a00000004.gdbtablx (deflated 99%)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " adding: da_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: da_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: da_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: da_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: da_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: da_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: da_2021.gdb/a00000009.gdbtable (deflated 31%)\n", - " adding: da_2021.gdb/a00000009.gdbtablx (deflated 45%)\n", - " adding: da_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: da_2021.gdb/a00000009.spx (deflated 88%)\n", - "Zipping db_2021.gdb\n", - " adding: db_2021.gdb/ (stored 0%)\n", - " adding: db_2021.gdb/gdb (stored 0%)\n", - " adding: db_2021.gdb/timestamps (deflated 98%)\n", - " adding: db_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: db_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: db_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: db_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: db_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: db_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: db_2021.gdb/a00000004.gdbtable (deflated 86%)\n", - " adding: db_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: db_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: db_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: db_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: db_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: db_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: db_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: db_2021.gdb/a00000009.gdbtable (deflated 59%)\n", - " adding: db_2021.gdb/a00000009.gdbtablx (deflated 45%)\n", - " adding: db_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: db_2021.gdb/a00000009.spx (deflated 88%)\n", - "Zipping ada_2021.gdb\n", - " adding: ada_2021.gdb/ (stored 0%)\n", - " adding: ada_2021.gdb/gdb (stored 0%)\n", - " adding: ada_2021.gdb/timestamps (deflated 98%)\n", - " adding: ada_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: ada_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: ada_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: ada_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: ada_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: ada_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: ada_2021.gdb/a00000004.gdbtable (deflated 82%)\n", - " adding: ada_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: ada_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: ada_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: ada_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: ada_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: ada_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: ada_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: ada_2021.gdb/a00000009.gdbtable (deflated 13%)\n", - " adding: ada_2021.gdb/a00000009.gdbtablx (deflated 51%)\n", - " adding: ada_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: ada_2021.gdb/a00000009.spx (deflated 89%)\n", - "Zipping pn_2021.gdb\n", - " adding: pn_2021.gdb/ (stored 0%)\n", - " adding: pn_2021.gdb/gdb (stored 0%)\n", - " adding: pn_2021.gdb/timestamps (deflated 98%)\n", - " adding: pn_2021.gdb/a00000001.gdbtable (deflated 34%)\n", - " adding: pn_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: pn_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: pn_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: pn_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: pn_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: pn_2021.gdb/a00000004.gdbtable (deflated 87%)\n", - " adding: pn_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: pn_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: pn_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: pn_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: pn_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: pn_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: pn_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: pn_2021.gdb/a00000009.gdbtable (deflated 93%)\n", - " adding: pn_2021.gdb/a00000009.gdbtablx (deflated 46%)\n", - " adding: pn_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: pn_2021.gdb/a00000009.spx (deflated 84%)\n", - "Zipping road_2021.gdb\n", - " adding: road_2021.gdb/ (stored 0%)\n", - " adding: road_2021.gdb/gdb (stored 0%)\n", - " adding: road_2021.gdb/timestamps (deflated 98%)\n", - " adding: road_2021.gdb/a00000001.gdbtable (deflated 35%)\n", - " adding: road_2021.gdb/a00000001.gdbtablx (deflated 99%)\n", - " adding: road_2021.gdb/a00000002.gdbtable (deflated 68%)\n", - " adding: road_2021.gdb/a00000002.gdbtablx (deflated 97%)\n", - " adding: road_2021.gdb/a00000003.gdbtable (deflated 56%)\n", - " adding: road_2021.gdb/a00000003.gdbtablx (deflated 99%)\n", - " adding: road_2021.gdb/a00000004.gdbtable (deflated 90%)\n", - " adding: road_2021.gdb/a00000004.gdbtablx (deflated 99%)\n", - " adding: road_2021.gdb/a00000005.gdbtable (deflated 37%)\n", - " adding: road_2021.gdb/a00000005.gdbtablx (deflated 97%)\n", - " adding: road_2021.gdb/a00000006.gdbtable (deflated 23%)\n", - " adding: road_2021.gdb/a00000006.gdbtablx (deflated 99%)\n", - " adding: road_2021.gdb/a00000007.gdbtable (deflated 47%)\n", - " adding: road_2021.gdb/a00000007.gdbtablx (deflated 98%)\n", - " adding: road_2021.gdb/a00000009.gdbtable (deflated 79%)\n", - " adding: road_2021.gdb/a00000009.gdbtablx (deflated 45%)\n", - " adding: road_2021.gdb/a00000009.gdbindexes (deflated 37%)\n", - " adding: road_2021.gdb/a00000009.spx (deflated 85%)\n" - ] - } - ], - "source": [ - "cd ${output_folder}\n", - "for file in ${files[@]}\n", - "do\n", - " output_file=\"${file}.gdb\"\n", - " echo \"Zipping ${output_file}\"\n", - " zip -r \"${output_file}.zip\" ${output_file}\n", - " rm -rf ${output_file}\n", - "done" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1bde70a8-e4ae-450b-9929-de35970172ab", + "execution_count": 13, + "id": "3334fd6f-f63c-4ff6-b646-f6c11de75efa", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "output_folder=\"${DATA_FOLDER}/boundaries/output/2021/cartographic_boundary_files\"\n", + "mkdir -p ${output_folder}" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "f2c21644-77e1-4ed6-856d-61912d10060c", + "metadata": {}, + "outputs": [], + "source": [ + "files=('country_2021_cartographic'\n", + " 'grc_2021_cartographic'\n", + " 'pr_2021_cartographic'\n", + " 'er_2021_cartographic'\n", + " 'car_2021_cartographic'\n", + " 'cd_2021_cartographic'\n", + " 'ccs_2021_cartographic'\n", + " 'cma_2021_cartographic'\n", + " 'csd_2021_cartographic'\n", + " 'fed_2021_2013_cartographic'\n", + " 'dpl_2021_cartographic'\n", + " 'fsa_2021_cartographic'\n", + " 'pop_ctr_2021_cartographic'\n", + " 'ct_2021_cartographic'\n", + " 'da_2021_cartographic'\n", + " 'db_2021_cartographic'\n", + " 'ada_2021_cartographic'\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "814e1471-7632-4db2-ae4d-6ac27564ada7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exporting silver.country_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/country_2021.parquet\n", + "Exporting silver.grc_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/grc_2021.parquet\n", + "Exporting silver.pr_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/pr_2021.parquet\n", + "Exporting silver.er_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/er_2021.parquet\n", + "Exporting silver.car_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/car_2021.parquet\n", + "Exporting silver.cd_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/cd_2021.parquet\n", + "Exporting silver.ccs_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/ccs_2021.parquet\n", + "Exporting silver.cma_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/cma_2021.parquet\n", + "Exporting silver.csd_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/csd_2021.parquet\n", + "Exporting silver.fed_2021_2013_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/fed_2021_2013.parquet\n", + "Exporting silver.dpl_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/dpl_2021.parquet\n", + "Exporting silver.fsa_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/fsa_2021.parquet\n", + "Exporting silver.pop_ctr_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/pop_ctr_2021.parquet\n", + "Exporting silver.ct_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/ct_2021.parquet\n", + "Exporting silver.da_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/da_2021.parquet\n", + "Exporting silver.db_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/db_2021.parquet\n", + "Exporting silver.ada_2021_cartographic table to /data/boundaries/output/2021/cartographic_boundary_files/ada_2021.parquet\n" + ] + } + ], + "source": [ + "for file in ${files[@]}\n", + "do\n", + " filename=\"${file/_cartographic/}.parquet\"\n", + " output_file=\"${output_folder}/${filename}\"\n", + " echo \"Exporting silver.${file} table to ${output_file}\" \n", + " ogr2ogr \\\n", + " -lco COMPRESSION=\"ZSTD\" \\\n", + " -lco CREATOR=\"www.dataforcanada.org\" \\\n", + " -lco WRITE_COVERING_BBOX=\"YES\" \\\n", + " -lco SORT_BY_BBOX=\"YES\" \\\n", + " -f Parquet \\\n", + " ${output_file} \\\n", + " \"PG:host=db dbname=${POSTGRES_DB} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} port=5432\" \\\n", + " \"silver.${file}\"\n", + "done" + ] } ], "metadata": {