dataforcanada/d4c-datapkg-statistical
1
0
Fork 0
mirror of https://github.com/dataforcanada/d4c-datapkg-statistical.git synced 2026-06-13 14:10:55 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
5e26ec282edd37a7ec6cd34d618fd2ce68b4356d
d4c-datapkg-statistical/national_address_register/process_2024_12.ipynb
T

548 lines
19 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "markdown",
"id": "11c50f98-ddeb-4af0-b69a-743f915fa904",
"metadata": {},
"source": [
"# Experimenting with processing this file. Still need to figure out how to structure this file"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "95feb0e4-b3b6-4235-8c8d-bc3652e82b3a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Buckaroo has been enabled as the default DataFrame viewer. To return to default dataframe visualization use `from buckaroo import disable; disable()`\n"
]
}
],
"source": [
"#!/usr/bin/env python\n",
"# coding: utf-8\n",
"import gc\n",
"import glob\n",
"import os\n",
"import sys \n",
"\n",
"import buckaroo\n",
"import duckdb\n",
"from IPython.core.interactiveshell import InteractiveShell \n",
"import geopandas as gpd\n",
"import pandas as pd\n",
"from sqlalchemy import create_engine\n",
"from sqlalchemy import text\n",
"\n",
"# Enable multiple outputs per cell\n",
"InteractiveShell.ast_node_interactivity = \"all\"\n",
"# Show all columns\n",
"pd.set_option('display.max_columns', None)\n",
"\n",
"DATABASE = os.environ.get(\"POSTGRES_DB\")\n",
"USER = os.environ.get(\"POSTGRES_USER\")\n",
"PASSWORD = os.environ.get(\"POSTGRES_PASSWORD\")\n",
"\n",
"engine = create_engine(f\"postgresql://{USER}:{PASSWORD}@db:5432/{DATABASE}\")"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "e10bc182-d0d6-4aa3-99b3-2fc396f9ce34",
"metadata": {},
"outputs": [],
"source": [
"input_folder = '/data/national_address_register/extracted'"
]
},
{
"cell_type": "markdown",
"id": "7f170d27-14ca-488a-811d-1c9836264bb6",
"metadata": {},
"source": [
"# 1. Process 2024-12 vintage"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "b27e0edc-59ca-4a63-8bff-1d85909aa174",
"metadata": {},
"outputs": [],
"source": [
"nar_addresses_csvs = glob.glob(f'{input_folder}/2024-12/Addresses/*.csv')\n",
"nar_locations_csvs = glob.glob(f'{input_folder}/2024-12/Locations/*.csv')\n",
"encoding = 'utf-8'"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "3fcca833-8547-43b1-b5c7-e5f463ef0bbc",
"metadata": {},
"outputs": [],
"source": [
"def process_nar_locations_csvs(csvs_to_process, encoding):\n",
" \"\"\"\n",
" 1. Reads subset of fields for National Address Register locations\n",
" 2. Appends all of the processed CSVs as one dataframe\n",
" \"\"\"\n",
" dataframes_to_concatenate = []\n",
" for filename in csvs_to_process:\n",
" print(f\"Processing {filename}\")\n",
" params = {\n",
" 'filepath_or_buffer': filename,\n",
" 'encoding': encoding,\n",
" 'usecols': ['LOC_GUID', \n",
" 'REPPOINT_LATITUDE', \n",
" 'REPPOINT_LONGITUDE'\n",
" ]\n",
" }\n",
" nar_location_df = pd.read_csv(**params)\n",
" # Lowercase columns\n",
" nar_location_df.columns = [x.lower() for x in nar_location_df.columns]\n",
" dataframes_to_concatenate.append(nar_location_df)\n",
" \n",
" print(\"Concatenating all dataframes into one\")\n",
" nar_locations_df = pd.concat(dataframes_to_concatenate)\n",
" \n",
" return nar_locations_df\n",
"\n",
"def process_nar_addresses_csvs(csvs_to_process, encoding):\n",
" \"\"\"\n",
" 1. Reads subset of fields for National Address Register addresses\n",
" 2. Appends all of the processed CSVs as one dataframe\n",
" \"\"\"\n",
" dataframes_to_concatenate = []\n",
" for filename in csvs_to_process:\n",
" print(f\"Processing {filename}\")\n",
" params = {\n",
" 'filepath_or_buffer': filename,\n",
" 'encoding': encoding,\n",
" 'usecols': ['LOC_GUID', \n",
" 'ADDR_GUID', \n",
" 'APT_NO_LABEL',\n",
" 'CIVIC_NO',\n",
" 'CIVIC_NO_SUFFIX',\n",
" 'OFFICIAL_STREET_NAME',\n",
" 'OFFICIAL_STREET_TYPE',\n",
" 'OFFICIAL_STREET_DIR',\n",
" 'MAIL_STREET_NAME',\n",
" 'MAIL_STREET_TYPE',\n",
" 'MAIL_STREET_DIR',\n",
" 'MAIL_MUN_NAME',\n",
" 'MAIL_POSTAL_CODE',\n",
" 'BG_DLS_LSD',\n",
" 'BG_DLS_QTR',\n",
" 'BG_DLS_SCTN',\n",
" 'BG_DLS_RNG',\n",
" 'BG_DLS_MRD',\n",
" # Removing since REPPOINT_LATITUDE and REPPOINT_LONGITUDE seem to have same purpose\n",
" #'BG_X',\n",
" #'BG_Y',\n",
" 'BU_USE',\n",
" 'BU_N_CIVIC_ADD'\n",
" ],\n",
" 'dtype': {\n",
" \"CIVIC_NO\": \"Int32\", \n",
" \"PROV_CODE\": object,\n",
" \"BU_USE\": \"Int8\",\n",
" \"BG_DLS_LSD\": object,\n",
" \"BG_DLS_QTR\": object,\n",
" \"BG_DLS_SCTN\": object,\n",
" \"BG_DLS_TWNSHP\": object,\n",
" \"BG_DLS_RNG\": object,\n",
" \"BG_DLS_MRD\": object\n",
" }\n",
" }\n",
" nar_address_df = pd.read_csv(**params)\n",
" # Lowercase columns\n",
" nar_address_df.columns = [x.lower() for x in nar_address_df.columns]\n",
" dataframes_to_concatenate.append(nar_address_df)\n",
" \n",
" print(\"Concatenating all dataframes into one\")\n",
" nar_addresses_df = pd.concat(dataframes_to_concatenate, ignore_index=True)\n",
" \n",
" return nar_addresses_df"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "f9bb3a4d-d913-4b9d-aa49-7ec2c4dfef60",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_10.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_11.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_12.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_13.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_24_part_1.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_24_part_2.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_24_part_3.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_24_part_4.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_35_part_1.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_35_part_2.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_35_part_3.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_35_part_4.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_35_part_5.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_46.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_47.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_48_part_1.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_48_part_2.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_59_part_1.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_59_part_2.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_60.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_61.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Locations/Location_62.csv\n",
"Concatenating all dataframes into one\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_10.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_11.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_12.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_13.csv\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_943/484766080.py:74: DtypeWarning: Columns (27) have mixed types. Specify dtype option on import or set low_memory=False.\n",
" nar_address_df = pd.read_csv(**params)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_24_part_1.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_24_part_2.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_24_part_3.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_24_part_4.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_24_part_5.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_35_part_1.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_35_part_2.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_35_part_3.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_35_part_4.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_35_part_5.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_35_part_6.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_35_part_7.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_46.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_47.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_48_part_1.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_48_part_2.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_48_part_3.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_59_part_1.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_59_part_2.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_59_part_3.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_60.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_61.csv\n",
"Processing /data/national_address_register/extracted/2024-12/Addresses/Address_62.csv\n",
"Concatenating all dataframes into one\n"
]
}
],
"source": [
"nar_locations = process_nar_locations_csvs(nar_locations_csvs, encoding)\n",
"nar_addresses = process_nar_addresses_csvs(nar_addresses_csvs, encoding)"
]
},
{
"cell_type": "markdown",
"id": "b49f6602-9bda-410f-82a2-54a5711311b0",
"metadata": {},
"source": [
"# TODO\n",
"- look into why there are locations with empty reppoint_latitude and reppoint_longitude\n",
" - There are 84,285 records that have an empty reppoint_latitude and reppoint_longitude"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "f3600645-c350-473f-81ea-0bd9ebe70e37",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Combining nar_addresses and nar_locations\n"
]
},
{
"data": {
"text/plain": [
"40"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"print(\"Combining nar_addresses and nar_locations\")\n",
"nar_addresses_combined = duckdb.sql(\"\"\"\n",
"SELECT a.addr_guid, a.apt_no_label, a.civic_no, a.civic_no_suffix, a.official_street_name, a.mail_street_name, a.official_street_type, a.mail_street_type,\n",
" a.official_street_dir AS official_street_direction, a.mail_street_dir AS mail_street_direction, a.mail_postal_code, a.mail_mun_name AS mail_municipality_name, \n",
" a.bu_n_civic_add, a.bu_use,\n",
" a.bg_dls_lsd, a.bg_dls_qtr, a.bg_dls_sctn, a.bg_dls_rng, a.bg_dls_mrd,\n",
" b.reppoint_latitude, b.reppoint_longitude\n",
"FROM nar_addresses AS a,\n",
" nar_locations AS b\n",
"WHERE a.loc_guid = b.loc_guid AND b.reppoint_latitude IS NOT NULL\n",
"\"\"\").df()\n",
"\n",
"del nar_addresses\n",
"del nar_locations\n",
"gc.collect()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "0d35eec6-8ab3-4e7a-a7fd-2915333e27f9",
"metadata": {},
"outputs": [],
"source": [
"gdf = gpd.GeoDataFrame(\n",
" nar_addresses_combined, \n",
" geometry=gpd.points_from_xy(nar_addresses_combined.reppoint_longitude,\n",
" nar_addresses_combined.reppoint_latitude),\n",
" crs=\"EPSG:4326\"\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "614ad773-adb3-413c-8b8f-da721afc85cd",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dropping 'reppoint_latitude', 'reppoint_longitude' from geodataframe\n"
]
}
],
"source": [
"print(\"Dropping 'reppoint_latitude', 'reppoint_longitude' from geodataframe\")\n",
"gdf.drop(columns=[\"reppoint_latitude\", \"reppoint_longitude\"], \n",
" inplace=True)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "cac024f8-6eb5-48ea-88ad-289edf505ba3",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"del nar_addresses_combined\n",
"gc.collect()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "d2f81e61-1fc7-4518-ad1f-d515e16ce22c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loading geodataframe to PostgreSQL as bronze.nar_2024_12\n"
]
}
],
"source": [
"print(\"Loading geodataframe to PostgreSQL as bronze.nar_2024_12\")\n",
"gdf.to_postgis(name=\"nar_2024_12\", \n",
" schema='bronze',\n",
" con=engine,\n",
" chunksize=150000)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "b7fb6976-0b95-445e-b1e8-022c39bce25b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"584"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"del(gdf)\n",
"gc.collect()"
]
},
{
"cell_type": "markdown",
"id": "384cd07b-c79f-42e8-81d6-ecbe78f167b1",
"metadata": {},
"source": [
"## Link to 2021 geographies\n",
"There are 10 records that were not linked to 2021 geographies"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "f7f56297-150b-4ad2-a5c2-2ee5e477f978",
"metadata": {},
"outputs": [],
"source": [
"sql = \"\"\"\n",
"DROP TABLE IF EXISTS silver.nar_2024_12;\n",
"CREATE TABLE silver.nar_2024_12 AS\n",
"SELECT DISTINCT\n",
" b.country_dguid,\n",
" b.country_en_name,\n",
" b.country_fr_name,\n",
" b.country_en_abbreviation,\n",
" b.country_fr_abbreviation,\n",
" b.grc_dguid,\n",
" b.grc_en_name,\n",
" b.grc_fr_name,\n",
" b.pr_dguid,\n",
" b.pr_en_name,\n",
" b.pr_fr_name,\n",
" b.pr_en_abbreviation,\n",
" b.pr_fr_abbreviation,\n",
" b.pr_iso_code,\n",
" b.car_dguid,\n",
" b.car_en_name,\n",
" b.car_fr_name,\n",
" b.er_dguid,\n",
" b.er_name,\n",
" b.cd_dguid,\n",
" b.cd_name,\n",
" b.cd_type,\n",
" b.ccs_dguid,\n",
" b.ccs_name,\n",
" b.cma_dguid,\n",
" b.cma_p_dguid,\n",
" b.cma_name,\n",
" b.cma_type,\n",
" b.csd_dguid,\n",
" b.csd_name,\n",
" b.csd_type,\n",
" b.sac_type,\n",
" b.sac_code,\n",
" b.fed_dguid,\n",
" b.fed_name,\n",
" b.fed_en_name,\n",
" b.fed_fr_name,\n",
" b.ct_dguid,\n",
" b.ada_dguid,\n",
" b.da_dguid,\n",
" b.db_dguid,\n",
" a.addr_guid,\n",
" a.apt_no_label,\n",
" a.civic_no,\n",
" a.civic_no_suffix,\n",
" a.official_street_name, \n",
" a.mail_street_name, \n",
" a.official_street_type,\n",
" a.mail_street_type,\n",
" a.official_street_direction,\n",
" a.mail_street_direction,\n",
" a.mail_postal_code,\n",
" a.mail_municipality_name,\n",
" a.bu_n_civic_add,\n",
" a.bu_use,\n",
" a.bg_dls_lsd,\n",
" a.bg_dls_qtr,\n",
" a.bg_dls_sctn,\n",
" a.bg_dls_rng,\n",
" a.bg_dls_mrd,\n",
" a.geometry AS geom\n",
"FROM bronze.nar_2024_12 AS a,\n",
" silver.db_2021_digital AS b\n",
"WHERE ST_Intersects(a.geometry, b.geom);\n",
"\n",
"-- Create spatial index\n",
"CREATE INDEX nar_2024_12_geom_idx ON silver.nar_2024_12 USING gist (geom) WITH (\n",
" fillfactor = 100\n",
");\n",
"\"\"\""
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "ae7babb4-a3e6-4c0b-93a6-3b52a4f89a1b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<sqlalchemy.engine.cursor.CursorResult at 0x7f38310c66d0>"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"with engine.connect() as conn:\n",
" conn.execute(text(sql))\n",
" conn.commit()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink