dataforcanada/d4c-datapkg-statistical
1
0
Fork 0
mirror of https://github.com/dataforcanada/d4c-datapkg-statistical.git synced 2026-06-13 14:10:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

Process 2024-12 national address register. Still need to make some improvements

Browse Source
This commit is contained in:
Diego Ripley
2025-06-06 22:49:47 +00:00
parent 2350d6d8d7
commit 5e26ec282e
12 changed files with 765 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files
national_address_register/export.ipynb
+155
View File
@@ -0,0 +1,155 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 3,
"id": "b6e053ec",
"metadata": {},
"outputs": [],
"source": [
"DATA_FOLDER=/data\n",
"\n",
"source ../.env"
]
},
{
"cell_type": "markdown",
"id": "12eca225-3d05-4bb7-95fa-7b9df694f53d",
"metadata": {},
"source": [
"# 1. Export National Address Register File (2024-12 vintage)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "4c5bb532",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Exporting silver.nar_2024_12 table to /data/national_address_register/output/2024-12/nar_2024_12.parquet\n"
]
}
],
"source": [
"output_folder=\"${DATA_FOLDER}/national_address_register/output/2024-12\"\n",
"output_file=\"${output_folder}/nar_2024_12.parquet\"\n",
"echo \"Exporting silver.nar_2024_12 table to ${output_file}\" \n",
"ogr2ogr \\\n",
" -lco COMPRESSION=\"ZSTD\" \\\n",
" -lco CREATOR=\"www.dataforcanada.org\" \\\n",
" -lco WRITE_COVERING_BBOX=\"YES\" \\\n",
" -lco SORT_BY_BBOX=\"YES\" \\\n",
" -f Parquet \\\n",
" ${output_file} \\\n",
" \"PG:host=db dbname=${POSTGRES_DB} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} port=5432\" \\\n",
" \"silver.nar_2024_12\""
]
},
{
"cell_type": "markdown",
"id": "245cd8fb-da36-4476-8e5c-4d3665e901d7",
"metadata": {},
"source": [
"# 2. Export National Address Register File (2024-06 vintage)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f3e2a5b5-9b31-4f82-88da-414bf23ba515",
"metadata": {},
"outputs": [],
"source": [
"output_folder=\"${DATA_FOLDER}/national_address_register/output/2024-06\"\n",
"output_file=\"${output_folder}/nar_2024_06.parquet\"\n",
"echo \"Exporting silver.nar_2024_06 table to ${output_file}\" \n",
"ogr2ogr \\\n",
" -lco COMPRESSION=\"ZSTD\" \\\n",
" -lco CREATOR=\"www.dataforcanada.org\" \\\n",
" -lco WRITE_COVERING_BBOX=\"YES\" \\\n",
" -lco SORT_BY_BBOX=\"YES\" \\\n",
" -f Parquet \\\n",
" ${output_file} \\\n",
" \"PG:host=db dbname=${POSTGRES_DB} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} port=5432\" \\\n",
" \"silver.nar_2024_06\""
]
},
{
"cell_type": "markdown",
"id": "56ef7755-06ef-473c-8a1f-0129f6b1dc28",
"metadata": {},
"source": [
"# 3. Export National Address Register File (2023 vintage)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "97df2904-c18d-4132-bc28-f0f454c48c87",
"metadata": {},
"outputs": [],
"source": [
"output_folder=\"${DATA_FOLDER}/national_address_register/output/2023\"\n",
"output_file=\"${output_folder}/nar_2023.parquet\"\n",
"echo \"Exporting silver.nar_2023 table to ${output_file}\"\n",
"ogr2ogr \\\n",
" -lco COMPRESSION=\"ZSTD\" \\\n",
" -lco CREATOR=\"www.dataforcanada.org\" \\\n",
" -lco WRITE_COVERING_BBOX=\"YES\" \\\n",
" -lco SORT_BY_BBOX=\"YES\" \\\n",
" -f Parquet \\\n",
" ${output_file} \\\n",
" \"PG:host=db dbname=${POSTGRES_DB} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} port=5432\" \\\n",
" \"silver.nar_2023\""
]
},
{
"cell_type": "markdown",
"id": "f1bfd838-19aa-4de7-9ca1-61b1215d7865",
"metadata": {},
"source": [
"# 4. Export National Address Register File (2022 vintage)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d2f1210f-0d9c-45de-a019-a438ff9dfa4a",
"metadata": {},
"outputs": [],
"source": [
"output_folder=\"${DATA_FOLDER}/national_address_register/output/2022\"\n",
"output_file=\"${output_folder}/nar_2022.parquet\"\n",
"echo \"Exporting silver.nar_2022 table to ${output_file}\"\n",
"ogr2ogr \\\n",
" -lco COMPRESSION=\"ZSTD\" \\\n",
" -lco CREATOR=\"www.dataforcanada.org\" \\\n",
" -lco WRITE_COVERING_BBOX=\"YES\" \\\n",
" -lco SORT_BY_BBOX=\"YES\" \\\n",
" -f Parquet \\\n",
" ${output_file} \\\n",
" \"PG:host=db dbname=${POSTGRES_DB} user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} port=5432\" \\\n",
" \"silver.nar_2022\""
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Bash",
"language": "bash",
"name": "bash"
},
"language_info": {
"codemirror_mode": "shell",
"file_extension": ".sh",
"mimetype": "text/x-sh",
"name": "bash"
}
},
"nbformat": 4,
"nbformat_minor": 5
}