dataforcanada/d4c-datapkg-statistical
1
0
Fork 0
mirror of https://github.com/dataforcanada/d4c-datapkg-statistical.git synced 2026-06-13 14:10:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Diego Ripley
2025-05-24 13:37:31 -04:00
commit f93e4d0cec
108 changed files with 11689 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/duckdb_census_of_population.ipynb
+242
View File
@@ -0,0 +1,242 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 17,
"id": "56ac906e",
"metadata": {},
"outputs": [],
"source": [
"import duckdb\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "708e293d",
"metadata": {},
"outputs": [],
"source": [
"con = duckdb.connect()\n",
"con.install_extension(\"spatial\")\n",
"con.load_extension(\"spatial\")"
]
},
{
"cell_type": "markdown",
"id": "5d97e882",
"metadata": {},
"source": [
"# DA"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c1b2ccf6",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b2fd24f9af2c45369a4aa93b249227c0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"<duckdb.duckdb.DuckDBPyConnection at 0x1474498ddc30>"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"con.execute(\"\"\"\n",
"DROP TABLE IF EXISTS geo_data;\n",
"CREATE TABLE geo_data AS\n",
"SELECT da.da_dguid, da_cop.* EXCLUDE dguid, da.geom FROM 'https://data.dataforcanada.org/processed/statistics_canada/census_of_population/2021/tabular/da_2021.parquet' AS da_cop,\n",
"'https://data.dataforcanada.org/processed/statistics_canada/boundaries/2021/digital_boundary_files/da_2021.parquet' AS da\n",
"WHERE da_cop.dguid = da.da_dguid\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "33a3819e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[(57936,)]"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"con.execute(\"SELECT count(*) FROM geo_data;\")\n",
"con.fetchall()"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "6d38118b",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ee1b19d9ad3c4a9483ba3cef3600ba8e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"<duckdb.duckdb.DuckDBPyConnection at 0x1474498ddc30>"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"con.execute(\"\"\"\n",
"COPY geo_data TO 'da_2021_cop.gdb'\n",
"WITH (\n",
" FORMAT GDAL,\n",
" DRIVER 'OpenFileGDB',\n",
" GEOMETRY_TYPE 'POLYGON',\n",
" SRS 'EPSG:4326'\n",
");\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "76ada8f9",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e4d6cb52cb864a0cac1941bb25315b79",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"<duckdb.duckdb.DuckDBPyConnection at 0x1477954f73b0>"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"con.execute(\"\"\"\n",
"COPY geo_data TO 'da_2021_cop.geojson'\n",
"WITH (\n",
" FORMAT GDAL,\n",
" DRIVER 'GeoJSON',\n",
" GEOMETRY_TYPE 'POLYGON',\n",
" SRS 'EPSG:4326'\n",
");\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "c32ab0f8",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c226f348d3f14c0abbeb2bfa907a7db6",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"<duckdb.duckdb.DuckDBPyConnection at 0x147a2c462b30>"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"con.execute(\"\"\"\n",
"COPY geo_data TO 'da_2021_cop_geom.parquet' (FORMAT PARQUET);\n",
"\"\"\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.9"
}
},
"nbformat": 4,
"nbformat_minor": 5
}