d4c-datapkg-statistical/experiments/duckdb_census_of_population.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "56ac906e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import duckdb\n",
    "\n",
    "output_data_folder = '/data/experiments'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "708e293d",
   "metadata": {},
   "outputs": [],
   "source": [
    "con = duckdb.connect()\n",
    "con.install_extension(\"spatial\")\n",
    "con.load_extension(\"spatial\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d97e882",
   "metadata": {},
   "source": [
    "# 1.0 Total private dwellings and private dwellings per square kilometer for Ottawa\n",
    "These values are from the 2021 Census of Population"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "580c82ad-f64d-439f-9055-2307fdf7cccd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d154b54d66c48d48637165066492606",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<duckdb.duckdb.DuckDBPyConnection at 0x7f0654926530>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con.execute(\"\"\"\n",
    "DROP TABLE IF EXISTS geo_data;\n",
    "CREATE TABLE geo_data AS\n",
    "SELECT\n",
    "    geo.da_dguid,\n",
    "    cop.count_total_4,\n",
    "    (cop.count_total_4 / (ST_Area_Spheroid(ST_FlipCoordinates(geo.geom)) / 1000000.0)) AS count_total_4_per_square_km,\n",
    "    geo.geom\n",
    "FROM\n",
    "    'https://data.dataforcanada.org/processed/statistics_canada/census_of_population/2021/tabular/da_2021.parquet' AS cop,\n",
    "    'https://data.dataforcanada.org/processed/statistics_canada/boundaries/2021/digital_boundary_files/da_2021.parquet' AS geo\n",
    "WHERE geo.csd_name IN ('Ottawa') AND cop.da_dguid = geo.da_dguid;\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c95fdadf-3dba-4328-8e1d-2a22baa3d293",
   "metadata": {},
   "source": [
    "## 1.1 Export result as a GeoJSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "0528eeb4-fb89-4640-94c6-c0eda4cbe764",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<duckdb.duckdb.DuckDBPyConnection at 0x7f0654926530>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con.execute(f\"\"\"\n",
    "COPY geo_data TO '{output_data_folder}/da_2021_private_dwellings.geojson'\n",
    "WITH (\n",
    "  FORMAT GDAL,\n",
    "  DRIVER 'GeoJSON',\n",
    "  GEOMETRY_TYPE 'POLYGON',\n",
    "  SRS 'EPSG:4326'\n",
    ");\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34ac7bd0-5068-4681-9405-ebe68f8dcf28",
   "metadata": {},
   "source": [
    "## 1.2 Export result as file geodatabase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "6d38118b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<duckdb.duckdb.DuckDBPyConnection at 0x7f0654926530>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con.execute(f\"\"\"\n",
    "COPY geo_data TO '{output_data_folder}/da_2021_private_dwellings.gdb'\n",
    "WITH (\n",
    "  FORMAT GDAL,\n",
    "  DRIVER 'OpenFileGDB',\n",
    "  GEOMETRY_TYPE 'POLYGON',\n",
    "  SRS 'EPSG:4326'\n",
    ");\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d57086e-31d5-4462-803d-38e60b74a043",
   "metadata": {},
   "source": [
    "## 1.3 Export result as GeoParquet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "c32ab0f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<duckdb.duckdb.DuckDBPyConnection at 0x7f0654926530>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con.execute(f\"\"\"\n",
    "COPY geo_data TO '{output_data_folder}/da_2021_private_dwellings.parquet' (FORMAT PARQUET);\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f519fe1-b641-4230-b3ed-645f1126e335",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}