d4c-datapkg-statistical/experiments/duckdb_census_of_population.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "56ac906e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import duckdb\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "708e293d",
   "metadata": {},
   "outputs": [],
   "source": [
    "con = duckdb.connect()\n",
    "con.install_extension(\"spatial\")\n",
    "con.load_extension(\"spatial\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d97e882",
   "metadata": {},
   "source": [
    "# DA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1b2ccf6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2fd24f9af2c45369a4aa93b249227c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<duckdb.duckdb.DuckDBPyConnection at 0x1474498ddc30>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con.execute(\"\"\"\n",
    "DROP TABLE IF EXISTS geo_data;\n",
    "CREATE TABLE geo_data AS\n",
    "SELECT da.da_dguid, da_cop.* EXCLUDE dguid, da.geom FROM 'https://data.dataforcanada.org/processed/statistics_canada/census_of_population/2021/tabular/da_2021.parquet' AS da_cop,\n",
    "'https://data.dataforcanada.org/processed/statistics_canada/boundaries/2021/digital_boundary_files/da_2021.parquet' AS da\n",
    "WHERE da_cop.dguid = da.da_dguid\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "33a3819e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(57936,)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con.execute(\"SELECT count(*) FROM geo_data;\")\n",
    "con.fetchall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6d38118b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee1b19d9ad3c4a9483ba3cef3600ba8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<duckdb.duckdb.DuckDBPyConnection at 0x1474498ddc30>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con.execute(\"\"\"\n",
    "COPY geo_data TO 'da_2021_cop.gdb'\n",
    "WITH (\n",
    "  FORMAT GDAL,\n",
    "  DRIVER 'OpenFileGDB',\n",
    "  GEOMETRY_TYPE 'POLYGON',\n",
    "  SRS 'EPSG:4326'\n",
    ");\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "76ada8f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4d6cb52cb864a0cac1941bb25315b79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<duckdb.duckdb.DuckDBPyConnection at 0x1477954f73b0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con.execute(\"\"\"\n",
    "COPY geo_data TO 'da_2021_cop.geojson'\n",
    "WITH (\n",
    "  FORMAT GDAL,\n",
    "  DRIVER 'GeoJSON',\n",
    "  GEOMETRY_TYPE 'POLYGON',\n",
    "  SRS 'EPSG:4326'\n",
    ");\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c32ab0f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c226f348d3f14c0abbeb2bfa907a7db6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<duckdb.duckdb.DuckDBPyConnection at 0x147a2c462b30>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con.execute(\"\"\"\n",
    "COPY geo_data TO 'da_2021_cop_geom.parquet' (FORMAT PARQUET);\n",
    "\"\"\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}