{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8b767621-b96b-4eaa-a16a-0329bab29c0f",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "# Notes\n",
    "- Make sure Statistics Canada knows about these issues:\n",
    "    - When downloading the XML for product_id 98100404 it just returns the structure document, not the data document\n",
    "    - The releaseTime value is in Eastern Standard Zone for https://www150.statcan.gc.ca/t1/wds/rest/getCubeMetadata and UTC for https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesListLite\n",
    "    - The releaseTime value is different when getting it from https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesListLite and https://www150.statcan.gc.ca/t1/wds/rest/getCubeMetadata . For example productId 10100007\n",
    "    - Currently having an issue with processing productId 13100442. Expected 18 fields but saw 19 at line 162\n",
    "    - There are cases where the `REF_DATE` is a range, ex. 2023/2024\n",
    "        - For productId 17100022 the period is from July 1 to June 30 (seen in the metadata), so can't just use January 1, 2023 and December 31, 2024\n",
    "    - There can be times when a table's `VALUE` field can have both float and integer data types. For example, productId 11100025 has `DECIMAL` values of 0 and 1, corresponding to integer and float data types\n",
    "    - Need to look into cases where there is no `DGUID`, but the `GEO` value tells you something like `All census metropolitan areas` (productId 11100025). I guess I can do the linking to the CMAs on the API side"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "98859cd6-6fa4-4aef-a113-455699524fae",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import glob\n",
    "from multiprocessing import Pool\n",
    "import json\n",
    "import os\n",
    "import sqlite3\n",
    "import zipfile\n",
    "from zoneinfo import ZoneInfo\n",
    "\n",
    "import pandas as pd\n",
    "import polars as pl\n",
    "import requests\n",
    "from tqdm import tqdm\n",
    "\n",
    "data_folder = \"/data/tables\"\n",
    "input_folder = f\"{data_folder}/input\"\n",
    "scratch_folder = f\"{data_folder}/scratch\"\n",
    "output_folder = f\"{data_folder}/output\"\n",
    "\n",
    "if not os.path.exists(f\"{data_folder}/processing.db\"):\n",
    "    con = sqlite3.connect(f\"{data_folder}/processing.db\", timeout=60.0)\n",
    "    cur = con.cursor()\n",
    "    cur.executescript(\"\"\"\n",
    "        CREATE TABLE IF NOT EXISTS downloaded (\n",
    "          product_id TEXT PRIMARY KEY,\n",
    "          last_updated TEXT,\n",
    "          last_processed TEXT\n",
    "        );\n",
    "\n",
    "        CREATE TABLE IF NOT EXISTS cubes (\n",
    "          product_id TEXT PRIMARY KEY,\n",
    "          last_updated TEXT\n",
    "        );\n",
    "    \"\"\")\n",
    "    con.commit()\n",
    "else:\n",
    "    con = sqlite3.connect(f\"{data_folder}/processing.db\")\n",
    "    cur = con.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "28ac4c01-c1c5-427f-bb2c-0da99a4c5591",
   "metadata": {},
   "outputs": [],
   "source": [
    "def setup():\n",
    "    \"\"\"\n",
    "    Makes data folders\n",
    "    \"\"\"\n",
    "    folders_to_create = [data_folder, input_folder, \n",
    "                         scratch_folder, output_folder,\n",
    "                         f\"{input_folder}/en\", f\"{output_folder}/en\",\n",
    "                         f\"{input_folder}/fr\", f\"{output_folder}/fr\",\n",
    "                         f\"{input_folder}/metadata\"]\n",
    "    for folder in folders_to_create:\n",
    "        if not os.path.exists(folder):\n",
    "            print(f\"Making folder {folder}\")\n",
    "            os.mkdir(folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9daa94f4-16c9-4d8b-951e-3d5d38eb618f",
   "metadata": {},
   "outputs": [],
   "source": [
    "setup()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0af9a4b3-7b59-460b-b933-504919d4bd2a",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def update_last_downloaded(product_id):\n",
    "    \"\"\"\n",
    "    Updates SQLite database with the last time the table was updated\n",
    "    The datetime is in Eastern timezone, so have to convert to UTC to\n",
    "    be consistent with https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesListLite\n",
    "    \"\"\"\n",
    "    filepath = f\"{input_folder}/metadata/{product_id}.json\"\n",
    "    print(f\"Reading metadata {filepath}\")\n",
    "    with open(filepath, 'r') as fp:\n",
    "        metadata = json.load(fp)\n",
    "    product_id = metadata.get(\"object\").get(\"productId\")\n",
    "    last_updated = metadata.get(\"object\").get(\"releaseTime\")\n",
    "    # Convert last_updated to UTC since /getAllcubesListLite uses UTC\n",
    "    last_updated = datetime.strptime(last_updated, \"%Y-%m-%dT%H:%M\")\n",
    "    last_updated = last_updated.replace(tzinfo=ZoneInfo(\"America/Toronto\"))\n",
    "    last_updated = last_updated.astimezone(ZoneInfo(\"UTC\")).isoformat()\n",
    "\n",
    "    data = (product_id, last_updated)\n",
    "    cur.execute(\"SELECT product_id FROM downloaded WHERE product_id = ?\", (product_id,))\n",
    "    result = cur.fetchone()\n",
    "    if not result:\n",
    "        cur.execute(\"INSERT INTO downloaded (product_id, last_updated) VALUES (?, ?)\", data)\n",
    "    else:\n",
    "        cur.execute(\"UPDATE downloaded SET last_updated = ? WHERE product_id = ?\", (last_updated, product_id))\n",
    "\n",
    "    con.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4b7996d2-75ab-4173-a17a-64fb7ab63740",
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_last_processed(product_id):\n",
    "    time_finished_processing = datetime.now().isoformat()\n",
    "    cur.execute(\"UPDATE downloaded SET last_processed = ? WHERE product_id = ?\", (time_finished_processing, product_id))\n",
    "    con.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "23cbabc3-0d4b-4e28-a4df-b2c5e8c7ea8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_tables():\n",
    "    \"\"\"\n",
    "    This currently does not work as expected because Statistics Canada has discrepancies.\n",
    "    The \"releaseTime\" listed in https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesListLite\n",
    "    for every pdocutId is not the same as \"releaseTime\" listed when making a POST \n",
    "    https://www150.statcan.gc.ca/t1/wds/rest/getCubeMetadata , for example:\n",
    "    [{\"productId\":10100007}]\n",
    "    \"\"\"\n",
    "    cur.execute(\"\"\"\n",
    "    DELETE FROM cubes;\n",
    "    \"\"\")\n",
    "    con.commit()\n",
    "    response = requests.get(\"https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesListLite\").json()\n",
    "    cubes_metadata = pl.from_dicts(response)[['productId', 'releaseTime']]\n",
    "    cubes_metadata = cubes_metadata.rename({\"productId\": \"product_id\", \"releaseTime\": \"last_updated\"})\n",
    "    cubes_metadata = cubes_metadata.rows()\n",
    "    cubes_metadata_new =  []\n",
    "    for cube in cubes_metadata:\n",
    "        product_id, last_updated = cube\n",
    "        # Update the date field so it is formatted the same as date field in downloaded table\n",
    "        last_updated = datetime.strptime(last_updated, \"%Y-%m-%dT%H:%M:%SZ\").astimezone(ZoneInfo(\"UTC\"))\n",
    "        last_updated = last_updated.isoformat()\n",
    "        cubes_metadata_new.append((product_id, last_updated))\n",
    "        \n",
    "    cur.executemany(\"INSERT INTO cubes VALUES(?, ?)\", cubes_metadata_new)\n",
    "    con.commit()\n",
    "\n",
    "    cur.execute(\"\"\"\n",
    "    SELECT a.product_id\n",
    "    FROM downloaded AS a,\n",
    "         cubes AS b\n",
    "    WHERE a.product_id = b.product_id\n",
    "    AND b.last_updated > a.last_updated\n",
    "    \"\"\")\n",
    "    \"\"\"\n",
    "    results = cur.fetchall()\n",
    "    for result in results:\n",
    "        product_id = result[0]\n",
    "        print(f\"Updating product_id: {product_id}\")\n",
    "        download_cube(product_id)\n",
    "        process_cube(product_id)\n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dc5573ef-734b-44d8-a4c4-0df19d655975",
   "metadata": {},
   "outputs": [],
   "source": [
    "update_tables()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eddf6501-8428-44cc-8d2d-e245803a3943",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def compute_ref_date_bounds(df):\n",
    "    \"\"\"\n",
    "    TODO: There are cases where the REF_DATE is a range, ex. 2023/2024.\n",
    "    For productId 17100022 the period is from July 1 to June 30 (seen in the metadata), so can't just \n",
    "    use January 1, 2023 and December 31, 2024\n",
    "    \"\"\"\n",
    "    series = df[\"REF_DATE\"]\n",
    "\n",
    "    # Initialize the two new columns with NaT\n",
    "    df[\"REF_START_DATE\"] = pd.NaT\n",
    "    df[\"REF_END_DATE\"] = pd.NaT\n",
    "\n",
    "    # Skip rows that contain slashes\n",
    "    valid_mask = ~series.str.contains(\"/\", na=False)\n",
    "\n",
    "    # Case 1: YYYY-MM-DD\n",
    "    full_mask = valid_mask & series.str.fullmatch(r\"\\d{4}-\\d{2}-\\d{2}\")\n",
    "    parsed_full = pd.to_datetime(series[full_mask], format=\"%Y-%m-%d\", errors=\"coerce\")\n",
    "    df.loc[full_mask, \"REF_START_DATE\"] = parsed_full\n",
    "    df.loc[full_mask, \"REF_END_DATE\"] = parsed_full\n",
    "\n",
    "    # Case 2: YYYY-MM\n",
    "    month_mask = valid_mask & series.str.fullmatch(r\"\\d{4}-\\d{2}\")\n",
    "    parsed_month = pd.to_datetime(series[month_mask], format=\"%Y-%m\", errors=\"coerce\")\n",
    "    df.loc[month_mask, \"REF_START_DATE\"] = parsed_month\n",
    "    df.loc[month_mask, \"REF_END_DATE\"] = parsed_month + pd.to_timedelta(\n",
    "        parsed_month.dt.days_in_month - 1, unit='D'\n",
    "    )\n",
    "\n",
    "    # Case 3: YYYY\n",
    "    year_mask = valid_mask & series.str.fullmatch(r\"\\d{4}\")\n",
    "    parsed_year = pd.to_datetime(series[year_mask], format=\"%Y\", errors=\"coerce\")\n",
    "    df.loc[year_mask, \"REF_START_DATE\"] = parsed_year\n",
    "    df.loc[year_mask, \"REF_END_DATE\"] = parsed_year + pd.offsets.YearEnd(0)\n",
    "\n",
    "    # Move columns after REF_DATE\n",
    "    ref_idx = df.columns.get_loc(\"REF_DATE\")\n",
    "    cols = list(df.columns)\n",
    "    cols.remove(\"REF_START_DATE\")\n",
    "    cols.remove(\"REF_END_DATE\")\n",
    "    cols[ref_idx + 1:ref_idx + 1] = [\"REF_START_DATE\", \"REF_END_DATE\"]\n",
    "\n",
    "    return df[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e67642d3-cc6c-4c5d-b5a3-2fe18364ad71",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_lowest_type(df):\n",
    "    \"\"\"\n",
    "    Convert columns to the best possible dtypes\n",
    "    For example, if the column is numerical and has a maximum value of 32,000 \n",
    "    we can assign it a type of int16\n",
    "    \"\"\"\n",
    "    dtypes = pd.DataFrame(df.dtypes)\n",
    "    # Downcast to the smallest numerical dtype\n",
    "    for row in dtypes.itertuples():\n",
    "        column = row[0]\n",
    "        the_type = str(row[1])\n",
    "        if the_type == 'Int64':\n",
    "            df[column] = pd.to_numeric(df[column], downcast='integer')\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "144e3716-b0e7-4a39-9a25-ededea506f4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_zipfile(product_id, language):\n",
    "    \"\"\"\n",
    "    It is faster to extract the zip file and read the CSV, than open\n",
    "    via zipfile and then Pandas\n",
    "    \"\"\"\n",
    "    zip_file = f\"{input_folder}/{language}/{product_id}.zip\"\n",
    "    with zipfile.ZipFile(zip_file) as myzip:\n",
    "        print(f\"Extracting {zip_file} to {scratch_folder}\")\n",
    "        myzip.extractall(path=scratch_folder)\n",
    "\n",
    "def get_cube_metadata(product_id):\n",
    "    url = f\"https://www150.statcan.gc.ca/t1/wds/rest/getCubeMetadata\"\n",
    "    cubes_payload = [{\"productId\": product_id}]\n",
    "    result = requests.post(url, json=cubes_payload)\n",
    "    result = result.json()[0]\n",
    "    return result\n",
    "\n",
    "def download_cube(product_id, language=\"en\"):\n",
    "    \"\"\"\n",
    "    Downloads the English CSV for a specific table\n",
    "    \"\"\"\n",
    "    download_url = f\"https://www150.statcan.gc.ca/t1/wds/rest/getFullTableDownloadCSV/{product_id}/en\"\n",
    "    response = requests.get(download_url).json()\n",
    "    zip_url = response['object']\n",
    "    zip_file_name = f\"{input_folder}/{language}/{product_id}.zip\"\n",
    "    print(f\"Downloading {zip_url} to {zip_file_name}\")\n",
    "    response = requests.get(zip_url, stream=True, headers={\"user-agent\": None})\n",
    "    progress_bar = tqdm(\n",
    "        desc=zip_file_name,\n",
    "        total=int(response.headers.get(\"content-length\", 0)),\n",
    "        unit=\"B\",\n",
    "        unit_scale=True\n",
    "    )\n",
    "    with open(zip_file_name, \"wb\") as handle:\n",
    "        for chunk in response.iter_content(chunk_size=512):\n",
    "            if chunk:  # filter out keep-alive new chunks\n",
    "                handle.write(chunk)\n",
    "                progress_bar.update(len(chunk))\n",
    "        progress_bar.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aa16ea14-1661-4dc8-8e2f-a09175b1cd17",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleanup_product(product_id):\n",
    "    \"\"\"\n",
    "    Remove the scratch files for a given productId\n",
    "    \"\"\"\n",
    "    print(f\"Removing scratch files for productId {product_id}\")\n",
    "    os.remove(f\"{scratch_folder}/{product_id}.csv\")\n",
    "    os.remove(f\"{scratch_folder}/{product_id}_MetaData.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "858e405e-7c02-4193-8abe-f23951761b09",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /data/tables/input/en/98100001.zip to /data/tables/scratch\n",
      "Reading /data/tables/scratch/98100001.csv\n",
      "Index(['REF_DATE', 'GEO', 'DGUID', 'Coordinate',\n",
      "       'Population and dwelling counts (11): Population, 2021 [1]', 'Symbols',\n",
      "       'Population and dwelling counts (11): Population, 2016 [2]',\n",
      "       'Symbols.1',\n",
      "       'Population and dwelling counts (11): Population percentage change, 2016 to 2021 [3]',\n",
      "       'Symbols.2',\n",
      "       'Population and dwelling counts (11): Total private dwellings, 2021 [4]',\n",
      "       'Symbols.3',\n",
      "       'Population and dwelling counts (11): Total private dwellings, 2016 [5]',\n",
      "       'Symbols.4',\n",
      "       'Population and dwelling counts (11): Total private dwellings percentage change, 2016 to 2021 [6]',\n",
      "       'Symbols.5',\n",
      "       'Population and dwelling counts (11): Private dwellings occupied by usual residents, 2021 [7]',\n",
      "       'Symbols.6',\n",
      "       'Population and dwelling counts (11): Private dwellings occupied by usual residents, 2016 [8]',\n",
      "       'Symbols.7',\n",
      "       'Population and dwelling counts (11): Private dwellings occupied by usual residents percentage change, 2016 to 2021 [9]',\n",
      "       'Symbols.8',\n",
      "       'Population and dwelling counts (11): Land area in square kilometres, 2021 [10]',\n",
      "       'Symbols.9',\n",
      "       'Population and dwelling counts (11): Population density per square kilometre, 2021 [11]',\n",
      "       'Symbols.10'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Examples: \n",
    "- productId 43100011 has all with DECIMAL = 1 (float64)\n",
    "- productId 17100009 has DECIMAL = 0 (int64)\n",
    "- productId 35100076 has multiple DECIMAL precisions [0, 1, 2] (int64, float64, float64)\n",
    "- The duplicate column issue just happens with two column names in all data products (\"Value\", \"VALUE\", and \"Status\", \"STATUS\")\n",
    "    - productId 10100164 has two columns named the same \"Value\" and \"VALUE\". DuckDB treats column names in a case insensitve manner, so \n",
    "    \"Value\" and \"VALUE\" are the same. So we will need to rename \"Value\" to \"Value.1\"\n",
    "    - productId 13100902 has two columns named the same \"Status\" and \"STATUS\". We will need to rename \"Status\" to \"STATUS\"\n",
    "- productId 13100442 has 18 fields, but 19 fields were seen in line 162\n",
    "- There are cases where the \"DECIMALS\" column does not exist in the CSV. productId 98100001 is one example.\n",
    "  In this case, we do let the .read_csv method guess the data types\n",
    "\"\"\"\n",
    "\n",
    "product_id = \"98100001\"\n",
    "#def process_cube(product_id, language=\"en\"):\n",
    "language = \"en\"\n",
    "cur.execute(\"SELECT product_id FROM downloaded WHERE product_id = ?\", (product_id,))\n",
    "result = cur.fetchone()\n",
    "if result:\n",
    "    print(f\"Already processed {product_id}\")\n",
    "    #return\n",
    "extract_zipfile(product_id, language)\n",
    "\"\"\"\n",
    "The pandas column reader is better than the Polars one\n",
    "Here is an example where polars was not reading it right:\n",
    "https://www150.statcan.gc.ca/n1/tbl/csv/98100404-eng.zip\n",
    "\"\"\"\n",
    "# Get metadata\n",
    "#metadata_file = f\"{input_folder}/metadata/{product_id}.json\"\n",
    "#metadata = get_cube_metadata(product_id)\n",
    "#print(f\"Writing metadata file {metadata_file}\")\n",
    "#with open(metadata_file, \"w\") as outfile:\n",
    "#    json.dump(metadata, outfile)\n",
    "# Read CSV using Pandas\n",
    "product_csv = f\"{scratch_folder}/{product_id}.csv\"\n",
    "print(f\"Reading {product_csv}\")\n",
    "parameters = {\n",
    "    \"filepath_or_buffer\": product_csv,\n",
    "    \"engine\": \"c\",\n",
    "    \"nrows\": 100000,\n",
    "    \"dtype\": {}\n",
    "}\n",
    "\n",
    "columns = pd.read_csv(product_csv, nrows=0).columns\n",
    "print(columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "59f77277-f8cd-4eee-93b2-c390ce136f84",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_rename = ['Value', 'Status']\n",
    "for column in columns_to_rename:\n",
    "    if column in columns:\n",
    "        print(f\"Renaming '{column}' to '{column}.1'\")\n",
    "        columns = [f'{column}.1' if x == column else x for x in columns]\n",
    "        # Explicitly tell pandas to not read column names from CSV\n",
    "        parameters[\"header\"] = 0\n",
    "        parameters[\"names\"] = columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c1f89175-78e5-4e95-8a3e-3b65f0cb4b2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'filepath_or_buffer': '/data/tables/scratch/98100001.csv', 'engine': 'c', 'nrows': 100000, 'dtype': {'REF_DATE': 'string', 'GEO': 'string', 'DGUID': 'string'}}\n",
      "Reading /data/tables/scratch/98100001.csv as a Pandas dataframe\n"
     ]
    }
   ],
   "source": [
    "columns_always_int_8 = [\"DECIMALS\", \"SCALAR_ID\"]\n",
    "for column in columns_always_int_8:\n",
    "    if column in columns:\n",
    "        parameters[\"dtype\"][column] = 'int8'\n",
    "\n",
    "columns_always_int_16 = [\"UOM_ID\"]\n",
    "for column in columns_always_int_16:\n",
    "    if column in columns:\n",
    "        parameters[\"dtype\"][column] = 'int16'\n",
    "\n",
    "# GEO, DGUID should always be string\n",
    "columns_always_string = [\"REF_DATE\", \"GEO\", \"DGUID\"]\n",
    "for column in columns_always_string:\n",
    "    if column in columns:\n",
    "        parameters[\"dtype\"][column] = 'string'\n",
    "\n",
    "# The remaining columns should be string, with the exception of VALUE\n",
    "# Added \"DECIMAL\" check as there can be numeric columns that are not the VALUE column\n",
    "if \"DECIMALS\" in columns:\n",
    "    for column in columns:\n",
    "        if column not in columns_always_int_8 and column not in columns_always_int_16 and column != \"VALUE\":\n",
    "            parameters[\"dtype\"][column] = 'string'\n",
    "\n",
    "print(parameters)\n",
    "if not parameters[\"dtype\"]:\n",
    "    del parameters[\"dtype\"]\n",
    "\n",
    "print(f\"Reading {product_csv} as a Pandas dataframe\")\n",
    "df = pd.read_csv(**parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "87ff5f69-ca1f-40e0-ac73-c73dd8c1bd4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>REF_DATE</th>\n",
       "      <th>GEO</th>\n",
       "      <th>DGUID</th>\n",
       "      <th>Coordinate</th>\n",
       "      <th>Population and dwelling counts (11): Population, 2021 [1]</th>\n",
       "      <th>Symbols</th>\n",
       "      <th>Population and dwelling counts (11): Population, 2016 [2]</th>\n",
       "      <th>Symbols.1</th>\n",
       "      <th>Population and dwelling counts (11): Population percentage change, 2016 to 2021 [3]</th>\n",
       "      <th>Symbols.2</th>\n",
       "      <th>...</th>\n",
       "      <th>Population and dwelling counts (11): Private dwellings occupied by usual residents, 2021 [7]</th>\n",
       "      <th>Symbols.6</th>\n",
       "      <th>Population and dwelling counts (11): Private dwellings occupied by usual residents, 2016 [8]</th>\n",
       "      <th>Symbols.7</th>\n",
       "      <th>Population and dwelling counts (11): Private dwellings occupied by usual residents percentage change, 2016 to 2021 [9]</th>\n",
       "      <th>Symbols.8</th>\n",
       "      <th>Population and dwelling counts (11): Land area in square kilometres, 2021 [10]</th>\n",
       "      <th>Symbols.9</th>\n",
       "      <th>Population and dwelling counts (11): Population density per square kilometre, 2021 [11]</th>\n",
       "      <th>Symbols.10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021</td>\n",
       "      <td>Canada</td>\n",
       "      <td>2021A000011124</td>\n",
       "      <td>1</td>\n",
       "      <td>36991981</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35151728</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>14978941</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14072079</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8788702.80</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.2</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021</td>\n",
       "      <td>Newfoundland and Labrador</td>\n",
       "      <td>2021A000210</td>\n",
       "      <td>2</td>\n",
       "      <td>510550</td>\n",
       "      <td>NaN</td>\n",
       "      <td>519716</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>223253</td>\n",
       "      <td>NaN</td>\n",
       "      <td>218673</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>358170.37</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021</td>\n",
       "      <td>Prince Edward Island</td>\n",
       "      <td>2021A000211</td>\n",
       "      <td>3</td>\n",
       "      <td>154331</td>\n",
       "      <td>NaN</td>\n",
       "      <td>142907</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>64570</td>\n",
       "      <td>NaN</td>\n",
       "      <td>59472</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5681.18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.2</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021</td>\n",
       "      <td>Nova Scotia</td>\n",
       "      <td>2021A000212</td>\n",
       "      <td>4</td>\n",
       "      <td>969383</td>\n",
       "      <td>NaN</td>\n",
       "      <td>923598</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>428228</td>\n",
       "      <td>NaN</td>\n",
       "      <td>401990</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52824.71</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18.4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021</td>\n",
       "      <td>New Brunswick</td>\n",
       "      <td>2021A000213</td>\n",
       "      <td>5</td>\n",
       "      <td>775610</td>\n",
       "      <td>NaN</td>\n",
       "      <td>747101</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>337651</td>\n",
       "      <td>NaN</td>\n",
       "      <td>319773</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>71248.50</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.9</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  REF_DATE                        GEO           DGUID  Coordinate  \\\n",
       "0     2021                     Canada  2021A000011124           1   \n",
       "1     2021  Newfoundland and Labrador     2021A000210           2   \n",
       "2     2021       Prince Edward Island     2021A000211           3   \n",
       "3     2021                Nova Scotia     2021A000212           4   \n",
       "4     2021              New Brunswick     2021A000213           5   \n",
       "\n",
       "   Population and dwelling counts (11): Population, 2021 [1]  Symbols  \\\n",
       "0                                           36991981              NaN   \n",
       "1                                             510550              NaN   \n",
       "2                                             154331              NaN   \n",
       "3                                             969383              NaN   \n",
       "4                                             775610              NaN   \n",
       "\n",
       "   Population and dwelling counts (11): Population, 2016 [2]  Symbols.1  \\\n",
       "0                                           35151728                NaN   \n",
       "1                                             519716                NaN   \n",
       "2                                             142907                NaN   \n",
       "3                                             923598                NaN   \n",
       "4                                             747101                NaN   \n",
       "\n",
       "   Population and dwelling counts (11): Population percentage change, 2016 to 2021 [3]  \\\n",
       "0                                                5.2                                     \n",
       "1                                               -1.8                                     \n",
       "2                                                8.0                                     \n",
       "3                                                5.0                                     \n",
       "4                                                3.8                                     \n",
       "\n",
       "   Symbols.2  ...  \\\n",
       "0        NaN  ...   \n",
       "1        NaN  ...   \n",
       "2        NaN  ...   \n",
       "3        NaN  ...   \n",
       "4        NaN  ...   \n",
       "\n",
       "   Population and dwelling counts (11): Private dwellings occupied by usual residents, 2021 [7]  \\\n",
       "0                                           14978941                                              \n",
       "1                                             223253                                              \n",
       "2                                              64570                                              \n",
       "3                                             428228                                              \n",
       "4                                             337651                                              \n",
       "\n",
       "   Symbols.6  \\\n",
       "0        NaN   \n",
       "1        NaN   \n",
       "2        NaN   \n",
       "3        NaN   \n",
       "4        NaN   \n",
       "\n",
       "   Population and dwelling counts (11): Private dwellings occupied by usual residents, 2016 [8]  \\\n",
       "0                                           14072079                                              \n",
       "1                                             218673                                              \n",
       "2                                              59472                                              \n",
       "3                                             401990                                              \n",
       "4                                             319773                                              \n",
       "\n",
       "   Symbols.7  \\\n",
       "0        NaN   \n",
       "1        NaN   \n",
       "2        NaN   \n",
       "3        NaN   \n",
       "4        NaN   \n",
       "\n",
       "   Population and dwelling counts (11): Private dwellings occupied by usual residents percentage change, 2016 to 2021 [9]  \\\n",
       "0                                                6.4                                                                        \n",
       "1                                                2.1                                                                        \n",
       "2                                                8.6                                                                        \n",
       "3                                                6.5                                                                        \n",
       "4                                                5.6                                                                        \n",
       "\n",
       "   Symbols.8  \\\n",
       "0        NaN   \n",
       "1        NaN   \n",
       "2        NaN   \n",
       "3        NaN   \n",
       "4        NaN   \n",
       "\n",
       "   Population and dwelling counts (11): Land area in square kilometres, 2021 [10]  \\\n",
       "0                                         8788702.80                                \n",
       "1                                          358170.37                                \n",
       "2                                            5681.18                                \n",
       "3                                           52824.71                                \n",
       "4                                           71248.50                                \n",
       "\n",
       "   Symbols.9  \\\n",
       "0        NaN   \n",
       "1        NaN   \n",
       "2        NaN   \n",
       "3        NaN   \n",
       "4        NaN   \n",
       "\n",
       "   Population and dwelling counts (11): Population density per square kilometre, 2021 [11]  \\\n",
       "0                                                4.2                                         \n",
       "1                                                1.4                                         \n",
       "2                                               27.2                                         \n",
       "3                                               18.4                                         \n",
       "4                                               10.9                                         \n",
       "\n",
       "   Symbols.10  \n",
       "0         NaN  \n",
       "1         NaN  \n",
       "2         NaN  \n",
       "3         NaN  \n",
       "4         NaN  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fde0e149-d146-4516-8966-0989e4ccf290",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "REF_DATE                                                                                                                  string[python]\n",
       "GEO                                                                                                                       string[python]\n",
       "DGUID                                                                                                                     string[python]\n",
       "Coordinate                                                                                                                         int64\n",
       "Population and dwelling counts (11): Population, 2021 [1]                                                                          int64\n",
       "Symbols                                                                                                                          float64\n",
       "Population and dwelling counts (11): Population, 2016 [2]                                                                          int64\n",
       "Symbols.1                                                                                                                        float64\n",
       "Population and dwelling counts (11): Population percentage change, 2016 to 2021 [3]                                              float64\n",
       "Symbols.2                                                                                                                        float64\n",
       "Population and dwelling counts (11): Total private dwellings, 2021 [4]                                                             int64\n",
       "Symbols.3                                                                                                                        float64\n",
       "Population and dwelling counts (11): Total private dwellings, 2016 [5]                                                             int64\n",
       "Symbols.4                                                                                                                        float64\n",
       "Population and dwelling counts (11): Total private dwellings percentage change, 2016 to 2021 [6]                                 float64\n",
       "Symbols.5                                                                                                                        float64\n",
       "Population and dwelling counts (11): Private dwellings occupied by usual residents, 2021 [7]                                       int64\n",
       "Symbols.6                                                                                                                        float64\n",
       "Population and dwelling counts (11): Private dwellings occupied by usual residents, 2016 [8]                                       int64\n",
       "Symbols.7                                                                                                                        float64\n",
       "Population and dwelling counts (11): Private dwellings occupied by usual residents percentage change, 2016 to 2021 [9]           float64\n",
       "Symbols.8                                                                                                                        float64\n",
       "Population and dwelling counts (11): Land area in square kilometres, 2021 [10]                                                   float64\n",
       "Symbols.9                                                                                                                        float64\n",
       "Population and dwelling counts (11): Population density per square kilometre, 2021 [11]                                          float64\n",
       "Symbols.10                                                                                                                       float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7579a135-1dfe-4fc0-991b-4b261d6577e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DECIMALS not in columns, using .convert_dtypes\n"
     ]
    }
   ],
   "source": [
    "if \"DECIMALS\" in columns:\n",
    "    unique_decimal_values = df[\"DECIMALS\"].unique()\n",
    "    print(unique_decimal_values)\n",
    "    if any(unique_decimal_values):\n",
    "        \"\"\"\n",
    "        A table can have both float and integer in the VALUE field. \n",
    "        productId 11100025 is an example\n",
    "        So if we have unique values for DECIMALS to be [0,1], then we convert to float64\n",
    "        \"\"\"\n",
    "        convert_dict = {\"VALUE\": \"float64\"}\n",
    "        print(convert_dict)\n",
    "        df = df.astype(convert_dict)\n",
    "    elif 0 in (unique_decimal_values):\n",
    "        if df[\"VALUE\"].dtype != \"Int64\":\n",
    "            # If DECIMALS = [0]\n",
    "            convert_dict = {\"VALUE\": \"Int64\"}\n",
    "            print(convert_dict)\n",
    "            df = df.astype(convert_dict)\n",
    "else:\n",
    "    parameters = {\n",
    "        \"convert_string\": True,\n",
    "        \"convert_boolean\": False\n",
    "    }\n",
    "    print(\"DECIMALS not in columns, using .convert_dtypes\")\n",
    "    df = df.convert_dtypes(**parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "07b6c4e9-75bd-485e-92b8-154c4f55ce51",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "REF_DATE                                                                                                                  string[python]\n",
       "GEO                                                                                                                       string[python]\n",
       "DGUID                                                                                                                     string[python]\n",
       "Coordinate                                                                                                                         Int64\n",
       "Population and dwelling counts (11): Population, 2021 [1]                                                                          Int64\n",
       "Symbols                                                                                                                            Int64\n",
       "Population and dwelling counts (11): Population, 2016 [2]                                                                          Int64\n",
       "Symbols.1                                                                                                                          Int64\n",
       "Population and dwelling counts (11): Population percentage change, 2016 to 2021 [3]                                              Float64\n",
       "Symbols.2                                                                                                                          Int64\n",
       "Population and dwelling counts (11): Total private dwellings, 2021 [4]                                                             Int64\n",
       "Symbols.3                                                                                                                          Int64\n",
       "Population and dwelling counts (11): Total private dwellings, 2016 [5]                                                             Int64\n",
       "Symbols.4                                                                                                                          Int64\n",
       "Population and dwelling counts (11): Total private dwellings percentage change, 2016 to 2021 [6]                                 Float64\n",
       "Symbols.5                                                                                                                          Int64\n",
       "Population and dwelling counts (11): Private dwellings occupied by usual residents, 2021 [7]                                       Int64\n",
       "Symbols.6                                                                                                                          Int64\n",
       "Population and dwelling counts (11): Private dwellings occupied by usual residents, 2016 [8]                                       Int64\n",
       "Symbols.7                                                                                                                          Int64\n",
       "Population and dwelling counts (11): Private dwellings occupied by usual residents percentage change, 2016 to 2021 [9]           Float64\n",
       "Symbols.8                                                                                                                          Int64\n",
       "Population and dwelling counts (11): Land area in square kilometres, 2021 [10]                                                   Float64\n",
       "Symbols.9                                                                                                                          Int64\n",
       "Population and dwelling counts (11): Population density per square kilometre, 2021 [11]                                          Float64\n",
       "Symbols.10                                                                                                                         Int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fe87b3b0-4e04-41a8-a704-75cb9829b0a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = convert_to_lowest_type(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "74507546-4080-4962-88fc-58c1e3943d17",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = compute_ref_date_bounds(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6c2781b3-8eea-4317-a8c0-083d97ee04fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>REF_DATE</th>\n",
       "      <th>REF_START_DATE</th>\n",
       "      <th>REF_END_DATE</th>\n",
       "      <th>GEO</th>\n",
       "      <th>DGUID</th>\n",
       "      <th>Coordinate</th>\n",
       "      <th>Population and dwelling counts (11): Population, 2021 [1]</th>\n",
       "      <th>Symbols</th>\n",
       "      <th>Population and dwelling counts (11): Population, 2016 [2]</th>\n",
       "      <th>Symbols.1</th>\n",
       "      <th>...</th>\n",
       "      <th>Population and dwelling counts (11): Private dwellings occupied by usual residents, 2021 [7]</th>\n",
       "      <th>Symbols.6</th>\n",
       "      <th>Population and dwelling counts (11): Private dwellings occupied by usual residents, 2016 [8]</th>\n",
       "      <th>Symbols.7</th>\n",
       "      <th>Population and dwelling counts (11): Private dwellings occupied by usual residents percentage change, 2016 to 2021 [9]</th>\n",
       "      <th>Symbols.8</th>\n",
       "      <th>Population and dwelling counts (11): Land area in square kilometres, 2021 [10]</th>\n",
       "      <th>Symbols.9</th>\n",
       "      <th>Population and dwelling counts (11): Population density per square kilometre, 2021 [11]</th>\n",
       "      <th>Symbols.10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021</td>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>2021-12-31</td>\n",
       "      <td>Canada</td>\n",
       "      <td>2021A000011124</td>\n",
       "      <td>1</td>\n",
       "      <td>36991981</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>35151728</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>...</td>\n",
       "      <td>14978941</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>14072079</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>6.4</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>8788702.8</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>4.2</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021</td>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>2021-12-31</td>\n",
       "      <td>Newfoundland and Labrador</td>\n",
       "      <td>2021A000210</td>\n",
       "      <td>2</td>\n",
       "      <td>510550</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>519716</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>...</td>\n",
       "      <td>223253</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>218673</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2.1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>358170.37</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>1.4</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021</td>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>2021-12-31</td>\n",
       "      <td>Prince Edward Island</td>\n",
       "      <td>2021A000211</td>\n",
       "      <td>3</td>\n",
       "      <td>154331</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>142907</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>...</td>\n",
       "      <td>64570</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>59472</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>8.6</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>5681.18</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>27.2</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021</td>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>2021-12-31</td>\n",
       "      <td>Nova Scotia</td>\n",
       "      <td>2021A000212</td>\n",
       "      <td>4</td>\n",
       "      <td>969383</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>923598</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>...</td>\n",
       "      <td>428228</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>401990</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>6.5</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>52824.71</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>18.4</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021</td>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>2021-12-31</td>\n",
       "      <td>New Brunswick</td>\n",
       "      <td>2021A000213</td>\n",
       "      <td>5</td>\n",
       "      <td>775610</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>747101</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>...</td>\n",
       "      <td>337651</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>319773</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>5.6</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>71248.5</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>10.9</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  REF_DATE REF_START_DATE REF_END_DATE                        GEO  \\\n",
       "0     2021     2021-01-01   2021-12-31                     Canada   \n",
       "1     2021     2021-01-01   2021-12-31  Newfoundland and Labrador   \n",
       "2     2021     2021-01-01   2021-12-31       Prince Edward Island   \n",
       "3     2021     2021-01-01   2021-12-31                Nova Scotia   \n",
       "4     2021     2021-01-01   2021-12-31              New Brunswick   \n",
       "\n",
       "            DGUID  Coordinate  \\\n",
       "0  2021A000011124           1   \n",
       "1     2021A000210           2   \n",
       "2     2021A000211           3   \n",
       "3     2021A000212           4   \n",
       "4     2021A000213           5   \n",
       "\n",
       "   Population and dwelling counts (11): Population, 2021 [1]  Symbols  \\\n",
       "0                                           36991981             <NA>   \n",
       "1                                             510550             <NA>   \n",
       "2                                             154331             <NA>   \n",
       "3                                             969383             <NA>   \n",
       "4                                             775610             <NA>   \n",
       "\n",
       "   Population and dwelling counts (11): Population, 2016 [2]  Symbols.1  ...  \\\n",
       "0                                           35151728               <NA>  ...   \n",
       "1                                             519716               <NA>  ...   \n",
       "2                                             142907               <NA>  ...   \n",
       "3                                             923598               <NA>  ...   \n",
       "4                                             747101               <NA>  ...   \n",
       "\n",
       "   Population and dwelling counts (11): Private dwellings occupied by usual residents, 2021 [7]  \\\n",
       "0                                           14978941                                              \n",
       "1                                             223253                                              \n",
       "2                                              64570                                              \n",
       "3                                             428228                                              \n",
       "4                                             337651                                              \n",
       "\n",
       "   Symbols.6  \\\n",
       "0       <NA>   \n",
       "1       <NA>   \n",
       "2       <NA>   \n",
       "3       <NA>   \n",
       "4       <NA>   \n",
       "\n",
       "   Population and dwelling counts (11): Private dwellings occupied by usual residents, 2016 [8]  \\\n",
       "0                                           14072079                                              \n",
       "1                                             218673                                              \n",
       "2                                              59472                                              \n",
       "3                                             401990                                              \n",
       "4                                             319773                                              \n",
       "\n",
       "   Symbols.7  \\\n",
       "0       <NA>   \n",
       "1       <NA>   \n",
       "2       <NA>   \n",
       "3       <NA>   \n",
       "4       <NA>   \n",
       "\n",
       "   Population and dwelling counts (11): Private dwellings occupied by usual residents percentage change, 2016 to 2021 [9]  \\\n",
       "0                                                6.4                                                                        \n",
       "1                                                2.1                                                                        \n",
       "2                                                8.6                                                                        \n",
       "3                                                6.5                                                                        \n",
       "4                                                5.6                                                                        \n",
       "\n",
       "   Symbols.8  \\\n",
       "0       <NA>   \n",
       "1       <NA>   \n",
       "2       <NA>   \n",
       "3       <NA>   \n",
       "4       <NA>   \n",
       "\n",
       "   Population and dwelling counts (11): Land area in square kilometres, 2021 [10]  \\\n",
       "0                                          8788702.8                                \n",
       "1                                          358170.37                                \n",
       "2                                            5681.18                                \n",
       "3                                           52824.71                                \n",
       "4                                            71248.5                                \n",
       "\n",
       "   Symbols.9  \\\n",
       "0       <NA>   \n",
       "1       <NA>   \n",
       "2       <NA>   \n",
       "3       <NA>   \n",
       "4       <NA>   \n",
       "\n",
       "   Population and dwelling counts (11): Population density per square kilometre, 2021 [11]  \\\n",
       "0                                                4.2                                         \n",
       "1                                                1.4                                         \n",
       "2                                               27.2                                         \n",
       "3                                               18.4                                         \n",
       "4                                               10.9                                         \n",
       "\n",
       "   Symbols.10  \n",
       "0        <NA>  \n",
       "1        <NA>  \n",
       "2        <NA>  \n",
       "3        <NA>  \n",
       "4        <NA>  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1043bfbc-201d-4ef0-9ecb-f8902ceb8106",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "REF_DATE                                                                                                                  string[python]\n",
       "REF_START_DATE                                                                                                            datetime64[ns]\n",
       "REF_END_DATE                                                                                                              datetime64[ns]\n",
       "GEO                                                                                                                       string[python]\n",
       "DGUID                                                                                                                     string[python]\n",
       "Coordinate                                                                                                                          Int8\n",
       "Population and dwelling counts (11): Population, 2021 [1]                                                                          Int32\n",
       "Symbols                                                                                                                             Int8\n",
       "Population and dwelling counts (11): Population, 2016 [2]                                                                          Int32\n",
       "Symbols.1                                                                                                                           Int8\n",
       "Population and dwelling counts (11): Population percentage change, 2016 to 2021 [3]                                              Float64\n",
       "Symbols.2                                                                                                                           Int8\n",
       "Population and dwelling counts (11): Total private dwellings, 2021 [4]                                                             Int32\n",
       "Symbols.3                                                                                                                           Int8\n",
       "Population and dwelling counts (11): Total private dwellings, 2016 [5]                                                             Int32\n",
       "Symbols.4                                                                                                                           Int8\n",
       "Population and dwelling counts (11): Total private dwellings percentage change, 2016 to 2021 [6]                                 Float64\n",
       "Symbols.5                                                                                                                           Int8\n",
       "Population and dwelling counts (11): Private dwellings occupied by usual residents, 2021 [7]                                       Int32\n",
       "Symbols.6                                                                                                                           Int8\n",
       "Population and dwelling counts (11): Private dwellings occupied by usual residents, 2016 [8]                                       Int32\n",
       "Symbols.7                                                                                                                           Int8\n",
       "Population and dwelling counts (11): Private dwellings occupied by usual residents percentage change, 2016 to 2021 [9]           Float64\n",
       "Symbols.8                                                                                                                           Int8\n",
       "Population and dwelling counts (11): Land area in square kilometres, 2021 [10]                                                   Float64\n",
       "Symbols.9                                                                                                                           Int8\n",
       "Population and dwelling counts (11): Population density per square kilometre, 2021 [11]                                          Float64\n",
       "Symbols.10                                                                                                                          Int8\n",
       "dtype: object"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "49cc1fa3-1ac8-4510-b4fd-7827a041e4a9",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exporting dataframe as parquet to /data/tables/output/en/12100013.parquet\n"
     ]
    }
   ],
   "source": [
    "output_parquet = f\"{output_folder}/{language}/{product_id}.parquet\"\n",
    "print(f\"Exporting dataframe as parquet to {output_parquet}\")\n",
    "parameters = {\n",
    "    \"path\": output_parquet,\n",
    "    \"engine\": \"pyarrow\",\n",
    "    \"compression\": \"zstd\",\n",
    "    \"index\": False,\n",
    "    \"compression_level\": 22\n",
    "}\n",
    "df.to_parquet(**parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "788bc668-8057-4e06-91a3-b99991e0a410",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cleanup_product(product_id)\n",
    "update_last_downloaded(product_id)\n",
    "update_last_processed(product_id)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
