{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d287c593-ab51-4042-b7a6-634916f7075f",
   "metadata": {},
   "source": [
    "# Purpose\n",
    "I need to find out what all possible date formats are for the \"REF_DATE\" field so that when I write the parquet file people will be able to filter on it\n",
    "\n",
    "These are all the dates I have encountered:\n",
    "- Just the year. Example found on productId 36100608: 2024\n",
    "- Year and month. Example found on productId 14100443: 2024-07\n",
    "- Year, month, date. Example found on productId 33100036: 2025-06-17\n",
    "- Range. Example found on productId 17100022: 2013/2014"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "57b802e1-ce9c-4e2e-abf5-aa29ba7e77f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 16)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>REF_DATE</th><th>GEO</th><th>DGUID</th><th>Restriction level</th><th>Vaccination status</th><th>UOM</th><th>UOM_ID</th><th>SCALAR_FACTOR</th><th>SCALAR_ID</th><th>VECTOR</th><th>COORDINATE</th><th>VALUE</th><th>STATUS</th><th>SYMBOL</th><th>TERMINATED</th><th>DECIMALS</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>i16</td><td>str</td><td>i8</td><td>str</td><td>str</td><td>f64</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td></tr></thead><tbody><tr><td>&quot;2020-01-01&quot;</td><td>&quot;Newfoundland and Labrador&quot;</td><td>&quot;2016A000210&quot;</td><td>&quot;Restriction index&quot;</td><td>&quot;Total population&quot;</td><td>&quot;Index&quot;</td><td>160</td><td>&quot;units&quot;</td><td>0</td><td>&quot;v1331468081&quot;</td><td>&quot;1.1.1&quot;</td><td>1.67</td><td>null</td><td>null</td><td>null</td><td>2</td></tr><tr><td>&quot;2020-01-01&quot;</td><td>&quot;Newfoundland and Labrador&quot;</td><td>&quot;2016A000210&quot;</td><td>&quot;Restriction index&quot;</td><td>&quot;Vaccinated persons&quot;</td><td>&quot;Index&quot;</td><td>160</td><td>&quot;units&quot;</td><td>0</td><td>&quot;v1331468097&quot;</td><td>&quot;1.1.2&quot;</td><td>1.67</td><td>null</td><td>null</td><td>null</td><td>2</td></tr><tr><td>&quot;2020-01-01&quot;</td><td>&quot;Newfoundland and Labrador&quot;</td><td>&quot;2016A000210&quot;</td><td>&quot;Restriction index&quot;</td><td>&quot;Unvaccinated persons&quot;</td><td>&quot;Index&quot;</td><td>160</td><td>&quot;units&quot;</td><td>0</td><td>&quot;v1331468113&quot;</td><td>&quot;1.1.3&quot;</td><td>1.67</td><td>null</td><td>null</td><td>null</td><td>2</td></tr><tr><td>&quot;2020-01-01&quot;</td><td>&quot;Newfoundland and Labrador&quot;</td><td>&quot;2016A000210&quot;</td><td>&quot;School closing&quot;</td><td>&quot;Total population&quot;</td><td>&quot;Index&quot;</td><td>160</td><td>&quot;units&quot;</td><td>0</td><td>&quot;v1331468082&quot;</td><td>&quot;1.2.1&quot;</td><td>0.0</td><td>null</td><td>null</td><td>null</td><td>2</td></tr><tr><td>&quot;2020-01-01&quot;</td><td>&quot;Newfoundland and Labrador&quot;</td><td>&quot;2016A000210&quot;</td><td>&quot;School closing&quot;</td><td>&quot;Vaccinated persons&quot;</td><td>&quot;Index&quot;</td><td>160</td><td>&quot;units&quot;</td><td>0</td><td>&quot;v1331468098&quot;</td><td>&quot;1.2.2&quot;</td><td>0.0</td><td>null</td><td>null</td><td>null</td><td>2</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 16)\n",
       "┌────────────┬─────────────┬────────────┬────────────┬───┬────────┬────────┬────────────┬──────────┐\n",
       "│ REF_DATE   ┆ GEO         ┆ DGUID      ┆ Restrictio ┆ … ┆ STATUS ┆ SYMBOL ┆ TERMINATED ┆ DECIMALS │\n",
       "│ ---        ┆ ---         ┆ ---        ┆ n level    ┆   ┆ ---    ┆ ---    ┆ ---        ┆ ---      │\n",
       "│ str        ┆ str         ┆ str        ┆ ---        ┆   ┆ i8     ┆ i8     ┆ i8         ┆ i8       │\n",
       "│            ┆             ┆            ┆ str        ┆   ┆        ┆        ┆            ┆          │\n",
       "╞════════════╪═════════════╪════════════╪════════════╪═══╪════════╪════════╪════════════╪══════════╡\n",
       "│ 2020-01-01 ┆ Newfoundlan ┆ 2016A00021 ┆ Restrictio ┆ … ┆ null   ┆ null   ┆ null       ┆ 2        │\n",
       "│            ┆ d and       ┆ 0          ┆ n index    ┆   ┆        ┆        ┆            ┆          │\n",
       "│            ┆ Labrador    ┆            ┆            ┆   ┆        ┆        ┆            ┆          │\n",
       "│ 2020-01-01 ┆ Newfoundlan ┆ 2016A00021 ┆ Restrictio ┆ … ┆ null   ┆ null   ┆ null       ┆ 2        │\n",
       "│            ┆ d and       ┆ 0          ┆ n index    ┆   ┆        ┆        ┆            ┆          │\n",
       "│            ┆ Labrador    ┆            ┆            ┆   ┆        ┆        ┆            ┆          │\n",
       "│ 2020-01-01 ┆ Newfoundlan ┆ 2016A00021 ┆ Restrictio ┆ … ┆ null   ┆ null   ┆ null       ┆ 2        │\n",
       "│            ┆ d and       ┆ 0          ┆ n index    ┆   ┆        ┆        ┆            ┆          │\n",
       "│            ┆ Labrador    ┆            ┆            ┆   ┆        ┆        ┆            ┆          │\n",
       "│ 2020-01-01 ┆ Newfoundlan ┆ 2016A00021 ┆ School     ┆ … ┆ null   ┆ null   ┆ null       ┆ 2        │\n",
       "│            ┆ d and       ┆ 0          ┆ closing    ┆   ┆        ┆        ┆            ┆          │\n",
       "│            ┆ Labrador    ┆            ┆            ┆   ┆        ┆        ┆            ┆          │\n",
       "│ 2020-01-01 ┆ Newfoundlan ┆ 2016A00021 ┆ School     ┆ … ┆ null   ┆ null   ┆ null       ┆ 2        │\n",
       "│            ┆ d and       ┆ 0          ┆ closing    ┆   ┆        ┆        ┆            ┆          │\n",
       "│            ┆ Labrador    ┆            ┆            ┆   ┆        ┆        ┆            ┆          │\n",
       "└────────────┴─────────────┴────────────┴────────────┴───┴────────┴────────┴────────────┴──────────┘"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import polars as pl\n",
    "\n",
    "data_folder = \"/data/tables\"\n",
    "input_folder = f\"{data_folder}/input\"\n",
    "scratch_folder = f\"{data_folder}/scratch\"\n",
    "output_folder = f\"{data_folder}/output\"\n",
    "\n",
    "def normalize_ref_date(series):\n",
    "    # Always cast to string first\n",
    "    series = series.cast(pl.Utf8)\n",
    "   \n",
    "    # Try parsing as full date (YYYY-MM-DD)\n",
    "    full = series.str.strptime(pl.Date, \"%Y-%m-%d\", strict=False)\n",
    "\n",
    "    # For nulls, try parsing as year-month (YYYY-MM)\n",
    "    ym = series.str.strptime(pl.Date, \"%Y-%m\", strict=False).dt.replace(day=1)\n",
    "    full = full.fill_null(ym)\n",
    "\n",
    "    # For remaining nulls, try just year (YYYY)\n",
    "    y = series.str.strptime(pl.Date, \"%Y\", strict=False).dt.replace(month=1, day=1)\n",
    "    full = full.fill_null(y)\n",
    "\n",
    "    return full\n",
    "\n",
    "filepath = f\"{output_folder}/en/other/en/33100496.parquet\"\n",
    "df = pl.read_parquet(filepath)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "6e0bb9eb-0638-48c6-9308-bc5a846ff27f",
   "metadata": {},
   "outputs": [],
   "source": [
    "skip_calculating_ref_date = False\n",
    "if df.schema[\"REF_DATE\"] == pl.String:\n",
    "    if df[\"REF_DATE\"].str.contains(\"/\").any():\n",
    "        # Skip the calculating of the field\n",
    "        skip_calculating_ref_date = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "5f93ba75-5c1c-4a7f-b526-89d7589f3834",
   "metadata": {},
   "outputs": [],
   "source": [
    "if skip_calculating_ref_date == False:\n",
    "    df = df.with_columns([\n",
    "        normalize_ref_date(pl.col(\"REF_DATE\")).alias(\"REF_DATE\")\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "d54d297c-ae68-4a44-96c6-457873ea2d10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 16)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>REF_DATE</th><th>GEO</th><th>DGUID</th><th>Restriction level</th><th>Vaccination status</th><th>UOM</th><th>UOM_ID</th><th>SCALAR_FACTOR</th><th>SCALAR_ID</th><th>VECTOR</th><th>COORDINATE</th><th>VALUE</th><th>STATUS</th><th>SYMBOL</th><th>TERMINATED</th><th>DECIMALS</th></tr><tr><td>date</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>i16</td><td>str</td><td>i8</td><td>str</td><td>str</td><td>f64</td><td>i8</td><td>i8</td><td>i8</td><td>i8</td></tr></thead><tbody><tr><td>2020-01-01</td><td>&quot;Newfoundland and Labrador&quot;</td><td>&quot;2016A000210&quot;</td><td>&quot;Restriction index&quot;</td><td>&quot;Total population&quot;</td><td>&quot;Index&quot;</td><td>160</td><td>&quot;units&quot;</td><td>0</td><td>&quot;v1331468081&quot;</td><td>&quot;1.1.1&quot;</td><td>1.67</td><td>null</td><td>null</td><td>null</td><td>2</td></tr><tr><td>2020-01-01</td><td>&quot;Newfoundland and Labrador&quot;</td><td>&quot;2016A000210&quot;</td><td>&quot;Restriction index&quot;</td><td>&quot;Vaccinated persons&quot;</td><td>&quot;Index&quot;</td><td>160</td><td>&quot;units&quot;</td><td>0</td><td>&quot;v1331468097&quot;</td><td>&quot;1.1.2&quot;</td><td>1.67</td><td>null</td><td>null</td><td>null</td><td>2</td></tr><tr><td>2020-01-01</td><td>&quot;Newfoundland and Labrador&quot;</td><td>&quot;2016A000210&quot;</td><td>&quot;Restriction index&quot;</td><td>&quot;Unvaccinated persons&quot;</td><td>&quot;Index&quot;</td><td>160</td><td>&quot;units&quot;</td><td>0</td><td>&quot;v1331468113&quot;</td><td>&quot;1.1.3&quot;</td><td>1.67</td><td>null</td><td>null</td><td>null</td><td>2</td></tr><tr><td>2020-01-01</td><td>&quot;Newfoundland and Labrador&quot;</td><td>&quot;2016A000210&quot;</td><td>&quot;School closing&quot;</td><td>&quot;Total population&quot;</td><td>&quot;Index&quot;</td><td>160</td><td>&quot;units&quot;</td><td>0</td><td>&quot;v1331468082&quot;</td><td>&quot;1.2.1&quot;</td><td>0.0</td><td>null</td><td>null</td><td>null</td><td>2</td></tr><tr><td>2020-01-01</td><td>&quot;Newfoundland and Labrador&quot;</td><td>&quot;2016A000210&quot;</td><td>&quot;School closing&quot;</td><td>&quot;Vaccinated persons&quot;</td><td>&quot;Index&quot;</td><td>160</td><td>&quot;units&quot;</td><td>0</td><td>&quot;v1331468098&quot;</td><td>&quot;1.2.2&quot;</td><td>0.0</td><td>null</td><td>null</td><td>null</td><td>2</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 16)\n",
       "┌────────────┬─────────────┬────────────┬────────────┬───┬────────┬────────┬────────────┬──────────┐\n",
       "│ REF_DATE   ┆ GEO         ┆ DGUID      ┆ Restrictio ┆ … ┆ STATUS ┆ SYMBOL ┆ TERMINATED ┆ DECIMALS │\n",
       "│ ---        ┆ ---         ┆ ---        ┆ n level    ┆   ┆ ---    ┆ ---    ┆ ---        ┆ ---      │\n",
       "│ date       ┆ str         ┆ str        ┆ ---        ┆   ┆ i8     ┆ i8     ┆ i8         ┆ i8       │\n",
       "│            ┆             ┆            ┆ str        ┆   ┆        ┆        ┆            ┆          │\n",
       "╞════════════╪═════════════╪════════════╪════════════╪═══╪════════╪════════╪════════════╪══════════╡\n",
       "│ 2020-01-01 ┆ Newfoundlan ┆ 2016A00021 ┆ Restrictio ┆ … ┆ null   ┆ null   ┆ null       ┆ 2        │\n",
       "│            ┆ d and       ┆ 0          ┆ n index    ┆   ┆        ┆        ┆            ┆          │\n",
       "│            ┆ Labrador    ┆            ┆            ┆   ┆        ┆        ┆            ┆          │\n",
       "│ 2020-01-01 ┆ Newfoundlan ┆ 2016A00021 ┆ Restrictio ┆ … ┆ null   ┆ null   ┆ null       ┆ 2        │\n",
       "│            ┆ d and       ┆ 0          ┆ n index    ┆   ┆        ┆        ┆            ┆          │\n",
       "│            ┆ Labrador    ┆            ┆            ┆   ┆        ┆        ┆            ┆          │\n",
       "│ 2020-01-01 ┆ Newfoundlan ┆ 2016A00021 ┆ Restrictio ┆ … ┆ null   ┆ null   ┆ null       ┆ 2        │\n",
       "│            ┆ d and       ┆ 0          ┆ n index    ┆   ┆        ┆        ┆            ┆          │\n",
       "│            ┆ Labrador    ┆            ┆            ┆   ┆        ┆        ┆            ┆          │\n",
       "│ 2020-01-01 ┆ Newfoundlan ┆ 2016A00021 ┆ School     ┆ … ┆ null   ┆ null   ┆ null       ┆ 2        │\n",
       "│            ┆ d and       ┆ 0          ┆ closing    ┆   ┆        ┆        ┆            ┆          │\n",
       "│            ┆ Labrador    ┆            ┆            ┆   ┆        ┆        ┆            ┆          │\n",
       "│ 2020-01-01 ┆ Newfoundlan ┆ 2016A00021 ┆ School     ┆ … ┆ null   ┆ null   ┆ null       ┆ 2        │\n",
       "│            ┆ d and       ┆ 0          ┆ closing    ┆   ┆        ┆        ┆            ┆          │\n",
       "│            ┆ Labrador    ┆            ┆            ┆   ┆        ┆        ┆            ┆          │\n",
       "└────────────┴─────────────┴────────────┴────────────┴───┴────────┴────────┴────────────┴──────────┘"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c7a9dd4-5001-4174-9074-8cb14721f79c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
