{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e4d2c52d-38be-4f84-a0bf-6bd8cb577ad9",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "# Purpose\n",
    "I need to find out what all possible date formats are for the \"REF_DATE\" field so that when I write the parquet file people will be able to filter on it\n",
    "\n",
    "These are all the dates I have encountered:\n",
    "- Just the year. Example found on productId 36100608: 2024\n",
    "- Year and month. Example found on productId 14100443: 2024-07\n",
    "- Year, month, date. Example found on productId 33100036: 2025-06-17\n",
    "- Range. Example found on productId 17100022: 2013/2014"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "43916131-250b-4fdf-9c75-39024822529c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "from multiprocessing import Pool\n",
    "import sqlite3\n",
    "import polars as pl\n",
    "\n",
    "data_folder = \"/data/tables\"\n",
    "input_folder = f\"{data_folder}/input\"\n",
    "scratch_folder = f\"{data_folder}/scratch\"\n",
    "output_folder = f\"{data_folder}/output\"\n",
    "\n",
    "con = sqlite3.connect(f\"{data_folder}/dates.db\")\n",
    "cur = con.cursor()\n",
    "cur.executescript(\"\"\"\n",
    "    DROP TABLE IF EXISTS dates;\n",
    "    CREATE TABLE IF NOT EXISTS dates (\n",
    "      product_id TEXT,\n",
    "      date TEXT\n",
    "    );\n",
    "\"\"\")\n",
    "con.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "553d58ff-81a4-45e6-a96e-6ea9ebb166fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_unique_dates(filepath):\n",
    "    \"\"\"\n",
    "    Finds unique dates for a table and writes to SQLite table\n",
    "    \"\"\"\n",
    "    product_id = filepath.split(\"/\")[-1].split(\".parquet\")[0]\n",
    "    print(f\"Processing {product_id}\")\n",
    "    try:\n",
    "        result = (\n",
    "        pl.scan_parquet(filepath)\n",
    "            .select(\"REF_DATE\")\n",
    "            .collect()\n",
    "        )\n",
    "    except Exception:\n",
    "        return\n",
    "    unique_dates = [(product_id, x) for x in result['REF_DATE'].unique(maintain_order=True).to_list()]\n",
    "    cur.executemany(\"INSERT INTO dates VALUES(?, ?)\", unique_dates)\n",
    "    con.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "507c069d-472f-44a9-872b-e690225fad17",
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    files_to_process = glob.glob(f\"{output_folder}/en/*.parquet\")\n",
    "    with Pool() as p:\n",
    "        p.map(find_unique_dates, files_to_process)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
