dataforcanada/d4c-datapkg-statistical
1
0
Fork 0
mirror of https://github.com/dataforcanada/d4c-datapkg-statistical.git synced 2026-06-13 14:10:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add DuckDB example for duplicate column name

Browse Source
This commit is contained in:
Diego Ripley
2025-06-25 15:38:12 +00:00
parent b71a7b326e
commit 4ed5fb4bbb
1 changed files with 11 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/statcan_products/check_duplicate_column_names.ipynb
+5 -152
View File
@@ -27,7 +27,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 37, "execution_count": 38,
"id": "04e61bd3-ab4c-46aa-9c0b-de949699ca0a", "id": "04e61bd3-ab4c-46aa-9c0b-de949699ca0a",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -73,126 +73,6 @@
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
" <tr>\n", " <tr>\n",
" <th>0</th>\n",
" <td>1961</td>\n",
" <td>1961-01-01</td>\n",
" <td>1961-12-31</td>\n",
" <td>Newfoundland and Labrador</td>\n",
" <td>None</td>\n",
" <td>Present value calculation, timber stocks, meth...</td>\n",
" <td>Dollars</td>\n",
" <td>81</td>\n",
" <td>millions</td>\n",
" <td>6</td>\n",
" <td>v3822242</td>\n",
" <td>2.1</td>\n",
" <td>470.0</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1961</td>\n",
" <td>1961-01-01</td>\n",
" <td>1961-12-31</td>\n",
" <td>Newfoundland and Labrador</td>\n",
" <td>None</td>\n",
" <td>Present value calculation, timber stocks, meth...</td>\n",
" <td>Dollars</td>\n",
" <td>81</td>\n",
" <td>millions</td>\n",
" <td>6</td>\n",
" <td>v3822243</td>\n",
" <td>2.2</td>\n",
" <td>539.1</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1961</td>\n",
" <td>1961-01-01</td>\n",
" <td>1961-12-31</td>\n",
" <td>Nova Scotia</td>\n",
" <td>None</td>\n",
" <td>Present value calculation, timber stocks, meth...</td>\n",
" <td>Dollars</td>\n",
" <td>81</td>\n",
" <td>millions</td>\n",
" <td>6</td>\n",
" <td>v3822244</td>\n",
" <td>3.1</td>\n",
" <td>0.0</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1961</td>\n",
" <td>1961-01-01</td>\n",
" <td>1961-12-31</td>\n",
" <td>Nova Scotia</td>\n",
" <td>None</td>\n",
" <td>Present value calculation, timber stocks, meth...</td>\n",
" <td>Dollars</td>\n",
" <td>81</td>\n",
" <td>millions</td>\n",
" <td>6</td>\n",
" <td>v3822245</td>\n",
" <td>3.2</td>\n",
" <td>76.6</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1961</td>\n",
" <td>1961-01-01</td>\n",
" <td>1961-12-31</td>\n",
" <td>New Brunswick</td>\n",
" <td>None</td>\n",
" <td>Present value calculation, timber stocks, meth...</td>\n",
" <td>Dollars</td>\n",
" <td>81</td>\n",
" <td>millions</td>\n",
" <td>6</td>\n",
" <td>v3822246</td>\n",
" <td>4.1</td>\n",
" <td>637.9</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>753</th>\n", " <th>753</th>\n",
" <td>2010</td>\n", " <td>2010</td>\n",
" <td>2010-01-01</td>\n", " <td>2010-01-01</td>\n",
@@ -294,17 +174,10 @@
" </tr>\n", " </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"<p>758 rows × 17 columns</p>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
" REF_DATE REF_START_DATE REF_END_DATE GEO DGUID \\\n", " REF_DATE REF_START_DATE REF_END_DATE GEO DGUID \\\n",
"0 1961 1961-01-01 1961-12-31 Newfoundland and Labrador None \n",
"1 1961 1961-01-01 1961-12-31 Newfoundland and Labrador None \n",
"2 1961 1961-01-01 1961-12-31 Nova Scotia None \n",
"3 1961 1961-01-01 1961-12-31 Nova Scotia None \n",
"4 1961 1961-01-01 1961-12-31 New Brunswick None \n",
".. ... ... ... ... ... \n",
"753 2010 2010-01-01 2010-12-31 Canada None \n", "753 2010 2010-01-01 2010-12-31 Canada None \n",
"754 2011 2011-01-01 2011-12-31 Canada None \n", "754 2011 2011-01-01 2011-12-31 Canada None \n",
"755 2011 2011-01-01 2011-12-31 Canada None \n", "755 2011 2011-01-01 2011-12-31 Canada None \n",
@@ -312,12 +185,6 @@
"757 2012 2012-01-01 2012-12-31 Canada None \n", "757 2012 2012-01-01 2012-12-31 Canada None \n",
"\n", "\n",
" Value UOM UOM_ID \\\n", " Value UOM UOM_ID \\\n",
"0 Present value calculation, timber stocks, meth... Dollars 81 \n",
"1 Present value calculation, timber stocks, meth... Dollars 81 \n",
"2 Present value calculation, timber stocks, meth... Dollars 81 \n",
"3 Present value calculation, timber stocks, meth... Dollars 81 \n",
"4 Present value calculation, timber stocks, meth... Dollars 81 \n",
".. ... ... ... \n",
"753 Present value calculation, timber stocks, meth... Dollars 81 \n", "753 Present value calculation, timber stocks, meth... Dollars 81 \n",
"754 Present value calculation, timber stocks, meth... Dollars 81 \n", "754 Present value calculation, timber stocks, meth... Dollars 81 \n",
"755 Present value calculation, timber stocks, meth... Dollars 81 \n", "755 Present value calculation, timber stocks, meth... Dollars 81 \n",
@@ -325,12 +192,6 @@
"757 Present value calculation, timber stocks, meth... Dollars 81 \n", "757 Present value calculation, timber stocks, meth... Dollars 81 \n",
"\n", "\n",
" SCALAR_FACTOR SCALAR_ID VECTOR COORDINATE VALUE_1 STATUS SYMBOL \\\n", " SCALAR_FACTOR SCALAR_ID VECTOR COORDINATE VALUE_1 STATUS SYMBOL \\\n",
"0 millions 6 v3822242 2.1 470.0 None None \n",
"1 millions 6 v3822243 2.2 539.1 None None \n",
"2 millions 6 v3822244 3.1 0.0 None None \n",
"3 millions 6 v3822245 3.2 76.6 None None \n",
"4 millions 6 v3822246 4.1 637.9 None None \n",
".. ... ... ... ... ... ... ... \n",
"753 millions 6 v3822241 1.2 124971.2 None None \n", "753 millions 6 v3822241 1.2 124971.2 None None \n",
"754 millions 6 v3822240 1.1 120498.5 None None \n", "754 millions 6 v3822240 1.1 120498.5 None None \n",
"755 millions 6 v3822241 1.2 120498.5 None None \n", "755 millions 6 v3822241 1.2 120498.5 None None \n",
@@ -338,22 +199,14 @@
"757 millions 6 v3822241 1.2 113132.6 None None \n", "757 millions 6 v3822241 1.2 113132.6 None None \n",
"\n", "\n",
" TERMINATED DECIMALS \n", " TERMINATED DECIMALS \n",
"0 None 1 \n",
"1 None 1 \n",
"2 None 1 \n",
"3 None 1 \n",
"4 None 1 \n",
".. ... ... \n",
"753 None 1 \n", "753 None 1 \n",
"754 None 1 \n", "754 None 1 \n",
"755 None 1 \n", "755 None 1 \n",
"756 None 1 \n", "756 None 1 \n",
"757 None 1 \n", "757 None 1 "
"\n",
"[758 rows x 17 columns]"
] ]
}, },
"execution_count": 37, "execution_count": 38,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@@ -363,7 +216,7 @@
"\n", "\n",
"issue = con.execute(\"SELECT * FROM '/data/tables/output/en/june_20_2025/38100105.parquet'\").df()\n", "issue = con.execute(\"SELECT * FROM '/data/tables/output/en/june_20_2025/38100105.parquet'\").df()\n",
"\n", "\n",
"issue" "issue.tail()"
] ]
}, },
{ {
@@ -378,7 +231,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 31, "execution_count": 40,
"id": "d52074f9-7746-4569-9aea-57c204eda2eb", "id": "d52074f9-7746-4569-9aea-57c204eda2eb",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [