From 4d8fb1e7e95d3099e93d275d83feba40e4bf70bc Mon Sep 17 00:00:00 2001 From: Diego Ripley Date: Fri, 27 Jun 2025 09:47:31 -0400 Subject: [PATCH] Download datasets using rclone. HTTP end-point is https://data-01.dataforcanada.org --- experiments/download_using_rclone/run.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 experiments/download_using_rclone/run.sh diff --git a/experiments/download_using_rclone/run.sh b/experiments/download_using_rclone/run.sh new file mode 100644 index 0000000..b7077c0 --- /dev/null +++ b/experiments/download_using_rclone/run.sh @@ -0,0 +1,10 @@ +# Make sure you have rclone installed + +echo '[data-01-dataforcanada]' >> ~/.config/rclone/config +echo 'type = http' >> ~/.config/rclone/config +echo 'url = https://data-01.dataforcanada.org' >> ~/.config/rclone/config + +# If you want to list the Census of Population dataset +rclone ls data-01-dataforcanada:/processed/statistics_canada/census_of_population +# If you want to copy the entire Census of Population dataset +rclone copy --progress --transfers=16 data-01-dataforcanada:/processed/statistics_canada/census_of_population census_of_population