From ce1b69a1b658600a854cd9d4e3e114601817b4a2 Mon Sep 17 00:00:00 2001 From: Diego Ripley Date: Mon, 9 Mar 2026 09:40:41 -0400 Subject: [PATCH] Add sample download script for Vancouver. Will use this txt download list with the Cloudflare HTTP ingestor https://github.com/dataforcanada/d4c-infra-distribution/tree/c9d051bc599e4f1dccfee6918a2f6dd2592bef27/scripts/05_cloudflare_http_ingestor --- .../download.sh | 0 .../README.md | 34 ++ ...magery_index_2022_075mm_2026-03-09.parquet | Bin 0 -> 35916 bytes ...thoimagery_index_2022_075mm_2026-03-09.txt | 470 ++++++++++++++++++ .../download.sh | 71 +++ .../generate_tiles.sh | 1 + .../tune.sh | 0 7 files changed, 576 insertions(+) delete mode 100644 scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/download.sh create mode 100644 scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/README.md create mode 100644 scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_index_2022_075mm_2026-03-09.parquet create mode 100644 scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_index_2022_075mm_2026-03-09.txt create mode 100755 scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/download.sh rename scripts/{ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm => ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm}/generate_tiles.sh (97%) rename scripts/{ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm => ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm}/tune.sh (100%) diff --git a/scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/download.sh b/scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/download.sh deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/README.md b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/README.md new file mode 100644 index 0000000..54c0729 --- /dev/null +++ b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/README.md @@ -0,0 +1,34 @@ +# Vancouver 2022 Orthoimagery — Download Script + +This directory contains the automation script for acquiring the **City of Vancouver 2022 Orthophoto Imagery** dataset (7.5 cm resolution) from [Vancouver Open Data](https://opendata.vancouver.ca/explore/dataset/orthophoto-imagery-2022/). + +## What the Script Does + +`download.sh` performs four sequential steps: + +1. **Download Index** — Uses `aria2c` to fetch the dataset catalogue as a Parquet file from the Vancouver Open Data API. +2. **Extract URLs** — Queries the Parquet file with `duckdb` to extract all MrSID image URLs into a plain-text file suitable for batch downloading. +3. **Create Output Directory** — Ensures the data input directory exists at `../../data/input/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/` (relative to this script). +4. **Download Images** — Uses `aria2c` to download all images in parallel (12 concurrent connections, 4 connections per server) into the data input directory. + +## Dependencies + +The following command-line tools must be installed and available on your `PATH`: + +| Tool | Purpose | Install | +|---|---|---| +| [aria2c](https://aria2.github.io/) | High-speed parallel downloads | `sudo apt install aria2` | +| [duckdb](https://duckdb.org/) | Query Parquet files from the CLI | [Install guide](https://duckdb.org/docs/installation/) | + +## Usage + +```bash +cd scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm +bash download.sh +``` + +The script will print progress for each step. Once complete, the downloaded MrSID image files will be located in: + +``` +data/input/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ +``` diff --git a/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_index_2022_075mm_2026-03-09.parquet b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_index_2022_075mm_2026-03-09.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d05464a76dc12aa2de0a7f149aa6b3624a6ba76f GIT binary patch literal 35916 zcmeFZRX|kT7dLzcW?%+3hoO;B5K++)h5?CDwsdz128xP`-Cfw&*cc#SVK=teVqv0U zx1x^S_5L<9ga7yTdvCsr=VG5>*V?Q1TI(E*4@h*k5L&!;w^&x+;{N>uGe~QTP5Ks` zDaU=Ww$|l1F3?>e(dRq``dpAi-yzuDT?d1~z5;V`h`U^(FLL2RBn`!(?j8aQJbeWE z!Z3GHGjW*0UBq(=Q8QtfLgXY2m$Qev$W9#Y?k#ZQ!Xh<1vj6<&fwVT?kQE{s)(HgmBOo{N(VMYG`IC7sa<#qkPH zK@TooTe00INF5(n-xu6KAZbq{K>uz(YF0&XP9l2D|2rf0`82ERT zYa8(wh>YRdidu1jl2cruOEP|M3j^InUcjyw7wqf|3_Eh6E;G1LoEhe_34C-z%K|SG zxd@l9;3JRL46O&+Q!dgaA6&E$M!So)bJ3!eTujzNE=E)W#w|kYgw_(wgsM0X2P3y~ zagb)5q!hpL;y8teqbnCDYKnH2i+7m<1u6m`GPs0n7cQ~Ih=`+Hl5-^(8_y+!sVP7v z1<0hhtOBV`LFyBLH5#@v1!_X+&Od#0~jRH|vWOe~`w;mAohVFa^dU9c$yoH4`FcWna#>-E30{dKm z;!`f(c{z?ZM_b7yM=S&bgK?Mz+7dK7Gztby#NiWhxC57Jd6NJkSzyu&7tcH zpaKv*6ThT2*g1_$b?L*kjgUg2gFw-8i2FDfgcbsoi;xTi=ZesJV=fIUF`f&Au?0u; zfb+1$8gDMdB?W4s50$tAS}nP-2zM@AM6uVLi*UhS*l?8GITS=E!FhOQm4x&I`xDXT zq5b5dL&PB54Q(RYXD-I&1{afC0MSHovCdaOG@OQ{94%d#AU7=+a|so%K~_3+#gt2o zm_%Xvws>4g4pH>jkc!p63)kziIlh)wv6r#21I>2de=tyjW*Uu{onr`aS?!@}M zlCXIVBbEi1&R-L?RFPeMqIAK!xK)9ITwzI|%*^0NL9j^)~h zZCSBz@QW*@?N4u6d0_avM-|J%wiX;3{q4ic+o!h{9vRQ+8rX$zTXl4j*sNKHGuw)e zPm?-Dt_a^=d~&9>N3T0)wwIiq>ku@(iUm!RHgu@ivF6+odDpWmB6hC5u-s>G z&7F#!>naQVCmGmB?plAjBz!@$j%Rn3UR@Vo61g&R_lD~m({}c{dv^E68(TXcpJ5-h zXVa~nJ+E%)cy7<;+hqeDpIsSMw&m`D5g%*ro-5mW|HuSg!}`&Cw>>;Q-K=@W`Muj8 zpPuIwRS>;z$J295J$m0ezi;RB%2h!#>&NWh_44Zaq>ULD_V0dmV_VmA1u+Nqyt%!1 z@T+?l4wSvUUp~pOLF~c3?;oF8(7e;dgZn-{zf=-c7<)*`aR~2r9oDtD*UVzZ{R!1N zb@AP@`+|3^eW5r#hYT4oSn1|AJgdu)L7Dvqch4H+IxMqa*ZxC?;jgZlZv6*j_3M^7 zB-2eHS11N&cXLBb=sGHUz&$^X^L&O!kjjq}BQz9@;(V@r;)DnfTUUMLScHffPJu^F zUjYjepN-;Jl!#3T6Knbj5FI){uiD6d)x8>b0fI$5x$ubFRi8K(FFxCdp6V0FMW`S3 zbn%z74i^#OToobYB5Q8za8WA!73CtY+QV^CHF!j;rU(&Y)f^MD;PF|t4q~1fD_w+- zH6KM>tZEd;#i$6bu8Ag+G z4XgSJgh}pMDfs1*YQlwFvPv%CQdCyDcwm1j1D*P16jzfZ=28K(kq$zlngSin>=AKo zRZjYNAhvF+GLYZ_Y|>Ojh{s6w*9!p`tSZ#uLKv+fsxlE5Qt`Zs!%t0LJuXxQBtlhF zgniqRnkTMm&2cd` z*-|b>bxQ|4nWu|Km;?_?_LmZTsyQzPi#~!yrOGNN9WJ5f9pi{Edz^*iRn67ql2j5Q z9>Wa43h*%*Gy|FB8Z#p_B5>e0C#5k(6G8Sw`NGyZm3T7oGPeD zsA{lF5-~-}gvd>p-5O6W3W6iKUq+wMl6Ua^AkQ>azoFLkR@fr*ys?zo0 z|8KI<#2Wmaq$<(F<0&MhD&rxwA5|+LJXL=KhzxQ|R^gHI(vDG-qPnkF1GG{lyQ|g< zxKzt9qiR3SvW=w54ZWIRNYuGEL$z(LJ|?mg)HIwBv%?*AxImSSJ|30?kIVSWGRTTy z8kB9pG$h!$v}z=yFF}_JEq%i<3}qOGS>7~+i9mu_8_u{Gp{j;Tt13(&L1<81=x#&|$Y+N##*;r4ZBDw7sc&B!P+ZT{ zg4RYc)s1G%ihkJD^lDnDl7c!V=w5YA4j zHSSt#j%3{^X8KX8$qumUBNC7d6k=3)Vu*Qz00H@jDH2n2qbe9+Turhr!gAAh22~?O z5G5nW@{fdxB;MJLsbI3I_Ay+b%;h$RQ;Q>tu{j#P>%~2ubV_^i$;T9uYo-<^kmqF!hZJX~fhw+SFYRDTzifDaEKlAwElQ1K=r)@#5DiF||g;*2Oyas&a)KOSe~# z31cv~p-(Q+0r^P#u(T$lRGQ!fWt|VE9F-b|WKJl^P?YuIX+F&NBT_1C6e!vVexb zp+u|xFy~)^Q1#@xDtPo5=5S)NlbDPlo*30~xPMD?G1Jmm)k+=sh|W?ll_`sAy&fL8 zG?7EdWQ^1V+$#o*TM&MtWrDcY<61T|to^MSCW28wBa)a#B*AUL?*ImTHicd4Da?RUnJiNo<*5i%V5O-c1q=t+a+-?r;1UcqSQW+|kbk%e zPC1-yjDYCCoroGUG1yVd7|1wEbqKV7@#C_RnX`z_HDJ7nu31E&73<6yMlo!0tjYqO zrgWG3Dw!yOh5&G!<(yg-h^ql-;=i1=tYONbYHr9S!X|KkCOW$@iYA=m)Q)^^DEZF=n1Ntum`hU)Vro1h?^8cIx zWpy3?Uk3DF2J~MB^j`+_Uk3DF2J~MB^j`+_Uk3DF2J~MB^xr$6{~x~t+H^)_G2y+y zV(-VQtp;^3ffhf}G4F_XKmuKXs3(3!Xyc-H`h35XQNG}ni&Jq?xc|J8iz@}vK6afm zCbcXM_uqM9;ouNVH=lhV(F=V#euJq%N|x;#_dy`1{#;-t7w!LJZoh9>Gb(IDm(i=j z{qwR5t^!2Y$8V1DfY>kl)b?OZpG;KFH%9;1#24`bsa<}W%6$x0tAS8=j^ZncAq(%# zomZJrZediOCdJwV-uTo=O4=Lu`6kHJ9M_$Xipay7qj3rKyn)gZnZa5dcVgk3k!R7J z_j&yAyB9Oczb)+MeGYvh|4<(wbk{bxQHSvhuS%Wy0P!{S#I24c;r`VvmUR+HrG2}9 z@O>|kHXXjH={-Q0ci_fX?A+AF?2;hUZrJ;xB^g+&!MB*B0bF!%vxt>0SeiMaEV>gW zcLqNX$CmW-gI)`+s8@QHn)Et#2Ww_Fx;kuiQK+sUn!h#UQ&Wt%&1pQ#2Pg_Id8`6Q zZXdeRd<}?J?;}}fdRYiEWpA>b_KXFe-sV5=E0D@^rQDjPhf-i+mJVI3!_+Kek_X`}!|~lBd}U? zblV=Y(*+6U-&@`{eX=~XR+GrxG1eUgnIv-$zeg)}B3)f55oB7oXy(2netCGHdf}@F zX(kZwbn|v83Cp!0qgy8SPBOy5c}E_dnYKFI-zs;pA@EAQmf!Oe%zg!?a@F3gFQB{0{lYE@eN zRoJSi77hhv_lxEXQ({(4**pJHe1n^ehnEu?<(~u(=WhM!bOe3V=^MYN6^8pOZ*}4g z(LeRH^}=-YNBr1v$^iY?L0dgeVf`p&t&P%T0UaWUZsM_|vjnqxrK94Xg4nr7MD=j? z#&bvBI!;y}NrMG_q=yWx)6Zp;+wOYMZaRoBZjgN~4gFt_^L{ukq^kuQzE@ZUv~Wze zP7^h;Tvrqx*iArIjp`K%ojr4OxVROlICc4k9^gj@7_8T_g>3R}v0m_z6ntv2J=2$r zxp4X|m@;n5EfWIyKH7P=CIu&LzBaoA8$VPITnDDa`rZi~StoAVO`!G`MAz4#8)Q6| zI6VSdwS>Hows4#rJ>%|tm(S?h3U%(SKATbgyV1!VfL|QH zumJEAF7!<7f>{lS>e6PNpvi)#!VedVnI_A0rsVW~SCLVklyTt-jBfeF2syU#VciB* z*2yZ`AgAkt62MOG(%5}HEVRGz$p(UfMBFDo&+&S+Zn(wgw*Zk|_|c>_HmR;X*#`_< zdW_z#WexqvQ8NLV6?bT@V28EuQ&*2KCEoKv;A4p@*oc#3KQFr~887}3^!u@A` zKjZcWdp>XNI3o$_uvro1?Th~TS$hP@=u2J>3RN=*zEOpAQITU99idLFl1p*g@cKzWR>KIKBK( z_z4@~&9#%yWR#~AuZmd)n7T>IA=va~?dI3oP4E2g{=HB&1&b2b@0`y=Q*OmgluSjx z>(+rEi_kyw*N3}u8aKy`Q>#-RFN}>+fm|VLnH9a5Cll%lj{3c5zwv zoq}UzYKP2OaEYf^^-t7~yi)5DClp^CauqA0x}{|JL0$&?`avW6>>7Uc@FjE|zV#Z? z0Mmb`nb}=NKQ#DP1aPk2mUlplGxt?lE6`|1%NJklvG&X1L3DUy zdI)fZlF4SY7(BVu<^HG9E5H+(_oJStvrlD|=l|8&0F5hsA5;W!l7@W*1=%m3$5Q14R9O{o0zaG{e>10B8^LOg3RIzBd#2UQ1;#}h`~)_c&dd*3Z!h( zQILWop6wYq4HOjYee8;pJYD8DK8Cd#-5Z~J`lI9S2UoWC_L~WZ5c9~tz5^)Q8D6=v z{+e+A-(ULP>kVv=Hc9!w`bQo#-pl%a&Z<1Ne!QCb@~F6HzP&KM*C+gZ1wY0>{M)=U zbofx!4_QYzuVbB>M(#P8Q6Bbm=?I{bH_D|cP`T~<*y`-Rdq!xX*d}n731-Y%W(Bf`niGc&WCTDvQY;hKG)Ny8!_Ge!^4}FRq#*W0M!AXZ-kaDay>Naqe!}WGjnKvc)i?$1} z2B5!Jfuat8yj~Lz;B|et$BTf*j8c`JVPg!GKYRi#xU%*5t^w z3!^VxIaT=-gD*B+o$U<;N*N}42w+?1=c{rINy09VN@&5z1L^!ZOxiAKbPSVAXV2eU zN5>+I?me;O(bT-~t`z0-$x*}f_!pvSd!JqZ#FG0;V(rb}<)b{Tz#K&C2f;0jV1E zO+7PH3RJiFoqwOXflkvQduZtg3>@p-Jv(3t$qG325(3%1;hcCAY(~T9lLC@I?A>ff zU74X@vt~7*IB}YO9(F}`bqD~dl_#1(FG%)i#RJFBnu8YU9g7CVBM$n7Nb+kVf{2Gp z`eM)1?yU|v4k6K*hq__UTYt9>T9U}rZ4Q+JcXH9_;jhg8BoRO@nM+}vVB8V z?5ik;wyxZhg3&|cUhTrq!4<+f8=<-avZVS+KK$E>1A4_{;T^~6o?nV1t8e*k!{RZ{ z_6zZ|PL&wv=%GRL`ujn{up;<(oB9Da$)EYL!kOFwuPZB&DAiY-OLL&+`>tGztah;M#hoaT*)B4s!|MYFUP-sqsO*5mZSgpo`&a$MNEM-OC2J{+= z&|X|E{LUY=B*QBFZQ!YuzWbJ*=Z{(|&s*Z=qbn>m!Q92Y7H9b%7Nk(GyU#y19VjVC z;#ZrQU)efl^xC3O+>@)#sDr77iH}AcKY&?TT$_^pClrb=+|NMh zc;AFS`3^_WwH1+Zgh z-q-K_4DKUORoRR8Hzm^?T#yLSc&!wB_V{Uwj9?JSFyb>zNNGQlwm3P~*RY=_m^8(4 zLxd71KX)DT6enl2J#LDhDz~$fwLGIaru3L8VbQAVBU4(Wfh$v-xf1jmJ1+!2(c?2s zx%WyPGfu9=5^Si=7C0THwFN9scIH*T{;jc;*k9TJ3nbqRvV5Sc`;TTvc8#2B4n=+E zzj(Igdq1oxy?rM74~R}()Ve7}zf;+VO?m6X{k^@TD{~Q%c2d5zosWL&Htklhe%j;m z18n_bbr3|3Y|an}<`e&_pD+2d7A!%R&Tk`{j+NcM`0W$ykR7bM@QNv^nc^PGo0<{P(aPUi)({&Rr-VZu zuM_5Ni2y*I@`}4IvBJetJv(q^-NF`kd|UJ2ghAt`$t}Tl>ouSHF)5H6yUw6nxFEVC z3$N`RkJ<2ZOvs{`qX-kpR(s?9bs#>yvRlo}=&V%b&L^4C=jQa9ne1tldkG7~8|*l< zq;6@~=~h0DCuQxF;g`_~6WRdY&c$T{iwO_ySa+9y zqH*N(ktrusb~U3xW>iNrCTnrg_t~d@2XMUsw`RuNEB9NMY^~g*uS(($S(Zy;3 zW)}}~&j+|?EkuW4OT}4Te5CNo8dQ3vPq4-{)OW4337>0BOkeg@V(rvRR)ctJ3({cj zhY-w8T3F5Q!yaoDP4TmF@6#kLgSb>B+(3`V`QH$vK7IS`yBCD`vx&~D7zC-sclA8^ z6;kroraOxm2aEl#YW)C<);4$C)fMV7`oU=nejQJGw$|&7vz}U=jl<#r>4OsRvuH@Q zO1p<_8)(0m@pezg5MO@3F|EBB`f)(4%-!ukzzZ_{;3Qa|EdJb zp0K38`dNQtzNal&|8~SJ{;fUvKxS>>JJ;8cHE`*^DE_yBtl*`+EEo*zWqf@iyhYWJ zj9E9(?Pof{i}~4cQ)7MtIkUSzbf z=k4e&ef4a#^Y}7hAEdb7m+ea20jT{)>NbZmd>Th~h5*@pc_$w6VhM@$t^5NlFD%|z zj-OVcLkjAUb(O5@&5xAGTn0%lcY`ciwtkd{HCYSByYn|}h~=KX?QpP{sn=MHzU|); zra}fjI%xMWNR5JQYq?C1rO_w;IG6FEVrpbhT3Pq7y$WK(-5 z&~@4oT>%|qK)T$&>d)KQk;QG7Iy`iu&FnPm`xww!2DHYOMDL!Mz&{rg@zO_o_z|}1 zY4UuNUm@g81Qj!H1l(kR!nd6(V=HvofGhQHvh(eIE;v6 zIrNHgPh|XdYc+VEswsP4b6`jFgr|98Y*XKXxg*PkfPbs`y>nXl)KZ6f=@65h4Mlv7 zgec6)+XaMDts1IlX@NZ*r_}`i`ZU&%waxi-301Fdc&67F)C@a~RVLSARa-=&`z5r!RtWy} zNufu`2j0(K=Jm2|Tn6m!<=UV-d_P0t+^$}K_zNx4Wb4i7&Cr$<5BKxPdB29|9)>cW z^VVyz8dr^HoaJudHtN!GI97V!P>mWz;@8$_=}~yHvdk~n`3yta&yX9hoTHrwo76mv z)HmtGk255PuYC68KM>7!T>)pS+|_`z^PRAZe?&+?_N)`|Eo6>I(MAYo`{9FKwTT$L zZ%@sP=pANJ8oqcP_?CcRa85^P(^mvlSk)y=SHGf-q=$ZrNq+aX!gIrmp^q!Hi^5oLs0?s7n7fbCK8A-_T}Y zN}VI5sl06IgV8NRv3%XA`eyuh5AKCh;z@gXYUq7#()%E<^h|d6XuWJQVxL)$1L`e6 za_RQQ`-jz<(EnApJqlU`m+COnm$%@_modqyyqla>DKsSI)WHv;5M?b3Ws|lTjA!p0!ur}=TM0YiKZmzPWf^L`7%X0Yv)mjHu_iqu~B_}a*sDfpNwRIlaSx(z% zUNmy2*oIow;|!GBG^0MYBO{Kt#oqI7i^9ycF|I7-kEn=M&>l#+r+Uc5MLNuVq354Nn#B^ocvE9MfR|f*Dc}D{yE&c4gX*h@2t2yZ?77xT$R=xg?`-~qsojBJGurpAOUP(meZ+Ir#*Y5bZm)PV_!@6-PaZa@ zKg=|0R@2JNN$~I!ls|1Qvy?{13n2?vO7=O(#)!dbp#D@L@%R=|hM;q2a0cAO1gEDX84t;sC?*TwO>*ylj zPiO*5V()QzB`>xh%TM;G4tN2uWZI1bv*7xs_PM!=2b2Z3Nd{87olPeA*XKiQ)R8Cj z$jXD9t}wa6!(+w%jA+0+o0Fo-v86 ziq-d^&$KkxUP7|_7Y?mt75003&L0j4CCBYsv+AP!{P+x9Nop51%mLLvVay{hZx7m5 z1DITB6yWH%o4hf1w?Z+6sk6R^?3#4H87sR#FqqT3PK_GL6= zrqFW#w9 zs75NU?XqWC?S@BFrtvM@u!^o#kfR1ub@{<&WWqN7UwmTNz$t(sSksPkymE zsmYw-%ump`IvQmu8{FunWH~4anA?7HfERyIBW3p&)!Wj)`w zC2gbysOp#e62(HYKKpVdqCIsW37f9fPq+tcuazkG2PA2v4ED8fBEMUVUdh81?`~Po zg!gm$cj8D3T4OnPGM ze;Jg|@+SME7A&Xz8*s#jAKOxMXl>3(6en}5cyA9`;W+8imoVPc)-RB!ksIwcd-GGY z_^k@3spb?uZm?D@W`|L@b#;cXwmhiEriTC52x<&!>ZgTh&>N#{6wDIg=1*f>d4aoh z{XK8ts>tAA^|UNxz>)Fn((;~d{hY_GR-06|{xfw`CcCs2uW%W_-tnpzlZ`fQ8=^u! z@Ij$5Z)Ae+PUyTq`dgEI(jzAHI70tMR9nX2N|giTxX8)sJ_{!>ZSS(OaVS5%1)ZnS zBVy2XS{j?sw{D!w|8*iQ6RzN0j`_8Y*}H<)h8j0wX``CU66I03aI?Y-C$yK+(b_Xj zoi%B&fp!EbF$!yEktCaHwpuG=+{oPxJBnD%PuXRh1q&YXjVYdLHX_pqGK&_kSQK_0 zo^I}wH;#XCb-mnHZ4Id$y1VlzDtuf-lu!0FW-{atUMJ=2b#*7M4Y&2hhrYC>LukxJQ;L(!fGtFujs#(|Dur! zEi_@=JmE|)YLBT3rYLAs8S^M-e(M#!+E~^r@o6l3St+x6@ePNTZXdaoSGaS5%85aB(G6`S)#wlyt)}`; zQx>s2Z%ltR)r=q>-jN;jiNDo^{JFNXJ^#d=Yp#48RI32&wKX|CHEtZ=!i5|k^3;R3 zXiVq0aL<*e8q=1I0UEp%db^1t|Eps)+h$zZ4D0En2Vlo8c0o) z876e6CT!EhWnj{K?Jjks)*QtlzwUQ=11TADIxdTkaSHV3_&+HezBc1HOWjloJ%`O7 zdy_N>_j0^clEY^y$hI$UI)Eqg0agzDUW7!945&DX3_LJR5I`y&K0XX6Ba?z}%-~Cn ziT#)X27Ehz;UEXE`I>x}#7`C6 zwUI@Wlb!iBTA@Ri)wZk9Y^P(S6xFfWQC-jTsXQq%^tj1OJV|-O?WeJMcxd5p9&)Qa zNthMj1s2xsC%Z%c9)s5!@IluKnxdhkxs>VCszCB-K|kaN)>cEhsqpqNu(yY_z5+>z!w(o#p- z(@_mn*x%SII*H-1If>TQ&^iXtAC6os-Ji&{^GZ)tFOAme*LB4jxWD~pHeKC{fOv7+ zhquA;uJ>v@AVv5Nglqf$c@I?Y_qO4_DVMgPGL5I_-YBCQ(%ucJz9W6^pa#p`RU*@Y z(|l?~eKfntGUspG`6KqE!O&LmJa~RSbrAddVUXpYx9Exwue^cuh^*2R>=j7G=~MDd zeT>Q0B#DJKe&H4=t=cemhXn0}k%oVWW=>JM%ZM)2jF1Z{;@ixa?~Y1PF?fj zNlV#~`*VW(z}2nxo+8DeW2S{Z;Y}L=&xV?)C2+s7vf7wVHmb|TG@|@#|8E4YVOIm% zq)i=Bv~2=2vC($_Dh0?Mka1x|u1$QA+$9i-umqsgwr1lQsPt6+PYzD0A&PUP$93g4 zR@b;fq89Q;49T_;Gg$t9;Z3dwx|+>%4K>O?uV?jIDF2!^N3Za!HRN-1l&0>Q`9G_h z6=Ob1=lwu_k~TP76-$Mbs7kx#z#(i{*ZUlbe#9`cj8)m~-e&Yd5Ux4rlOHvwByjfP z7g(9`*}#@>M9H`%j~-)X>h#t7`2(7S8rfv)*4wOmvc$jRoNYlJn3DH(qcIwf&P`Ch zGo;{)k1d@WtAP&X$hW0&v7Cw0RXdj6jlOlG`)vQ!MSgapzTIet7+RstrUrG4YtNU* znn%xjes#sl(5rE@aaZcCbz=H$ARY0`8{CV?T|bE9QlH6up$^m`lHDK>DP9LXi#?P3_x zAg5NkM2%VzdPs{dG(ME)W+`JkGyPW|JvFYYE#ppcSI31_ zYeOOI@>pUjajZm=OD37EG3QqXXlOHuBTwHr}*ud0Y5j>4UM8f5UD zQ5X86<3K*poj%uYI4Y;-UAg1R*REu3qeJ1mjg&YiHvhxd-Nx_>77mP-x=JLr*h*P^ z5!6Lc5NjF`XHfb=jdIjQ}wgH>U$;6Nxfi`T3_?ZCMz zUxm_hp(Hmo&XT_!O1e7Xy+Q-6#XgJI98s5w0fLtfxJpNM(*VhLL<&nchjCk#7s51S zwIvkh+1w*EBefYZHHn}Rnu~%?b)Y_uG&+o~)%bYzr;~8`!p$AMyb{zKiGgHK2I4I8 z(oU4aH;SilMkwg4(uOUtodxdy>l+w>@S%Qg^t9IKkP9H%hr>c`NX@9Q5PGvKQy9ee z>Jd|S>$U14-p^)olf5Grvr^)wg~5a0@$H8zdk?4DP=waLv!rHSxOxnl8nxP* zAm^nt>_j(YXvRP`A}0U(+-YU=PMR?|mLr4cEGg|R)r|P+(sW(iq@+_>%wfL0C7Ec9 zZ>dpgPy?1*qp-joQjW=@@tQ(AzWE@px`$mr>TWFyzgY4XAFr0_&?P#WEk7LD#f$_m zZ=_}fS9Prxa&MJKwPS~}DEw@@P-p32JBUw-?BmrF z*wx)ahL1nV>+d<;_}#%Q##56#(-ZNKT^Q=rbiF3+Wc3_1Z|d&D8Ve2cv<#(EYbv+q zGQchJ+~~n=exC(tv`X5DuNbBQ>|m`1LaqgKYjA0w7$0|>uM@L_HJ7KP-D&Syk$vO|P6v3=h;HIzH4=Kz z$L(pYtI^;W-h_Fh^Xw}l%}7IU`nM%*;6*L9i~vVohEPc;P1c$z^k;$>{cOe%S2Gai zj-u0d4~n+|wIig%Yx5JRBFdUAkjQVL)Y#qZCu9B36?-cAcA?~ujZY2l)PZEI`Z7r% z6|VKR9MOauX)~gU2B7*?4JulAv7kQUMq~Q5vCzfE3k)l6uNhQVUgnJWhb>T@9>SzJ zzMW( z%3h(eUZ0#+vO?J2qU(XYu9h`-JTKUJ5a~=G@w0~qjb`OR?EsIjb>^I!KDWr zdN-wYxmaO$)O~`3S;Z$9ZcST7qrW`G_)t+%If8ZxqITN44EAT%evNZ}xb|EDoBid< z-Dkc{&BNxkidNtvEK+&tY2`a5m>x=}YiCp9qFzS4j*kAy*H|Ry+jhM=3FY;9zb2=J z!C8(r9_4{IDaeboOPMQC6H})OV7C+dhC_^hb)#rBTXQmMr0R8dv?_x;USv*}hR`Y6 zHBfzcRJ@?qepJMCs&*GXIC9R`K!VNHj*0R5Mp4H<81GXv2Cz^gWcV-?2?N@rxCBpz z;+~oVf!=g`Gj=<5@z#v2HlsNs@j_%)I@OlIH5Y)?6nC;_)x9t(vY`{2(az1(BgEv> za$DZsMchm;-gHzb*UM&?8Rv>e{e@=SPMd3H#O%9HM||8OK0NNWsiXF^S|o|+%9||S z!kqLO&{W9h534JY(#jE}dY+(~Z_vzoO5T!P^AHyucKKI8E3=_DY}m&!C=jXX$?~B^ z%{Wu#(Pngtk7lf=FAedbSw6Uyw)fSHspGz*&2S&aYsXIcFiDLwp{Plyp*3ty^?m6x z9~#(PGlGL)gis*u7ddPTq=X6?>O1OPCz@+yd$X-(1lUY!PQ!fZLk+jYcr&56)Q-Bh zr?;BWeOfE8y=cYmEdx#W`VxOr(+2!{U%Ed{0|lBixH-cHh^kNO+MM2Zp?_VtT}rWb zl}>Y3+VOLt5~XGp{%+V@*ngy#&j8v)dlb&J9mpy`DB-Aq0X&4+Y9KwU1vQ7krhGUU z=NhhAg_uRHxGHT}g;k-o73lifH8{V0Q+mbp-+kP_9b?kiFnXiE22O3^l%fvRL0eT& z>nk7rPrPcUhf?S74!L1WKeGJmAK`xt(13W@ng;qYTivL2TM9o7a)7HQ3guJC3l{oQ zOBDROnt+emC~&ucUKg;6b%@sTz(a^0npU830h%ZfK6GI}4P@=-r4)vuKI_du`mH~W zZ$?jO*WeaWpM@Jmy=GSJ)q+HhYMt+nzSzQTQXVLxj((a|DU>2D1thJwNIk12-T0K* zcwv}o$#?PN&MC8I)9@zTAmx0`Cd4Anjp_T=|Ca)zc8U>JxX~I2!~m`NUnL*bm@aNC ze3y-oYN6Hj#WEIg)^A4}%+%nnCin*{ zZE8lcsD=xNwdEod#s-?w=2Cc~m8~?ZV*Ai%k#u$|+E+WGUZpdE)|Jm-n0bxp_C^eU zH9z4sW7rlX-&)LRQ6oCPJ^O}gg*N1bn)K;Sdnsw8#wNH%%Co&_y=Z16dwn&qKpL>R z-m|XUO`%~JM4@fEW{r4USaZ`2y-3kU`$oKfBk}dPgQjPDks|P*FC$s&$O0ldCxyu4N;LN0qK} z+&|)-b7=o|$RvB`QseO&=*Yi8e_5b~U|c9gd*M8Kvx=iBNv-j-O@^DJuWomYUNuc`Y+om^#KPd|ZkkhUukdWq^sJ>125yF0yj)j;Ey1S-%$$UpTfNRHffm>zzqxSg zy@?p$Tb#&Ixi01iH*ANypq_N$HuIyWUB`0r=Hky?FS5@b_lU2wcU5qn53{T*&SRk4 zp31kQ3hc*v>V10I+ZqFln!NLRvC_Vtr^BiGm#i?Lda!@!(G~XfNKA?UM8F*I$lSse zhZ5n49-e}F<|h(*>{_-QXufJuKe`7#nh7jfbIxxy&}=lf_15kfsQ+#Atx2vjfhP%$ z`qT{r-p=0^8yDIupD!XDpRpp+v+_f7gpE38Y=}x>vO!y=0LfL@z`-iP}V8GIJ ziSc_^Gl6`QZr<6g81NmrX2UVpGaSkIV>|_D-O>3N;l_Be^VgcdHS;kLGtf2palW5SUbAb=#Cxd3rQ5gY z9?uT>-E{8j<{01_Mh{DK{Unf`Ps&@`3U!UB8ro%Ebs3=jxQ4WblX>X(ltatI-9MY4N8~wf7VaI1n}%|OF#6NUU%7S z==zY8Pf#?eIf)HtPB*o_&d^r*ngvNQko#nBv6*XQ9l3$z#g-2Eis635{63kk!v!R; z(x^W`^jng!QR$i?kPq;YtX+@KmPHQh4L^ewB-O#n8w=i**yXN+JV^a+3E$EG(;=Y6 zR@Y+!S-_-qS)W+;_4f1w1r#mw*uftD=r^u-esMZOdUA_3EwJIJpT*{j&`g>1%QodK z{1dhT_02zm4aB&k%t%m=>~Z(I%(ybTe(t52m~J$pQ*4gw3te*4fBR7E|7$6i7y(1F zbKSh|qI%LSpY$Ttw388B``T2-_IWx`{qrcCYjDbsCzjUc|Ezyz9>$109 zB}^|hNcW@Sj)>?82Z;gUXMEZGiLOREc1|T1a6ua?i#UGk?JZTJ3+EK9*sf z`)1m)9~j7)_iWctoFUceH)~%w=(KLu#>6#>^K8?vdqOl83?v_)>|AHRRletQUSt>s z4%PepV+EsY`>1;!3tFxGj1BcdKgQ+T zcavA=PwuxE8+7(r9@)gK%0G@id<+wb_FU~5z8?O;Z%}*TBMc1t{N!>yW|@grw+6Pr zz?Zx2Mt@`q{PugJ0I=rqxT`junc4a4R>sJ%z*TX`dpy%WgU25a6m=rm0n;rMKZR1C z^=CX?1>MMg*>8VX1nKa#&<@i_3j2;xKvsIQ_lM{FBsr3{ixne8vg2KPZ8CuYb?On` zQ(-P55zoKR$$3OF_I@^0*a+p zuY)-`uX)n_4Kt?^*QTWd-Qp9=CK6aE@n4*}3FrBA7^ee(M831-Sn&LBab^Izo+W)Y z7<`5?729;2oV~_=to-Axz}?Kcmg~KauYyM$6Q+kb&#hD(7s>ywzCHUDbk|1Nr~3rgmAa(j z@6Ta;m_+(W5jOWZ)It`rLb2Kwns{W)+F-?FvAO@R110q?@H%4g-@Pqe_vp$8E$v_V z5d03*88~n%JHqt!t5;9U$i%-*hC}nnfjKUL0oTn7Z~iu$v(`^Wo@Beg+@&5NWjlW` zpY?I<*&W8lkVzb;PMM=MJdGFYyJM&3f#J_ui3-~93%mo8&5!2X5H6r zG?Ug-*CB2Sl@4h#77BSd6#wBHV#{j1IKS?yeB=#a_D*A@w)h}pBxEg9^qZHuM?Schw~ zuQN=38Q<8Q=_{T7>77``*k&B?Wg(`6qcWc<;6%G8OreJ$a1$6B|&p|6QHt+DSx=8V9o-EsylIQMmf*%1wu~ zJ+~53P4xns{RPZx+7H|FI&+V?a!=7IvyzS_M1I!yJU_}vsx;|U%I*RA6p7OI0yJyM zy7+h=C`u>ST)A>CbB}mPgNurn`ouv;xcOn3^ms{FQYrJxUZ*b1w!wORm)6y9*}cCp z!<1ry_pWhYi<#p6YH_Z!=@TMOymnnN%7{31o;&g3CF$?#-~A^uz0&Kspbei-$m1VA zFBQKDNw`pz!upeJr~YR5(2Qm6%)y~2%j>PgwEWScM^}EsyJgK>(>en#RpuMped;OtgYDn&_<}AtVB3GUg~auw0pY!=_g#`nvy~g<9rX*|g zEkZZeKW5i@qgkQghWn@$HJ@XYxebU+V6qEmQ>6Ucg zTJom1um`v{gTyypxk7txZ}4pPuz*PXj6W*EjmT3Sn+pCnk&f;Wfzc#MAkocF9v&>JovWKhD8SS#@1dV5PUR0`suqVyJHsF zzuC(Ju;nA2;R;VN$vI}%836j^b=X}1S0~#%vv~6;Y-pzpdcZU;vFF_;m=7}ZKXMKC zw5)4xq3a14_Nhn04~p$#a&lGgET#Y(YYh29M$nm_Av2*QY`Jo5Pb z%jSxuVp6)L*r>={y7ANqKG(HUm)MzZe~nA!)bxVaigZ14EV%3;KsQZ&Ka)9`hBNu` z=-%iuwUO6JBk|S)D!r(`v&Vf0>%&hV(){zC+l_PGE|%SNYLT=A3l8h|@>hgO zWPj$bcesxmVs-nw(-m-{T`XG+#{q5bv>xKGM`KQM@#Ac={0N8S!QC^MD>5#d(yC|j%9~^YmSF) zoB?E05f#Rv5e1Ba84w&mWd~7IcEMo>5flxs1Q!HjP;p}s5QvI;pNd>^5nO^Bu0f)x zQ85HG8c{Gryj4RIjW_Q{eBb+b=Yun+PIXt;sjlj(>gm(xKGVo$C+#jMsl*b>&y8Qh zGmy5wj4X9*TA3* zBib_xylD)WnsfcK#dIMnG+(PT7L&S=ytCe@j?7mwHrp4{L?(SSB4Tnabs{qL)?PEH#}5%6O|ztb5zZD9y<>wDw67dyK` zXGIbdO<0dKXUP-{x#he2Cjzq#c@RhH=nFkcI4eJc^yYBdt&UC%Wv1?0_bXH&ZEN-& zMfOeaoN`10#IrqPE-J&LdvZq1IX>$?WA;ryH2T|uj&(T5?5ceeI7yJNBuuRiGOvY% zrc``rV<|=hR!@SKF+DmYr7gFE8tD=xzp;*Z(<%?MXq^t3RPsg+EP3ZX+Yy17iY$0h zyN*%PWlsxE%@GQ+S)$*OhG}R)if{*o@uJeEUx)G1WdmJKuQma9D`ZVsyKOM4t@L+k zbQ4_DA#FZ|r@_B&p4rKvYgEYOs0jt3yGU84z5s7YP`J<^+^bF5k*-~WXX<2og=HI* z{>e1PmtN5z)0Be3)n+k=@!Jr| z*b=hWeNzbCqd~fAjGtx8k%6z<4KPwzf9aeS5bJoJ3wzS3kZX%yw?FDq4XG-?-fk{e z&CZTwGopJ}3<$t&oRsg8Uf;n zlh?zR{e@1R6gVosn4=B(M59;kl7%S7SGL`bjI%cBIoab1N)AELX#=PpeIlllcIuMU zapR*I2yH@KViaf=pU4%yF#`0PUXBAP zvs&)ZK}5H!w@1;r+QdKT70gu--+3ncP|Me^+3yLSi%cl^awe5%kpOdNGv=Z$k$=?q z5U2@j&E!Pr@k_jpYlfj`lk0y5(L2eBJ>E_4bI9EjVT}v~HgzMnK0y~R3iiQ#SLjbW z5&X8Gx)0IwEL~<`cpiCjn+vEQd7#+;jKb(}E$Lonu@$@erBPa^;It-5Fniex)xUQ* zAchgv@~(fOM-Ny@_@2w=w8@C*MfI*`D4+HG(Unt-(+Fy{f+7Y~VQfm?1&nu>NGvZh zt9azxp3m#(P96ztI(`;3D6^z@auW_GlTW?5D)%KREWbM+xgS|y+yKpWUA!@sE>N3T z%7G0)z-r6(H_j<99cQ6VW{te0%#<0E9*>Z7khuvZ^My5&fT5>X*@eOA%(6l~+ls45 z+pSSRpvBkIP3Cfr``wcJ+#al+-B4@WWMxkP2eF#*k2B4z7;9t$x0z^ZEp7rlVw zr~2m%SCjR=lgR%@P^Cm3zrA^)_%?ebVMXCmfk2I1Qdf0mW*f7w4hQY1LMvVo`$a88 zaF;I}o(TaTvGeI&1h)|4W&0E(uJgJ1x0rF-AjyWrIgJ#Q?o{ZfNl?EsrF{WP`0=AFgT+SGrrHbVilXW6a6n z#F?jmmaaD@3;I*oMuz<+&!z4HhE?W*I*Udp;?r}kgZ@*QNY|{|!1Qa8huJnwlU zUyn2aZV2zlWm}Y0X|-Vo1f162tQJy;nVVc}+Oh|9C0nNl=apxvkof-MJ_t`!$J||m zrr~&SY7Rs-x3zDp&`K_8{MX1?OpPWP?A+%LvjUdd(M8XKAt_$z>SK!jzPYj@o_R8y zI6dEeknYzfTLxQwuzdfmmM)A-QUgWj5Vo-9DN|t5GS3F4Rw#3FWazmYC3s2U~>V`N5%GUk(U~c z*c<@=QN*82f#yb^P4Y~GI%{!in?kI>SA}SH)o24VY-g_fBMR}cFvxk4+7qyY{^Poj zFXD{B1()Sj#Z9`qU%p)HP&VRA(i6VTJ9~f?o~>6oi&}eHsNBaqQ)jCzLMv?0-}yH2 zbC|WGh_YRxDUBONRAU>o7%gq~hGu!vIh^DftFiHGCUH2Kc5R9l)ifZc-3#J?$Iq2p z4xu7;Ro(dx2ojIfFCD1aDAKl&EP>AJ@Siw9KOasuc75<}mE8W93UHW1R9<5bNGp#^yNBeBFfwwAV~GWX3XVw|cr~6a-(w zS(hF5VyqNV6qD+RF1>nd+a~5)brR*mpGF(hNy4``ni(|>a_sPmdis+x$$zx*rd$N8 z+LJeGBaWOJ4z9m4x3r1xA;&@5#*o731skBp#=ofEi-D@utbYk%6073u<5?I-y&P0N z3zKAimrrh@8m8wm~hK9PluC5t@TGSAF(St6AZ%XyVT{2y80_m zRS|`=s;FrvouWY!P6nz#?Nx6JU(g>6VI{R=JM=HPoR#Z@oCGqS1C07G(w^Tw4;y0) zn=dnDaa#4anV^ zymFZFAropw@1(#5t!iMPbz0)EiGsYmVq2aTTfvj>N3GBy6`!0k0SYTs{M<$TG|0%! zhF+K*cnIxWaPn>QojX%OEF)J|nnL14CLFyhWHzXimI%wI6sAx#MAFP+T@o~E3?CKw zc~r6o5F2ct7KkR?by#$-*h+V9@YHeHyLagm*B`r4?Y)+!enWxCOyW`wgPGE>3!tQ` zD&)+o=LLYS&F(43aHm;c0j3#A>@t(pm{S^V3p9jW*|YAxHNB%oW(U38f-(+oQ9X}s zDPOs|8cHYGnZ%-NDe+B6ht#*%#wd-x;F6nepDqOKdBqA)NAN50D@%dafZ%8yPEEV2 z7}PTs>g?4ucXMGSf@^y)CO?)fSMMni|R z^gZOEww$uo9u6>T!c0!{7l52f=kE1qpzHiUYkqMOKE-8{@q%P3s87=VgWU1)KU$Sk;!8Fl*3U|2Er*x2UG=NR1tfx?O1F|cpe^R2vCxGkHVB4iGFqbDJ>aJmZ`hvYA;YzEo7?^oZqix%z)$22Hif5>bknrs_ieh{KrHU0cVTBQ3) z-MEIE_ybMm7oUu;ghv|x`wRVlf1&@#U+DQ-@_e3x*jmWv$p_i%$@94!j_jKK$GkFF zh;MP@^X&e9-5`6}3*bJw{h=3jeO&uk_dnVXk;4~m;{pZTFCW)x<1=cwRvR~Mz_r@A zH8if(Mi>`sBaExHagzsJ3648X<4SPcOZ&t6eAuS^efs#k4(<_%&+F`kJz8*&K!kCV zKpS`%5O>GJz5IUP9uGIqLl}3@!+nrHtp8p|>d97F;9>_{{UBTI0M8inc|+H6^u#F{ z96s}4>GN+D4Y{pGdaqPL8901H?scDc=WT1{-U09u@^?V<+us2_MZVOQ!&6QPUlHPH*m_ohyLzJebszIHW02%WD?0Dv&Fe4N zAC$5GJ+S|o{Ga&$U6z019(vHcWzI*i|ISYSF8l9U=U=e@Wxk@GKKJ|F|6pI%?L)k` zTv6gy{vu}%8fShfPY${N|FxhG4D>?-{=W_Ye<6lLLt@yVs8^uyr(XP%$`6U-hkwa& zo0pq|$(g_UTmhx=*XI12m49gcf6;o2to7bJMZIJWx9v6q_G7NfJ)EPc3qyb%)!LWT3{^@T17-IHnTJ_OH7W53r)_LAD$phi<6<>H&iS!gh_## zOU=-pWhasF4*H0g_-LdT=pn{(h>J`GQmGkZ7+VG^l$aoBWeJn|M4Bt-g1FDl&4J~1rbCu&BVH;r z-WG)h+Q!RxlSX)Y$H@4RwI3YhlMxwUFN_QlOT)2k*OVB484YisYi%DTlh+KC2b8xH zi4mR&{~~7@CKyj$-pMz>3;GMi>+LPWgK!a$9r56T2e%L8gV#jj8Q9eeKdA2}!@a97 z>mRI7g)&S8mYi|j|W7Eef}sk z)c-P2 Step 1: Downloading Parquet index file..." +aria2c \ + "${PARQUET_URL}" \ + -d "${SCRIPT_DIR}" \ + -o "${INDEX_BASE}.parquet" + +echo " Saved to: ${PARQUET_FILE}" + +# --------------------------------------------------------------------------- +# Step 2 — Extract image URLs from the Parquet file using DuckDB +# --------------------------------------------------------------------------- +echo "==> Step 2: Extracting image URLs with DuckDB..." +duckdb -noheader -csv -c \ + "SELECT mrsid_url FROM '${PARQUET_FILE}';" \ + | tr -d '"' \ + > "${URL_FILE}" + +echo " URL list written to: ${URL_FILE}" + +# --------------------------------------------------------------------------- +# Step 3 — Create the data input directory +# --------------------------------------------------------------------------- +echo "==> Step 3: Creating data input directory..." +mkdir -p "${DATA_INPUT_DIR}" +# Resolve to an absolute path to avoid any relative-path surprises +DATA_INPUT_DIR="$(cd "${DATA_INPUT_DIR}" && pwd)" +echo " Directory ready: ${DATA_INPUT_DIR}" + +# --------------------------------------------------------------------------- +# Step 4 — Download all images into the data input directory +# --------------------------------------------------------------------------- +echo "==> Step 4: Downloading images with aria2c..." +aria2c \ + -i "${URL_FILE}" \ + -j 12 \ + -x 4 \ + -d "${DATA_INPUT_DIR}" + +echo "==> Done. All files downloaded to: ${DATA_INPUT_DIR}" diff --git a/scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/generate_tiles.sh b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/generate_tiles.sh similarity index 97% rename from scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/generate_tiles.sh rename to scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/generate_tiles.sh index 2449eb5..5f5bffd 100755 --- a/scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/generate_tiles.sh +++ b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/generate_tiles.sh @@ -1,5 +1,6 @@ #!/bin/bash +# TODO: Update paths in this script PROJECT_DIR="~/Documents/Personal/Projects/dataforcanada/process-orthoimagery-dev" DATA_DIR="${PROJECT_DIR}/data" DATA_INPUT_DIR="${DATA_DIR}/input/maptiler_engine_experiments" diff --git a/scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/tune.sh b/scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/tune.sh similarity index 100% rename from scripts/ca-bc_vancouver-2021A00055915022_orthoimagery_2022_075mm/tune.sh rename to scripts/ca-bc_vancouver-2022A00055915022_d4c-datapkg-orthoimagery_2022_075mm/tune.sh