#!/bin/bash
# Re-fetch the sky-map discriminator data products (too large to bundle). Run from DFD_CLASS_verification/.
# Produces: foreground_data/sdss_z*.csv, foreground_data/2mrs_raw.tsv, lensing_map/.../dat_klm.fits
set -e
mkdir -p foreground_data lensing_map

echo "[1/3] SDSS spectroscopic galaxies (12 z-bins, ~2.1M rows) from SkyServer DR17..."
URL="https://skyserver.sdss.org/dr17/SkyServerWS/SearchTools/SqlSearch"
q(){ python3 -c "import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1]))" "$1"; }
for lo in 0.01 0.05 0.10 0.15 0.20 0.25 0.30 0.35 0.40 0.45 0.50 0.55; do
  hi=$(python3 -c "print(f'{$lo+0.05:.2f}')")
  sql="SELECT ra,dec,z FROM SpecObj WHERE class='GALAXY' AND zWarning=0 AND z>=$lo AND z<$hi"
  curl -sS -m 180 "$URL?format=csv&cmd=$(q "$sql")" | grep -E "^[0-9-]" > foreground_data/sdss_z${lo}.csv
  echo "    z[$lo,$hi]: $(wc -l < foreground_data/sdss_z${lo}.csv) galaxies"
done

echo "[2/3] 2MRS (Huchra+2012) from VizieR (bundled copy may already exist)..."
[ -f foreground_data/2mrs_raw.tsv ] || curl -sS -m 120 \
  "https://vizier.cds.unistra.fr/viz-bin/asu-tsv?-source=J/ApJS/199/26/table3&-out=RAJ2000,DEJ2000,cz,Ktmag&-out.max=unlimited" \
  | grep -vE "^#|^$" > foreground_data/2mrs_raw.tsv

echo "[3/3] Planck 2018 lensing map (~192MB) from the PLA..."
curl -L -sS -o lensing_map/planck_lensing_R3.tgz \
  "https://pla.esac.esa.int/pla-sl/data-action?COSMOLOGY.FILE_ID=COM_Lensing_4096_R3.00.tgz"
tar xzf lensing_map/planck_lensing_R3.tgz -C lensing_map
echo "Done. Pantheon+ catalog: see planck_campaign_data/PantheonPlusSH0ES.dat (+ FETCH_DATA.sh for its covariance)."
