From 11657af14eb55ed440822378e6b7be947ea52872 Mon Sep 17 00:00:00 2001 From: bungerbuilder Date: Wed, 15 Jul 2026 09:29:22 -0400 Subject: [PATCH 01/10] Add files via upload --- crowdsource/Rubin_proc.py | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 crowdsource/Rubin_proc.py diff --git a/crowdsource/Rubin_proc.py b/crowdsource/Rubin_proc.py new file mode 100644 index 0000000..3b7b559 --- /dev/null +++ b/crowdsource/Rubin_proc.py @@ -0,0 +1,66 @@ +import numpy as np +import argparse, os, pdb +import crowdsource.psf as psfmod +from crowdsource import crowdsource_base +from lsst.daf.butler import Butler +from functools import partial + + +def process(visitId, detector, nx = 4, ny = 4, maxstars = 10000000, fewstars = 60, **kw): + """Use the RSP Butler to find the Rubin image""" + + butler = Butler("dp1", collections="LSSTComCam/DP1") + + dataset_refs = list(butler.query_datasets( + "visit_image", + where="visit.id = :visitId AND detector.id = :detector", + bind={"visitId": visitId, "detector": detector}, + )) + + if len(dataset_refs) == 0: + raise RuntimeError( + f"No visit_image found for visit={visitId}, detector={detector}") + + else: + print(f"Visit image found for visit={visitId}, detector={detector}") + + ref = list(dataset_refs)[0] + visit_image = butler.get(ref) + + """Get the PSF using wise_psf_fit""" + + rubin_psf = visit_image.getPsf() + visit_center = visit_image.getBBox().getCenter() + psf_stamp_visit = rubin_psf.computeImage(visit_center).array + + stamp = np.clip(np.array(psf_stamp_visit), 1e-10, np.inf) + stamp = stamp / np.sum(stamp) + + psf = psfmod.SimplePSF(stamp) + psf.fitfun = partial(psfmod.wise_psf_fit, psfstamp=stamp) + + """Making the variables for CROWDSOURCE""" + + im = visit_image.image.array.astype(np.float32) + + var = visit_image.variance.array + sqivar = np.where(var > 0, 1.0 / np.sqrt(var), 0.0).astype(np.float32) + + mask = visit_image.mask + + bad_bits = 0 + for plane in ("BAD", "SAT", "CR", "NO_DATA", "EDGE", "INTRP"): + if plane in mask.getMaskPlaneDict(): + bad_bits |= mask.getPlaneBitMask(plane) + + flag = (mask.array & bad_bits).astype(np.uint32) + sqivar[(mask.array & bad_bits) != 0] = 0.0 + + """Run CROWDSOURCE on an image""" + + res = crowdsource_base.fit_im(im, psf, sqivar, dq=flag, refit_psf=True, + verbose = True, ntilex=nx, ntiley=ny, maxiter = 10, threshold = 400, **kw) + + print("CROWDSOURCE is done!") + + return res, visit_image \ No newline at end of file From c533636a62d0ebe083f793f5a4d251c45ce018d7 Mon Sep 17 00:00:00 2001 From: bungerbuilder Date: Wed, 15 Jul 2026 09:32:28 -0400 Subject: [PATCH 02/10] Rename Rubin_proc.py to rubin_proc.py --- crowdsource/{Rubin_proc.py => rubin_proc.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename crowdsource/{Rubin_proc.py => rubin_proc.py} (98%) diff --git a/crowdsource/Rubin_proc.py b/crowdsource/rubin_proc.py similarity index 98% rename from crowdsource/Rubin_proc.py rename to crowdsource/rubin_proc.py index 3b7b559..e8f4d30 100644 --- a/crowdsource/Rubin_proc.py +++ b/crowdsource/rubin_proc.py @@ -63,4 +63,4 @@ def process(visitId, detector, nx = 4, ny = 4, maxstars = 10000000, fewstars = 6 print("CROWDSOURCE is done!") - return res, visit_image \ No newline at end of file + return res, visit_image From b50e89269ea9cfeac88d959f9f6c5b25bd2b9bb4 Mon Sep 17 00:00:00 2001 From: bungerbuilder Date: Wed, 15 Jul 2026 09:36:44 -0400 Subject: [PATCH 03/10] Update rubin_proc.py Changed res to not include hard-coded analysis settings. --- crowdsource/rubin_proc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crowdsource/rubin_proc.py b/crowdsource/rubin_proc.py index e8f4d30..d32e0f8 100644 --- a/crowdsource/rubin_proc.py +++ b/crowdsource/rubin_proc.py @@ -59,7 +59,7 @@ def process(visitId, detector, nx = 4, ny = 4, maxstars = 10000000, fewstars = 6 """Run CROWDSOURCE on an image""" res = crowdsource_base.fit_im(im, psf, sqivar, dq=flag, refit_psf=True, - verbose = True, ntilex=nx, ntiley=ny, maxiter = 10, threshold = 400, **kw) + verbose = True, ntilex=nx, ntiley=ny, **kw) print("CROWDSOURCE is done!") From e6456896320f8fa509eba68eec0c0e00c037c6af Mon Sep 17 00:00:00 2001 From: bungerbuilder Date: Wed, 15 Jul 2026 14:11:22 -0400 Subject: [PATCH 04/10] Update crowdsource/rubin_proc.py Co-authored-by: Eddie Schlafly --- crowdsource/rubin_proc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crowdsource/rubin_proc.py b/crowdsource/rubin_proc.py index d32e0f8..ae09bc1 100644 --- a/crowdsource/rubin_proc.py +++ b/crowdsource/rubin_proc.py @@ -39,7 +39,7 @@ def process(visitId, detector, nx = 4, ny = 4, maxstars = 10000000, fewstars = 6 psf = psfmod.SimplePSF(stamp) psf.fitfun = partial(psfmod.wise_psf_fit, psfstamp=stamp) - """Making the variables for CROWDSOURCE""" + """Making the variables for crowdsource""" im = visit_image.image.array.astype(np.float32) From cbf31f9235f8f447707d1492d540d4dbfeb73845 Mon Sep 17 00:00:00 2001 From: bungerbuilder Date: Wed, 15 Jul 2026 14:11:34 -0400 Subject: [PATCH 05/10] Update crowdsource/rubin_proc.py Co-authored-by: Eddie Schlafly --- crowdsource/rubin_proc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crowdsource/rubin_proc.py b/crowdsource/rubin_proc.py index ae09bc1..5265584 100644 --- a/crowdsource/rubin_proc.py +++ b/crowdsource/rubin_proc.py @@ -6,7 +6,7 @@ from functools import partial -def process(visitId, detector, nx = 4, ny = 4, maxstars = 10000000, fewstars = 60, **kw): +def process(visitId, detector, nx=4, ny=4, maxstars=10000000, fewstars=60, **kw): """Use the RSP Butler to find the Rubin image""" butler = Butler("dp1", collections="LSSTComCam/DP1") From 8387c104a9d8a9e7549649937dbd7f9cfa210849 Mon Sep 17 00:00:00 2001 From: bungerbuilder Date: Wed, 15 Jul 2026 14:11:46 -0400 Subject: [PATCH 06/10] Update crowdsource/rubin_proc.py Co-authored-by: Eddie Schlafly --- crowdsource/rubin_proc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crowdsource/rubin_proc.py b/crowdsource/rubin_proc.py index 5265584..ca22f59 100644 --- a/crowdsource/rubin_proc.py +++ b/crowdsource/rubin_proc.py @@ -56,7 +56,7 @@ def process(visitId, detector, nx=4, ny=4, maxstars=10000000, fewstars=60, **kw) flag = (mask.array & bad_bits).astype(np.uint32) sqivar[(mask.array & bad_bits) != 0] = 0.0 - """Run CROWDSOURCE on an image""" + """Run crowdsource on an image""" res = crowdsource_base.fit_im(im, psf, sqivar, dq=flag, refit_psf=True, verbose = True, ntilex=nx, ntiley=ny, **kw) From f6b630cfd7868da5f6de553dff525bb373f6e89c Mon Sep 17 00:00:00 2001 From: bungerbuilder Date: Wed, 15 Jul 2026 14:11:51 -0400 Subject: [PATCH 07/10] Update crowdsource/rubin_proc.py Co-authored-by: Eddie Schlafly --- crowdsource/rubin_proc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crowdsource/rubin_proc.py b/crowdsource/rubin_proc.py index ca22f59..22ba864 100644 --- a/crowdsource/rubin_proc.py +++ b/crowdsource/rubin_proc.py @@ -61,6 +61,6 @@ def process(visitId, detector, nx=4, ny=4, maxstars=10000000, fewstars=60, **kw) res = crowdsource_base.fit_im(im, psf, sqivar, dq=flag, refit_psf=True, verbose = True, ntilex=nx, ntiley=ny, **kw) - print("CROWDSOURCE is done!") + print("crowdsource is done!") return res, visit_image From 765fb148ac630413a76b9a0fde69896994b9fa4d Mon Sep 17 00:00:00 2001 From: bungerbuilder Date: Thu, 16 Jul 2026 15:54:49 -0400 Subject: [PATCH 08/10] Update rubin_proc.py Changed so that process() only gives back res. Also added arguments for a CLI line. --- crowdsource/rubin_proc.py | 106 ++++++++++++++++++++++++++++++++------ 1 file changed, 91 insertions(+), 15 deletions(-) diff --git a/crowdsource/rubin_proc.py b/crowdsource/rubin_proc.py index 22ba864..e3c3bef 100644 --- a/crowdsource/rubin_proc.py +++ b/crowdsource/rubin_proc.py @@ -1,14 +1,32 @@ + import numpy as np import argparse, os, pdb import crowdsource.psf as psfmod from crowdsource import crowdsource_base from lsst.daf.butler import Butler from functools import partial + + +def process(visitId, detector, nx = 4, ny = 4, maxstars = 10000000, fewstars = 60, threshold = 5, **kw): + """ + Parameters + ---------- + visitId : Rubin images are taken with an associated visitId attatched in the metadata. There are nine + images associated with one visitId. + detector: Determines which of the nine images will be processed. + + Process: + 1. Use the RSP Butler to find the Rubin image + 2. Get the PSF using wise_psf_fit + 3. Making the variables for CROWDSOURCE + 4. Run CROWDSOURCE on an image + Returns + ------- + res : The processed image as an array. -def process(visitId, detector, nx=4, ny=4, maxstars=10000000, fewstars=60, **kw): - """Use the RSP Butler to find the Rubin image""" - + """ + butler = Butler("dp1", collections="LSSTComCam/DP1") dataset_refs = list(butler.query_datasets( @@ -16,19 +34,18 @@ def process(visitId, detector, nx=4, ny=4, maxstars=10000000, fewstars=60, **kw) where="visit.id = :visitId AND detector.id = :detector", bind={"visitId": visitId, "detector": detector}, )) - + if len(dataset_refs) == 0: raise RuntimeError( f"No visit_image found for visit={visitId}, detector={detector}") - + else: print(f"Visit image found for visit={visitId}, detector={detector}") ref = list(dataset_refs)[0] visit_image = butler.get(ref) - """Get the PSF using wise_psf_fit""" - + #Getting the PSF rubin_psf = visit_image.getPsf() visit_center = visit_image.getBBox().getCenter() psf_stamp_visit = rubin_psf.computeImage(visit_center).array @@ -39,8 +56,7 @@ def process(visitId, detector, nx=4, ny=4, maxstars=10000000, fewstars=60, **kw) psf = psfmod.SimplePSF(stamp) psf.fitfun = partial(psfmod.wise_psf_fit, psfstamp=stamp) - """Making the variables for crowdsource""" - + #crowdsource variables im = visit_image.image.array.astype(np.float32) var = visit_image.variance.array @@ -56,11 +72,71 @@ def process(visitId, detector, nx=4, ny=4, maxstars=10000000, fewstars=60, **kw) flag = (mask.array & bad_bits).astype(np.uint32) sqivar[(mask.array & bad_bits) != 0] = 0.0 - """Run crowdsource on an image""" - + #run crowdsource res = crowdsource_base.fit_im(im, psf, sqivar, dq=flag, refit_psf=True, - verbose = True, ntilex=nx, ntiley=ny, **kw) - - print("crowdsource is done!") + verbose = True, ntilex=nx, ntiley=ny, maxiter = 10, threshold = threshold, **kw) + + print("CROWDSOURCE is done!") - return res, visit_image + return res + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Run CROWDSOURCE on a Rubin visit_image retrieved via the Butler." + ) + parser.add_argument("visitId", type=int, + help="Visit ID of the Rubin image to process.") + parser.add_argument("detector", type=int, + help="Detector ID (selects which of the nine images to process).") + parser.add_argument("--nx", type=int, default=4, + help="Number of tiles in x for CROWDSOURCE (default: 4).") + parser.add_argument("--ny", type=int, default=4, + help="Number of tiles in y for CROWDSOURCE (default: 4).") + parser.add_argument("--maxstars", type=int, default=10000000, + help="Maximum number of stars to fit (default: 10000000).") + parser.add_argument("--fewstars", type=int, default=60, + help="Threshold below which a tile is considered to have few stars (default: 60).") + parser.add_argument("--threshold", type=float, default=5, + help="Detection threshold in sigma (default: 5).") + parser.add_argument("--maxiter", type=int, default=10, + help="Maximum number of CROWDSOURCE iterations (default: 10).") + parser.add_argument("-o", "--outfile", type=str, default=None, + help="Path to save the output (pickle). If not given, " + "defaults to 'rubin_visit{visitId}_det{detector}.pkl'.") + parser.add_argument("--pdb", action="store_true", + help="Drop into a pdb debugger on exception.") + return parser.parse_args() + + +def main(): + args = parse_args() + + try: + res = process( + args.visitId, + args.detector, + nx=args.nx, + ny=args.ny, + maxstars=args.maxstars, + fewstars=args.fewstars, + threshold=args.threshold, + maxiter=args.maxiter, + ) + except Exception: + if args.pdb: + pdb.post_mortem() + raise + + outfile = args.outfile or f"rubin_visit{args.visitId}_det{args.detector}.pkl" + + import pickle + with open(outfile, "wb") as f: + pickle.dump(res, f) + + print(f"Results saved to {outfile}") + + +if __name__ == "__main__": + main() + From fef56a63e6bd7c9ee927b81ae3183b8d0b0a7d29 Mon Sep 17 00:00:00 2001 From: bungerbuilder Date: Thu, 16 Jul 2026 16:02:17 -0400 Subject: [PATCH 09/10] Update rubin_proc.py Changed from functions --> more like simple_proc codes. --- crowdsource/rubin_proc.py | 82 +++++++++++++-------------------------- 1 file changed, 27 insertions(+), 55 deletions(-) diff --git a/crowdsource/rubin_proc.py b/crowdsource/rubin_proc.py index e3c3bef..b516491 100644 --- a/crowdsource/rubin_proc.py +++ b/crowdsource/rubin_proc.py @@ -81,62 +81,34 @@ def process(visitId, detector, nx = 4, ny = 4, maxstars = 10000000, fewstars = 6 return res -def parse_args(): - parser = argparse.ArgumentParser( - description="Run CROWDSOURCE on a Rubin visit_image retrieved via the Butler." - ) - parser.add_argument("visitId", type=int, - help="Visit ID of the Rubin image to process.") - parser.add_argument("detector", type=int, - help="Detector ID (selects which of the nine images to process).") - parser.add_argument("--nx", type=int, default=4, - help="Number of tiles in x for CROWDSOURCE (default: 4).") - parser.add_argument("--ny", type=int, default=4, - help="Number of tiles in y for CROWDSOURCE (default: 4).") - parser.add_argument("--maxstars", type=int, default=10000000, - help="Maximum number of stars to fit (default: 10000000).") - parser.add_argument("--fewstars", type=int, default=60, - help="Threshold below which a tile is considered to have few stars (default: 60).") - parser.add_argument("--threshold", type=float, default=5, - help="Detection threshold in sigma (default: 5).") - parser.add_argument("--maxiter", type=int, default=10, - help="Maximum number of CROWDSOURCE iterations (default: 10).") - parser.add_argument("-o", "--outfile", type=str, default=None, - help="Path to save the output (pickle). If not given, " - "defaults to 'rubin_visit{visitId}_det{detector}.pkl'.") - parser.add_argument("--pdb", action="store_true", - help="Drop into a pdb debugger on exception.") - return parser.parse_args() - - -def main(): - args = parse_args() - - try: - res = process( - args.visitId, - args.detector, - nx=args.nx, - ny=args.ny, - maxstars=args.maxstars, - fewstars=args.fewstars, - threshold=args.threshold, - maxiter=args.maxiter, - ) - except Exception: - if args.pdb: - pdb.post_mortem() - raise - - outfile = args.outfile or f"rubin_visit{args.visitId}_det{args.detector}.pkl" - - import pickle - with open(outfile, "wb") as f: - pickle.dump(res, f) +if __name__ == "__main__": + from astropy.io import fits - print(f"Results saved to {outfile}") + parser = argparse.ArgumentParser(description='Run crowdsource on a Rubin visit_image') + # 3 arguments: visitId, detector, outfn + parser.add_argument('visitId', type=int, nargs=1) + parser.add_argument('detector', type=int, nargs=1) + parser.add_argument('outfn', type=str, nargs=1) + parser.add_argument('--nx', '-x', type=int, default=4, + help='number of tiles in x') + parser.add_argument('--ny', '-y', type=int, default=4, + help='number of tiles in y') + parser.add_argument('--maxstars', type=int, default=10000000, + help='maximum number of stars to fit') + parser.add_argument('--fewstars', type=int, default=60, + help='number of stars below which a tile is considered to have few stars') + parser.add_argument('--threshold', '-t', type=float, default=5, + help='detection threshold in sigma') + args = parser.parse_args() + visitId = args.visitId[0] + detector = args.detector[0] + outfn = args.outfn[0] -if __name__ == "__main__": - main() + res = process(visitId, detector, nx=args.nx, ny=args.ny, + maxstars=args.maxstars, fewstars=args.fewstars, + threshold=args.threshold) + fits.writeto(outfn, res[0], overwrite=True) + fits.append(outfn, res[1][0]) + fits.append(outfn, res[2][0]) From 78b42a09e9baa57fde988ae06b017cf0fa82e35d Mon Sep 17 00:00:00 2001 From: bungerbuilder Date: Wed, 29 Jul 2026 20:33:33 -0400 Subject: [PATCH 10/10] Update rubin_proc.py Now using fit_variable_moffat_psf --- crowdsource/rubin_proc.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/crowdsource/rubin_proc.py b/crowdsource/rubin_proc.py index b516491..ebb55fc 100644 --- a/crowdsource/rubin_proc.py +++ b/crowdsource/rubin_proc.py @@ -5,8 +5,8 @@ from crowdsource import crowdsource_base from lsst.daf.butler import Butler from functools import partial - - + + def process(visitId, detector, nx = 4, ny = 4, maxstars = 10000000, fewstars = 60, threshold = 5, **kw): """ Parameters @@ -48,13 +48,16 @@ def process(visitId, detector, nx = 4, ny = 4, maxstars = 10000000, fewstars = 6 #Getting the PSF rubin_psf = visit_image.getPsf() visit_center = visit_image.getBBox().getCenter() - psf_stamp_visit = rubin_psf.computeImage(visit_center).array + + psf_stamp_visit = rubin_psf.computeKernelImage(visit_center).array stamp = np.clip(np.array(psf_stamp_visit), 1e-10, np.inf) stamp = stamp / np.sum(stamp) psf = psfmod.SimplePSF(stamp) - psf.fitfun = partial(psfmod.wise_psf_fit, psfstamp=stamp) + + print("Using fit_variable_moffat_psf") + psf.fitfun = psfmod.fit_variable_moffat_psf #crowdsource variables im = visit_image.image.array.astype(np.float32) @@ -65,7 +68,7 @@ def process(visitId, detector, nx = 4, ny = 4, maxstars = 10000000, fewstars = 6 mask = visit_image.mask bad_bits = 0 - for plane in ("BAD", "SAT", "CR", "NO_DATA", "EDGE", "INTRP"): + for plane in ("BAD", "SAT", "CR", "NO_DATA", "EDGE", "INTRP", "STREAK"): if plane in mask.getMaskPlaneDict(): bad_bits |= mask.getPlaneBitMask(plane) @@ -80,7 +83,6 @@ def process(visitId, detector, nx = 4, ny = 4, maxstars = 10000000, fewstars = 6 return res - if __name__ == "__main__": from astropy.io import fits