Issue/86/config bug

examples/cosmos_demo.ipynb

@@ -54,7 +54,7 @@
    "source": [
     "lephare_config = lp.default_cosmos_config\n",
     "# For useable science results you must use a denser redshift grid by commenting out the following line which will revert to the config dz of 0.01.\n",
-    "lephare_config['Z_STEP']= \".1,0.,7.\"\n",
+    "lephare_config['Z_STEP']= \"0.1,0.,7.\"\n",
     "nobj=100 # Increase to run on more objects. Set to -1 to run on all.\n",
     "\n",
     "lp.data_retrieval.get_auxiliary_data(keymap=lephare_config, additional_files=[\"examples/COSMOS.in\",\"examples/output.para\"])"
@@ -130,11 +130,16 @@
     "    nondetect_val=np.nan,\n",
     "    model=\"lephare.pkl\",\n",
     "    hdf5_groupname=\"\",\n",
-    "    lephare_config=lephare_config,\n",
+    "    **{f\"lephare.{k}\": v for k, v in lephare_config.items()},\n",
     "    bands=flux_cols,\n",
     "    err_bands=flux_err_cols,\n",
     "    ref_band=flux_cols[0],\n",
     "    do_prepare = True,\n",
+    "\n",
+    "    # These overwrite the config ZSTEP so must be set\n",
+    "    zmin=float(lephare_config['Z_STEP'].split(',')[1]),\n",
+    "    zmax=float(lephare_config['Z_STEP'].split(',')[2]),\n",
+    "    nzbins=1+float(lephare_config['Z_STEP'].split(',')[2])/float(lephare_config['Z_STEP'].split(',')[0]),\n",
     ")\n",
     "\n"
    ]
@@ -282,7 +287,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "rail",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -296,7 +301,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.13.2"
+   "version": "3.10.18"
   }
  },
  "nbformat": 4,

examples/small_demo.ipynb

@@ -68,7 +68,7 @@
    "outputs": [],
    "source": [
     "lephare_config_file = \"../tests/data/lsst.para\"\n",
-    "lephare_config = lp.read_config(lephare_config_file)\n",
+    "lephare_config = lp.keymap_to_string_dict(lp.read_config(lephare_config_file))\n",
     "\n",
     "lp.data_retrieval.get_auxiliary_data(keymap=lephare_config)"
    ]
@@ -92,7 +92,7 @@
     "    model=\"lephare.pkl\",\n",
     "    hdf5_groupname=\"\",\n",
     "    # Use a sparse redshift grid to speed up the notebook\n",
-    "    lephare_config=lp.keymap_to_string_dict(lephare_config),\n",
+    "    **{f\"lephare.{k}\": v for k, v in lephare_config.items()},\n",
     "    zmin=0,\n",
     "    zmax=5,\n",
     "    nzbins=51,\n",
@@ -201,7 +201,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "rail",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -215,7 +215,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.13.2"
+   "version": "3.10.18"
   }
  },
  "nbformat": 4,

src/rail/estimation/algos/lephare.py

@@ -37,19 +37,17 @@
     }
 )
 
-star_default_config=dict(
-    LIB_ASCII="YES"
-)
+star_default_config = dict(LIB_ASCII="YES")
 
-gal_default_config=dict(
+gal_default_config = dict(
     LIB_ASCII="YES",
     MOD_EXTINC="18,26,26,33,26,33,26,33",
     EXTINC_LAW="SMC_prevot.dat,SB_calzetti.dat,SB_calzetti_bump1.dat,SB_calzetti_bump2.dat",
     EM_LINES="EMP_UV",
     EM_DISPERSION="0.5,0.75,1.,1.5,2.",
 )
 
-qso_default_config=dict(
+qso_default_config = dict(
     LIB_ASCII="YES",
     MOD_EXTINC="0,1000",
     EB_V="0.,0.1,0.2,0.3",
@@ -71,12 +69,14 @@ def _add_sub_config(
         default = val
         param = Param(dtype=dtype, default=default)
         config[f"{prefix}{key}"] = param
-    
+
 
 def _get_sub_config(config: StageConfig, prefix: str) -> dict[str, Any]:
     """Extract all config parameters that start with a
     particular prefix into a dict"""
-    out_dict = {key[len(prefix):]: val for key, val in config.items() if key.find(prefix) == 0}
+    out_dict = {
+        key[len(prefix) :]: val for key, val in config.items() if key.find(prefix) == 0
+    }
     return out_dict
 
 
@@ -101,18 +101,18 @@ class LephareInformer(CatInformer):
         ref_band=SHARED_PARAMS,
         redshift_col=SHARED_PARAMS,
     )
-    _add_sub_config(config_options, lsst_default_config, "lephare.")             
+    _add_sub_config(config_options, lsst_default_config, "lephare.")
     _add_sub_config(config_options, star_default_config, "star.")
     _add_sub_config(config_options, gal_default_config, "gal.")
-    _add_sub_config(config_options, qso_default_config, "qso.")        
+    _add_sub_config(config_options, qso_default_config, "qso.")
 
     def __init__(self, args, **kwargs):
         """Init function, init config stuff (COPIED from rail_bpz)"""
 
         super().__init__(args, **kwargs)
 
     def validate(self):
-        self.lephare_config = _get_sub_config(self.config, 'lephare.')
+        self.lephare_config = _get_sub_config(self.config, "lephare.")
 
         # Put something in place to allow for not rerunning the prepare stage
         try:
@@ -164,10 +164,10 @@ def run(self):
         # Get number of sources
         ngal = len(training_data[self.config.ref_band])
 
-        star_config = _get_sub_config(self.config, 'star.')
-        gal_config = _get_sub_config(self.config, 'gal.')
-        qso_config = _get_sub_config(self.config, 'qso.')
-        
+        star_config = _get_sub_config(self.config, "star.")
+        gal_config = _get_sub_config(self.config, "gal.")
+        qso_config = _get_sub_config(self.config, "qso.")
+
         # The three main lephare specific inform tasks
         if self.do_prepare:
             lp.prepare(
@@ -185,15 +185,11 @@ def run(self):
         self.szs = training_data[self.config.redshift_col]
 
         # Run auto adapt on training sample
-        input = _rail_to_lephare_input(
+        input_table = _rail_to_lephare_input(
             training_data, self.config.bands, self.config.err_bands
         )
         # This will return zeros if AUTO_ADAPT is NO
-        offsets = lp.calculate_offsets_from_input(self.lephare_config, input)
-        # We must make a string dictionary to allow pickling and saving
-        lephare_config = lp.keymap_to_string_dict(
-            lp.all_types_to_keymap(self.lephare_config)
-        )
+        offsets = lp.calculate_offsets_from_input(self.lephare_config, input_table)
         # Give principle inform config 'model' to instance.
         self.model = dict(
             lephare_version=lp.__version__,
@@ -272,6 +268,11 @@ class LephareEstimator(CatEstimator):
                 "is to facilitate manually moving intermediate files."
             ),
         ),
+        write_outputs=Param(
+            bool,
+            False,
+            msg="Whether to write the output files.",
+        ),
     )
     _add_sub_config(config_options, lsst_default_config, "lephare.")
 
@@ -288,10 +289,7 @@ def open_model(self, **kwargs):
             self.lephare_config = self.model["lephare_config"]
         else:
             self.lephare_config = _get_sub_config(self.config, "lephare.")
-        # Use string dictionary config in case keymap passed to estimate stage
-        self.lephare_config = lp.keymap_to_string_dict(
-            lp.all_types_to_keymap(self.lephare_config)
-        )
+
         Z_STEP = self.model["lephare_config"]["Z_STEP"]
         self.lephare_config["Z_STEP"] = Z_STEP
         self.dz = float(Z_STEP.split(",")[0])
@@ -310,12 +308,20 @@ def _process_chunk(self, start, end, data, first):
         Run the equivalent of zphota and get the PDF for every source.
         """
         # Create the lephare input table
-        input = _rail_to_lephare_input(data, self.config.bands, self.config.err_bands)
+        input_table = _rail_to_lephare_input(
+            data, self.config.bands, self.config.err_bands
+        )
         # Set the desired offsets estimate config overide lephare config overide inform offsets
         if self.config["use_inform_offsets"] and self.model["offsets"] is not None:
             offsets = self.model["offsets"]
             self.lephare_config["APPLY_SYSSHIFT"] = ",".join([str(o) for o in offsets])
-        output, photozlist = lp.process(self.lephare_config, input)
+
+        output, photozlist = lp.process(
+            self.lephare_config,
+            input_table,
+            write_outputs=self.config["write_outputs"],
+        )
+
         ng = data[self.config.bands[0]].shape[0]
         # Unpack the pdfs for galaxies
         pdfs = []
@@ -340,6 +346,8 @@ def _process_chunk(self, start, end, data, first):
         ancil = dict(zmode=zmode, zmean=zmean)
         # Add the requested outputs.
         for c in self.config["output_keys"]:
+            if output[c].dtype.kind in ["U", "O"]:
+                output[c] = output[c].astype("S")
             ancil[c] = np.array(output[c])
         qp_dstn.set_ancil(ancil)
         self._do_chunk_output(qp_dstn, start, end, first, data=data)
@@ -359,41 +367,41 @@ def _rail_to_lephare_input(data, mag_cols, mag_err_cols):
 
     Returns
     =======
-    input : astropy.table.Table
+    input_table : astropy.table.Table
         The lephare input
 
 
     """
     ng = data[mag_cols[0]].shape[0]
     # Make input catalogue in standard lephare format
-    input = Table()
+    input_table = Table()
     try:
-        input["id"] = data["id"]
+        input_table["id"] = data["id"]
     except KeyError:
-        input["id"] = np.arange(ng)
+        input_table["id"] = np.arange(ng)
     # Add all available magnitudes
 
     context = np.full(ng, 0)
     for n in np.arange(len(mag_cols)):
-        input[mag_cols[n]] = data[mag_cols[n]].T
-        input[mag_err_cols[n]] = data[mag_err_cols[n]].T
+        input_table[mag_cols[n]] = data[mag_cols[n]].T
+        input_table[mag_err_cols[n]] = data[mag_err_cols[n]].T
         # Shall we allow negative fluxes?
-        mask = input[mag_cols[n]] > 0
-        mask &= ~np.isnan(input[mag_cols[n]])
-        mask &= input[mag_err_cols[n]] > 0
-        mask &= ~np.isnan(input[mag_err_cols[n]])
+        mask = input_table[mag_cols[n]] > 0
+        mask &= ~np.isnan(input_table[mag_cols[n]])
+        mask &= input_table[mag_err_cols[n]] > 0
+        mask &= ~np.isnan(input_table[mag_err_cols[n]])
         context += mask * 2**n
     # Set context to data value if set or else exclude all negative and nan values
     try:
-        input["context"] = data["context"]
+        input_table["context"] = data["context"]
     except KeyError:
-        input["context"] = context
+        input_table["context"] = context
     try:
-        input["zspec"] = data["redshift"]
+        input_table["zspec"] = data["redshift"]
     except KeyError:
-        input["zspec"] = np.full(ng, -99.0)
-    input["string_data"] = " "
-    return input
+        input_table["zspec"] = np.full(ng, -99.0)
+    input_table["string_data"] = " "
+    return input_table
 
 
 def _update_lephare_env(lepharedir, lepharework):

tests/lephare/test_algos.py

@@ -36,18 +36,20 @@ def test_informer_and_estimator(test_data_dir: str):
 
     # Load the test params with a sparse redshift grid
     lephare_config_file = os.path.join(test_data_dir, "lsst.para")
-    lephare_config = lp.read_config(lephare_config_file)
+    lephare_config = lp.keymap_to_string_dict(lp.read_config(lephare_config_file))
     lp.data_retrieval.get_auxiliary_data(
         keymap=lephare_config,
         additional_files=["examples/output.para"],
     )
 
+    lephare_config["TEST_CONFIG"] = "dummy"
+
     inform_lephare = LephareInformer.make_stage(
         name="inform_Lephare",
         nondetect_val=np.nan,
         model="lephare.pkl",
         hdf5_groupname="",
-        lephare_config=lp.keymap_to_string_dict(lephare_config),
+        **{f"lephare.{k}": v for k, v in lephare_config.items()},
         # Use a very sparse redshift grid to speed up test:
         zmin=0,
         zmax=5,
@@ -56,6 +58,8 @@ def test_informer_and_estimator(test_data_dir: str):
 
     inform_lephare.inform(train_data_handle)
 
+    assert inform_lephare.model["lephare_config"]["TEST_CONFIG"] == "dummy"
+
     assert os.path.isfile(f"{lp.dm.LEPHAREWORK}/lib_bin/LSST_GAL_BIN.bin")
 
     estimate_lephare = LephareEstimator.make_stage(