Add final tests (#231)

* Add new readers

* Add Biral SWS250 configuration files

* Add new L2M configuration files

* Improve sensor configurations

* Improve plot_nd function

* Improve tests

* Ensure keeps only second accuracy in dataset time

* Ensure correct time encoding

* Improve L2M fitting routines

* Add SWS150 sensor into optical sensors

* Ensure rounding of summary table statistics

* Solve some codacy issues

* Ensure xarray dataset is saved with datetime[s]

* Improve L0 config files tests

* Improve L0 config files tests

* Improve L2M logs and lognormal PSD GS fitting

* Add ULiege reader

* Finalize L1 tests

* Update env for bottleneck

Author: ghiggi

ci/environment.yml

@@ -16,6 +16,8 @@ dependencies:
   - scipy
   - dask[distributed]
   - xarray
+  - bottleneck
+  - flox
   - tectonic
   - matplotlib-base
   # Docs

ci/environment_latest.yml

@@ -16,6 +16,8 @@ dependencies:
   - pandas
   - dask[distributed]
   - xarray
+  - bottleneck
+  - flox
   - tectonic
   - matplotlib-base
   # Docs

disdrodb/api/create_directories.py

@@ -85,8 +85,6 @@ def create_l0_directory_structure(
     ``product = "L0A"`` will call ``run_l0a``.
     ``product = "L0B"`` will call ``run_l0b_nc``.
     """
-    from disdrodb.configs import get_data_archive_dir, get_metadata_archive_dir
-
     # Retrieve the DISDRODB Metadata Archive directory
     data_archive_dir = get_data_archive_dir(data_archive_dir)
     metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)

disdrodb/cli/disdrodb_create_summary.py

@@ -39,13 +39,22 @@
 @click_data_archive_dir_option
 @click_metadata_archive_dir_option
 @click.option("-p", "--parallel", type=bool, show_default=True, default=False, help="Read files in parallel")
+@click.option(
+    "-t",
+    "--temporal_resolution",
+    type=str,
+    show_default=True,
+    default="1MIN",
+    help="Temporal resolution of the L2E product to be used for the summary.",
+)
 def disdrodb_create_summary(
     # Stations options
     data_sources: Optional[str] = None,
     campaign_names: Optional[str] = None,
     station_names: Optional[str] = None,
     # Processing options:
     parallel=False,
+    temporal_resolution="1MIN",
     # DISDRODB root directories
     data_archive_dir: Optional[str] = None,
     metadata_archive_dir: Optional[str] = None,
@@ -92,6 +101,7 @@ def disdrodb_create_summary(
         station_names=station_names,
         # Options
         parallel=parallel,
+        temporal_resolution=temporal_resolution,
         # DISDRODB root directory
         data_archive_dir=data_archive_dir,
         metadata_archive_dir=metadata_archive_dir,

disdrodb/cli/disdrodb_create_summary_station.py

@@ -36,13 +36,22 @@
 @click_station_arguments
 @click_data_archive_dir_option
 @click.option("-p", "--parallel", type=bool, show_default=True, default=False, help="Read files in parallel")
+@click.option(
+    "-t",
+    "--temporal_resolution",
+    type=str,
+    show_default=True,
+    default="1MIN",
+    help="Temporal resolution of the L2E product to be used for the summary.",
+)
 def disdrodb_create_summary_station(
     # Station arguments
     data_source: str,
     campaign_name: str,
     station_name: str,
     # Processing options:
     parallel=False,
+    temporal_resolution="1MIN",
     # DISDRODB root directories
     data_archive_dir: Optional[str] = None,
 ):
@@ -81,6 +90,7 @@ def disdrodb_create_summary_station(
         station_name=station_name,
         # Options
         parallel=parallel,
+        temporal_resolution=temporal_resolution,
         # DISDRODB root directory
         data_archive_dir=data_archive_dir,
     )

disdrodb/constants.py

@@ -41,7 +41,7 @@
     "time",
     "sample_interval",
 ]
-OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "PWS100"]
+OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "PWS100", "SWS250"]
 IMPACT_SENSORS = ["RD80"]
 
 PRODUCTS = ["RAW", "L0A", "L0B", "L0C", "L1", "L2E", "L2M"]

disdrodb/etc/products/L1/global.yaml

@@ -1,5 +1,5 @@
 product_options:
-  fall_velocity_method: "Beard1976"
+  fall_velocity_model: "Beard1976"
   minimum_diameter: 0
   maximum_diameter: 10
   minimum_velocity: 0

disdrodb/etc/products/L2E/5MIN.yaml

@@ -0,0 +1 @@
+radar_enabled: True

disdrodb/etc/products/L2E/global.yaml

@@ -1,4 +1,4 @@
-temporal_resolutions: ["1MIN", "5MIN", "10MIN", "ROLL1MIN"]
+temporal_resolutions: [1MIN"] # "5MIN", "10MIN", "ROLL1MIN"]
 archive_options:
   strategy: time_block
   strategy_options:

disdrodb/etc/products/L2M/GAMMA_GS_ND_MAE.yaml

@@ -0,0 +1,6 @@
+psd_model: "GammaPSD"
+optimization: "GS"
+optimization_kwargs:
+  target: "ND"
+  transformation: "identity"
+  error_order: 1