[FIX] Mismtached quantiles in loss and valid_loss (#1470)

Author: marcopeix

neuralforecast/core.py

@@ -73,9 +73,9 @@
 )
 
 from .common._base_auto import BaseAuto, MockTrial
-from .common._base_model import DistributedConfig
+from .common._base_model import DistributedConfig, MULTIQUANTILE_LOSSES
 from .compat import SparkDataFrame
-from .losses.pytorch import HuberIQLoss, IQLoss
+from .losses.pytorch import HuberIQLoss, IQLoss, sCRPS
 
 # this disables warnings about the number of workers in the dataloaders
 # which the user can't control
@@ -248,6 +248,26 @@ def __init__(
             model.h == models[0].h for model in models
         ), "All models should have the same horizon"
 
+        for model in models:
+            valid_loss = getattr(model, "valid_loss", None)
+            if isinstance(valid_loss, sCRPS):
+                valid_qs = valid_loss.mql.quantiles
+            elif isinstance(valid_loss, MULTIQUANTILE_LOSSES):
+                valid_qs = valid_loss.quantiles
+            else:
+                continue
+            loss = getattr(model, "loss", None)
+            loss_qs = getattr(loss, "quantiles", None)
+            if loss_qs is None:
+                continue
+            if sorted(loss_qs.tolist()) != sorted(valid_qs.tolist()):
+                raise ValueError(
+                    f"{model.__class__.__name__}: `loss` ({loss.__class__.__name__}) "
+                    f"quantiles {loss_qs.tolist()} do not match `valid_loss` "
+                    f"({valid_loss.__class__.__name__}) quantiles {valid_qs.tolist()}. "
+                    f"Ensure both use the same `level` or `quantiles` argument."
+                )
+
         self.h = models[0].h
         self.models_init = models
         self.freq = freq

tests/test_core.py

@@ -59,6 +59,7 @@
     PMM,
     DistributionLoss,
     MQLoss,
+    sCRPS,
 )
 from neuralforecast.tsdataset import TimeSeriesDataset
 from neuralforecast.utils import (
@@ -2251,3 +2252,30 @@ def test_mase_validation_loss_scale(setup_airplane_data):
         f"MASE validation loss is {valid_loss}, which indicates the scale mismatch "
         f"bug may have regressed. Expected < 50 for a properly scaled MASE."
     )
+
+
+@pytest.mark.parametrize("loss,valid_loss", [
+    # DistributionLoss uses NLL — mismatched levels are allowed
+    (DistributionLoss(distribution="Normal", level=[80, 90]), DistributionLoss(distribution="Normal", level=[50])),
+    # GMM + sCRPS with matching quantiles
+    (GMM(n_components=5, level=[80, 90]), sCRPS(level=[80, 90])),
+    # MQLoss with matching quantiles
+    (MQLoss(level=[80, 90]), MQLoss(level=[80, 90])),
+])
+def test_loss_valid_loss_quantiles_allowed(loss, valid_loss):
+    """Loss/valid_loss combinations that should not raise a quantile mismatch error."""
+    model = NHITS(h=12, input_size=24, loss=loss, valid_loss=valid_loss, max_steps=1)
+    NeuralForecast(models=[model], freq="M")
+
+
+@pytest.mark.parametrize("loss,valid_loss", [
+    # MQLoss quantiles embedded in loss computation — mismatch must be rejected
+    (MQLoss(level=[80, 90]), MQLoss(level=[50])),
+    # GMM + sCRPS reproduces the issue
+    (GMM(n_components=5, level=[80, 90]), sCRPS(level=[50])),
+])
+def test_loss_valid_loss_quantiles_mismatch_raises(loss, valid_loss):
+    """Loss/valid_loss combinations with mismatched quantiles must raise ValueError."""
+    model = NHITS(h=12, input_size=24, loss=loss, valid_loss=valid_loss, max_steps=1)
+    with pytest.raises(ValueError, match="quantiles.*do not match"):
+        NeuralForecast(models=[model], freq="M")