[FEAT] Pass a separate validation dataframe (#1498)

Author: marcopeix

neuralforecast/core.py

@@ -464,6 +464,7 @@ def fit(
         df: Optional[Union[DataFrame, SparkDataFrame, Sequence[str]]] = None,
         static_df: Optional[Union[DataFrame, SparkDataFrame]] = None,
         val_size: Optional[int] = 0,
+        val_df: Optional[DataFrame] = None,
         use_init_models: bool = False,
         verbose: bool = False,
         id_col: str = "unique_id",
@@ -481,7 +482,11 @@ def fit(
             df (pandas, polars or spark DataFrame, or a list of parquet files containing the series, optional): DataFrame with columns [`unique_id`, `ds`, `y`] and exogenous variables.
                 If None, a previously stored dataset is required.
             static_df (pandas, polars or spark DataFrame, optional): DataFrame with columns [`unique_id`] and static exogenous.
-            val_size (int, optional): Size of validation set.
+            val_size (int, optional): Size of validation set. Cannot be used together with `val_df`.
+            val_df (pandas or polars DataFrame, optional): Explicit validation DataFrame with columns [`unique_id`, `ds`, `y`] and exogenous variables.
+                `val_df` can be temporally independent (no requirement that it starts immediately after `df`).
+                Cannot be used together with `val_size`. Only supported when `df` is a pandas or polars DataFrame.
+                All series in `val_df` must have the same length.
             use_init_models (bool, optional): Use initial model passed when NeuralForecast object was instantiated.
             verbose (bool): Print processing steps.
             id_col (str): Column that identifies each serie.
@@ -496,12 +501,22 @@ def fit(
         if (df is None) and not (hasattr(self, "dataset")):
             raise Exception("You must pass a DataFrame or have one stored.")
 
+        if val_df is not None and val_size != 0:
+            raise ValueError(
+                "val_df and val_size cannot be set together. "
+                "Set val_size=0 (default) when providing val_df."
+            )
+
+        if val_df is not None and not isinstance(val_df, (pd.DataFrame, pl_DataFrame)):
+            raise ValueError("val_df must be a pandas or polars DataFrame.")
+
         # Model and datasets interactions protections
         if (
             any(model.early_stop_patience_steps > 0 for model in self.models)
             and val_size == 0
+            and val_df is None
         ):
-            raise Exception("Set val_size>0 if early stopping is enabled.")
+            raise Exception("Set val_size>0 or provide a val_df if early stopping is enabled.")
 
         if (val_size is not None) and (0 < val_size < self.h):
             raise ValueError(
@@ -578,6 +593,28 @@ def fit(
                 f"`df` must be a pandas, polars or spark DataFrame, or a list of parquet files containing the series, or `None`, got: {type(df)}"
             )
 
+        if val_df is not None:
+            if isinstance(df, (SparkDataFrame,)) or (
+                isinstance(df, Sequence) and not isinstance(df, str)
+            ):
+                raise ValueError(
+                    "val_df is only supported when df is a pandas or polars DataFrame."
+                )
+            val_dataset = self.dataset.align(
+                val_df, id_col=id_col, time_col=time_col, target_col=target_col
+            )
+            if val_dataset.min_size != val_dataset.max_size:
+                raise ValueError(
+                    "All series in val_df must be of equal length. "
+                    "Found series lengths ranging from "
+                    f"{val_dataset.min_size} to {val_dataset.max_size}"
+                )
+            val_size = val_dataset.min_size
+            self.dataset = self.dataset.append(val_dataset)
+            _, _, self.last_dates, _ = TimeSeriesDataset.from_df(
+                df=val_df, id_col=id_col, time_col=time_col, target_col=target_col
+            )
+
         if val_size is not None:
             if self.dataset.min_size < val_size:
                 warnings.warn(

tests/test_core.py

@@ -139,7 +139,7 @@ def test_neural_forecast_early_stopping(setup_airplane_data):
     AirPassengersPanel_train, _ = setup_airplane_data
     models = [NHITS(h=12, input_size=12, max_steps=1, early_stop_patience_steps=5)]
     nf = NeuralForecast(models=models, freq="M")
-    with pytest.raises(Exception, match="Set val_size>0 if early stopping is enabled."):
+    with pytest.raises(Exception, match="Set val_size>0 or provide a val_df if early stopping is enabled."):
         nf.fit(AirPassengersPanel_train)
 
 
@@ -1336,6 +1336,51 @@ def test_order_of_variables_no_effect_on_val_loss(setup_airplane_data, scaler_ty
     assert valid_losses[-1][1] > 10, "Validation loss is too low"
 
 
+def test_val_df_parameter_validation(setup_airplane_data):
+    AirPassengersPanel_train, _ = setup_airplane_data
+    nf = NeuralForecast(
+        models=[NHITS(h=12, input_size=24, max_steps=1)], freq="M"
+    )
+    val_df = (
+        AirPassengersPanel_train.groupby("unique_id", observed=True)
+        .tail(12)
+        .reset_index(drop=True)
+    )
+    with pytest.raises(ValueError, match="val_df and val_size cannot be set together"):
+        nf.fit(AirPassengersPanel_train, val_size=12, val_df=val_df)
+
+
+def test_val_df_equivalence_with_val_size(setup_airplane_data):
+    # Splitting off the last 12 rows per series as val_df and passing them
+    # explicitly must produce the same valid_trajectories as using val_size=12
+    # on the full training DataFrame (same combined dataset, same random seed).
+    AirPassengersPanel_train, _ = setup_airplane_data
+    val_size = 12
+
+    train_df = (
+        AirPassengersPanel_train.groupby("unique_id", observed=True)
+        .apply(lambda x: x.iloc[:-val_size])
+        .reset_index(drop=True)
+    )
+    val_df = (
+        AirPassengersPanel_train.groupby("unique_id", observed=True)
+        .tail(val_size)
+        .reset_index(drop=True)
+    )
+
+    model_kwargs = dict(h=12, input_size=24, max_steps=10, random_seed=42)
+
+    nf_val_size = NeuralForecast(models=[NHITS(**model_kwargs)], freq="M")
+    nf_val_size.fit(AirPassengersPanel_train, val_size=val_size)
+
+    nf_val_df = NeuralForecast(models=[NHITS(**model_kwargs)], freq="M")
+    nf_val_df.fit(train_df, val_df=val_df)
+
+    losses_val_size = nf_val_size.models[0].valid_trajectories
+    losses_val_df = nf_val_df.models[0].valid_trajectories
+
+    np.testing.assert_allclose(losses_val_size, losses_val_df, atol=1e-4)
+
 
 @pytest.mark.parametrize("model,expected_error", [
     (NHITS(h=12, input_size=24, max_steps=50, hist_exog_list=["not_included"], scaler_type="robust"),