[FEAT] Choose monitor for validation (#1488)

Author: marcopeix

neuralforecast/common/_base_model.py

@@ -118,6 +118,7 @@ def __init__(
         step_size: int = 1,
         num_lr_decays: int = 0,
         early_stop_patience_steps: int = -1,
+        val_monitor: str = "ptl/val_loss",
         scaler_type: str = "identity",
         futr_exog_list: Union[List, None] = None,
         hist_exog_list: Union[List, None] = None,
@@ -295,11 +296,17 @@ def __init__(
 
         # Callbacks
         if early_stop_patience_steps > 0:
+            valid_monitors = ["ptl/val_loss", "valid_loss", "train_loss"]
+            if val_monitor not in valid_monitors:
+                raise ValueError(
+                    f"val_monitor='{val_monitor}' is not supported. "
+                    f"Valid options are: {valid_monitors}."
+                )
             if "callbacks" not in trainer_kwargs:
                 trainer_kwargs["callbacks"] = []
             trainer_kwargs["callbacks"].append(
                 EarlyStopping(
-                    monitor="ptl/val_loss", patience=early_stop_patience_steps
+                    monitor=val_monitor, patience=early_stop_patience_steps
                 )
             )
 
@@ -398,6 +405,7 @@ def __init__(
             max(max_steps // self.num_lr_decays, 1) if self.num_lr_decays > 0 else 10e7
         )
         self.early_stop_patience_steps = early_stop_patience_steps
+        self.val_monitor = val_monitor
         self.val_check_steps = val_check_steps
         self.windows_batch_size = windows_batch_size
         self.step_size = step_size

neuralforecast/models/autoformer.py

@@ -431,6 +431,7 @@ class Autoformer(BaseModel):
         learning_rate (float): Learning rate between (0, 1).
         num_lr_decays (int): Number of learning rate decays, evenly distributed across max_steps.
         early_stop_patience_steps (int): Number of validation iterations before early stopping.
+        val_monitor (str): metric to monitor for early stopping. Valid options: "ptl/val_loss", "valid_loss", "train_loss". Default: "ptl/val_loss".
         val_check_steps (int): Number of training steps between every validation loss check.
         batch_size (int): number of different series in each batch.
         valid_batch_size (int): number of different series in each validation and test batch, if None uses batch_size.
@@ -487,6 +488,7 @@ def __init__(
         learning_rate: float = 1e-4,
         num_lr_decays: int = -1,
         early_stop_patience_steps: int = -1,
+        val_monitor: str = "ptl/val_loss",
         val_check_steps: int = 100,
         batch_size: int = 32,
         valid_batch_size: Optional[int] = None,
@@ -519,6 +521,7 @@ def __init__(
             learning_rate=learning_rate,
             num_lr_decays=num_lr_decays,
             early_stop_patience_steps=early_stop_patience_steps,
+            val_monitor=val_monitor,
             val_check_steps=val_check_steps,
             batch_size=batch_size,
             valid_batch_size=valid_batch_size,

neuralforecast/models/bitcn.py

@@ -104,6 +104,7 @@ class BiTCN(BaseModel):
         learning_rate (float): Learning rate between (0, 1). Default: 1e-3.
         num_lr_decays (int): Number of learning rate decays, evenly distributed across max_steps. Default: -1.
         early_stop_patience_steps (int): Number of validation iterations before early stopping. Default: -1.
+        val_monitor (str): metric to monitor for early stopping. Valid options: "ptl/val_loss", "valid_loss", "train_loss". Default: "ptl/val_loss".
         val_check_steps (int): Number of training steps between every validation loss check. Default: 100.
         batch_size (int): number of different series in each batch. Default: 32.
         valid_batch_size (int): number of different series in each validation and test batch, if None uses batch_size. Default: None.
@@ -153,6 +154,7 @@ def __init__(
         learning_rate: float = 1e-3,
         num_lr_decays: int = -1,
         early_stop_patience_steps: int = -1,
+        val_monitor: str = "ptl/val_loss",
         val_check_steps: int = 100,
         batch_size: int = 32,
         valid_batch_size: Optional[int] = None,
@@ -185,6 +187,7 @@ def __init__(
             learning_rate=learning_rate,
             num_lr_decays=num_lr_decays,
             early_stop_patience_steps=early_stop_patience_steps,
+            val_monitor=val_monitor,
             val_check_steps=val_check_steps,
             batch_size=batch_size,
             valid_batch_size=valid_batch_size,

neuralforecast/models/deepar.py

@@ -37,6 +37,7 @@ class DeepAR(BaseModel):
         learning_rate (float): Learning rate between (0, 1).
         num_lr_decays (int): Number of learning rate decays, evenly distributed across max_steps.
         early_stop_patience_steps (int): Number of validation iterations before early stopping.
+        val_monitor (str): metric to monitor for early stopping. Valid options: "ptl/val_loss", "valid_loss", "train_loss". Default: "ptl/val_loss".
         val_check_steps (int): Number of training steps between every validation loss check.
         batch_size (int): number of different series in each batch.
         valid_batch_size (int): number of different series in each validation and test batch, if None uses batch_size.
@@ -92,6 +93,7 @@ def __init__(
         learning_rate: float = 1e-3,
         num_lr_decays: int = 3,
         early_stop_patience_steps: int = -1,
+        val_monitor: str = "ptl/val_loss",
         val_check_steps: int = 100,
         batch_size: int = 32,
         valid_batch_size: Optional[int] = None,
@@ -130,6 +132,7 @@ def __init__(
             learning_rate=learning_rate,
             num_lr_decays=num_lr_decays,
             early_stop_patience_steps=early_stop_patience_steps,
+            val_monitor=val_monitor,
             val_check_steps=val_check_steps,
             batch_size=batch_size,
             valid_batch_size=valid_batch_size,

neuralforecast/models/deepnpts.py

@@ -38,6 +38,7 @@ class DeepNPTS(BaseModel):
         learning_rate (float): Learning rate between (0, 1).
         num_lr_decays (int): Number of learning rate decays, evenly distributed across max_steps.
         early_stop_patience_steps (int): Number of validation iterations before early stopping.
+        val_monitor (str): metric to monitor for early stopping. Valid options: "ptl/val_loss", "valid_loss", "train_loss". Default: "ptl/val_loss".
         val_check_steps (int): Number of training steps between every validation loss check.
         batch_size (int): number of different series in each batch.
         valid_batch_size (int): number of different series in each validation and test batch, if None uses batch_size.
@@ -89,6 +90,7 @@ def __init__(
         learning_rate: float = 1e-3,
         num_lr_decays: int = 3,
         early_stop_patience_steps: int = -1,
+        val_monitor: str = "ptl/val_loss",
         val_check_steps: int = 100,
         batch_size: int = 32,
         valid_batch_size: Optional[int] = None,
@@ -136,6 +138,7 @@ def __init__(
             learning_rate=learning_rate,
             num_lr_decays=num_lr_decays,
             early_stop_patience_steps=early_stop_patience_steps,
+            val_monitor=val_monitor,
             val_check_steps=val_check_steps,
             batch_size=batch_size,
             valid_batch_size=valid_batch_size,

neuralforecast/models/dilated_rnn.py

@@ -309,6 +309,7 @@ class DilatedRNN(BaseModel):
         learning_rate (float): Learning rate between (0, 1).
         num_lr_decays (int): Number of learning rate decays, evenly distributed across max_steps.
         early_stop_patience_steps (int): Number of validation iterations before early stopping.
+        val_monitor (str): metric to monitor for early stopping. Valid options: "ptl/val_loss", "valid_loss", "train_loss". Default: "ptl/val_loss".
         val_check_steps (int): Number of training steps between every validation loss check.
         batch_size (int): number of different series in each batch.
         valid_batch_size (int): number of different series in each validation and test batch.
@@ -362,6 +363,7 @@ def __init__(
         learning_rate: float = 1e-3,
         num_lr_decays: int = 3,
         early_stop_patience_steps: int = -1,
+        val_monitor: str = "ptl/val_loss",
         val_check_steps: int = 100,
         batch_size=32,
         valid_batch_size: Optional[int] = None,
@@ -395,6 +397,7 @@ def __init__(
             learning_rate=learning_rate,
             num_lr_decays=num_lr_decays,
             early_stop_patience_steps=early_stop_patience_steps,
+            val_monitor=val_monitor,
             val_check_steps=val_check_steps,
             batch_size=batch_size,
             valid_batch_size=valid_batch_size,

neuralforecast/models/dlinear.py

@@ -64,6 +64,7 @@ class DLinear(BaseModel):
         learning_rate (float): Learning rate between (0, 1).
         num_lr_decays (int): Number of learning rate decays, evenly distributed across max_steps.
         early_stop_patience_steps (int): Number of validation iterations before early stopping.
+        val_monitor (str): metric to monitor for early stopping. Valid options: "ptl/val_loss", "valid_loss", "train_loss". Default: "ptl/val_loss".
         val_check_steps (int): Number of training steps between every validation loss check.
         batch_size (int): number of different series in each batch.
         valid_batch_size (int): number of different series in each validation and test batch, if None uses batch_size.
@@ -111,6 +112,7 @@ def __init__(
         learning_rate: float = 1e-4,
         num_lr_decays: int = -1,
         early_stop_patience_steps: int = -1,
+        val_monitor: str = "ptl/val_loss",
         val_check_steps: int = 100,
         batch_size: int = 32,
         valid_batch_size: Optional[int] = None,
@@ -143,6 +145,7 @@ def __init__(
             learning_rate=learning_rate,
             num_lr_decays=num_lr_decays,
             early_stop_patience_steps=early_stop_patience_steps,
+            val_monitor=val_monitor,
             val_check_steps=val_check_steps,
             batch_size=batch_size,
             windows_batch_size=windows_batch_size,

neuralforecast/models/fedformer.py

@@ -427,6 +427,7 @@ class FEDformer(BaseModel):
         learning_rate (float): Learning rate between (0, 1).
         num_lr_decays (int): Number of learning rate decays, evenly distributed across max_steps.
         early_stop_patience_steps (int): Number of validation iterations before early stopping.
+        val_monitor (str): metric to monitor for early stopping. Valid options: "ptl/val_loss", "valid_loss", "train_loss". Default: "ptl/val_loss".
         val_check_steps (int): Number of training steps between every validation loss check.
         batch_size (int): number of different series in each batch.
         valid_batch_size (int): number of different series in each validation and test batch, if None uses batch_size.
@@ -485,6 +486,7 @@ def __init__(
         learning_rate: float = 1e-4,
         num_lr_decays: int = -1,
         early_stop_patience_steps: int = -1,
+        val_monitor: str = "ptl/val_loss",
         val_check_steps: int = 100,
         batch_size: int = 32,
         valid_batch_size: Optional[int] = None,
@@ -516,6 +518,7 @@ def __init__(
             learning_rate=learning_rate,
             num_lr_decays=num_lr_decays,
             early_stop_patience_steps=early_stop_patience_steps,
+            val_monitor=val_monitor,
             val_check_steps=val_check_steps,
             batch_size=batch_size,
             valid_batch_size=valid_batch_size,

neuralforecast/models/gru.py

@@ -47,6 +47,7 @@ class GRU(BaseModel):
         learning_rate (float): Learning rate between (0, 1).
         num_lr_decays (int): Number of learning rate decays, evenly distributed across max_steps.
         early_stop_patience_steps (int): Number of validation iterations before early stopping.
+        val_monitor (str): metric to monitor for early stopping. Valid options: "ptl/val_loss", "valid_loss", "train_loss". Default: "ptl/val_loss".
         val_check_steps (int): Number of training steps between every validation loss check.
         batch_size (int): number of different series in each batch.
         valid_batch_size (int): number of different series in each validation and test batch.
@@ -105,6 +106,7 @@ def __init__(
         learning_rate: float = 1e-3,
         num_lr_decays: int = -1,
         early_stop_patience_steps: int = -1,
+        val_monitor: str = "ptl/val_loss",
         val_check_steps: int = 100,
         batch_size=32,
         valid_batch_size: Optional[int] = None,
@@ -142,6 +144,7 @@ def __init__(
             learning_rate=learning_rate,
             num_lr_decays=num_lr_decays,
             early_stop_patience_steps=early_stop_patience_steps,
+            val_monitor=val_monitor,
             val_check_steps=val_check_steps,
             batch_size=batch_size,
             valid_batch_size=valid_batch_size,

neuralforecast/models/hint.py

@@ -160,6 +160,7 @@ def __init__(
         self.h = h
         self.model = model
         self.early_stop_patience_steps = model.early_stop_patience_steps
+        self.val_monitor = model.val_monitor
         self.S = S
         self.reconciliation = reconciliation
         self.loss = model.loss