Add SOFTSSharp model (#1516)

Co-authored-by: Marco <marco@nixtla.io>

Author: hljubic

docs/mintlify/docs.json

@@ -112,6 +112,7 @@
               "models.rmok.html",
               "models.rnn.html",
               "models.softs.html",
+              "models.softssharp.html",
               "models.stemgnn.html",
               "models.tcn.html",
               "models.tft.html",
@@ -162,4 +163,4 @@
       "href": "https://github.com/Nixtla/neuralforecast"
     }
   }
-}
+}

docs/mintlify/imgs_models/softssharp.png

@@ -31,7 +31,7 @@ The optimization process uses temporal cross-validation where the validation set
 
 ## 2. Available AutoModels
 
-NeuralForecast provides 34 `AutoModel` variants, each wrapping a specific forecasting model with automatic hyperparameter optimization. Each `AutoModel` has a `default_config` attribute that defines sensible search spaces for its corresponding model.
+NeuralForecast provides 35 `AutoModel` variants, each wrapping a specific forecasting model with automatic hyperparameter optimization. Each `AutoModel` has a `default_config` attribute that defines sensible search spaces for its corresponding model.
 
 ### RNN-Based Models
 Recurrent neural networks for sequential forecasting:
@@ -83,6 +83,7 @@ Models designed for specific forecasting scenarios:
 - `AutoKAN`: [Kolmogorov-Arnold Network for time series](./models.kan.html)
 - `AutoStemGNN`: [Graph neural network for multivariate forecasting](./models.stemgnn.html)
 - `AutoSOFTS`: [Spectral Optimal Fourier Transform model](./models.softs.html)
+- `AutoSOFTSSharp`: [SOFTS extension with stochastic variable-position encoding](./models.softssharp.html)
 - `AutoTimeMixer`: [Temporal mixing architecture](./models.timemixer.html)
 - `AutoRMoK`: [Random Mixture of Kernels](./models.rmok.html)
 - `AutoHINT`: [Hierarchical forecasting with automatic reconciliation](./models.hint.html)
@@ -191,4 +192,4 @@ model = AutoHINT(
 
 model.fit(dataset=dataset, val_size=4)
 y_hat = model.predict(dataset=dataset)
-```
+```

docs/models.html.md

@@ -0,0 +1,78 @@
+---
+description: >-
+  SOFTSSharp: SOFTS extension with stochastic variable-position encoding for multivariate time series forecasting.
+output-file: models.softssharp.html
+title: SOFTSSharp
+---
+
+SOFTSSharp extends SOFTS by stochastically adding variable-position embeddings and multiple dropout layers inside the STAD aggregation-redistribution component, aiming to improve forecasting accuracy while preserving linear complexity.
+
+
+![Figure 1. Architecture of SOFTSSharp](imgs_models/softssharp.png)
+*Figure 1. Architecture of SOFTSSharp*
+
+## 1. SOFTSSharp
+
+::: neuralforecast.models.softssharp.SOFTSSharp
+    options:
+      members:
+        - fit
+        - predict
+      heading_level: 3
+
+### Usage example
+
+```python
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from neuralforecast import NeuralForecast
+from neuralforecast.models import SOFTSSharp
+from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic
+from neuralforecast.losses.pytorch import MASE
+
+Y_train_df = AirPassengersPanel[AirPassengersPanel.ds<AirPassengersPanel['ds'].values[-12]].reset_index(drop=True)
+Y_test_df = AirPassengersPanel[AirPassengersPanel.ds>=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True)
+
+model = SOFTSSharp(h=12,
+                   input_size=24,
+                   n_series=2,
+                   hidden_size=256,
+                   d_core=256,
+                   e_layers=2,
+                   d_ff=64,
+                   dropout=0.1,
+                   pe_keep_prob=0.5,
+                   use_norm=True,
+                   loss=MASE(seasonality=4),
+                   early_stop_patience_steps=3,
+                   batch_size=32)
+
+fcst = NeuralForecast(models=[model], freq='ME')
+fcst.fit(df=Y_train_df, static_df=AirPassengersStatic, val_size=12)
+forecasts = fcst.predict(futr_df=Y_test_df)
+
+fig, ax = plt.subplots(1, 1, figsize = (20, 7))
+Y_hat_df = forecasts.reset_index(drop=False).drop(columns=['unique_id','ds'])
+plot_df = pd.concat([Y_test_df, Y_hat_df], axis=1)
+plot_df = pd.concat([Y_train_df, plot_df])
+
+plot_df = plot_df[plot_df.unique_id=='Airline1'].drop('unique_id', axis=1)
+plt.plot(plot_df['ds'], plot_df['y'], c='black', label='True')
+plt.plot(plot_df['ds'], plot_df['SOFTSSharp'], c='blue', label='Forecast')
+ax.set_title('AirPassengers Forecast', fontsize=22)
+ax.set_ylabel('Monthly Passengers', fontsize=20)
+ax.set_xlabel('Year', fontsize=20)
+ax.legend(prop={'size': 15})
+ax.grid()
+```
+
+## 2. Auxiliary functions
+
+::: neuralforecast.models.softssharp.PositionalEmbedding
+    options:
+      members: []
+
+::: neuralforecast.models.softssharp.STADSharp
+    options:
+      members: []

docs/models.softssharp.html.md

@@ -37,6 +37,7 @@
     "|`RMoK` | `AutoRMoK` | KAN | Multivariate | Direct | - |\n",
     "|`RNN` | `AutoRNN` | RNN | Univariate | Both<sup>8</sup>  | F/H/S | \n",
     "|`SOFTS` | `AutoSOFTS` | MLP | Multivariate | Direct  | - | \n",
+    "|`SOFTSSharp` | `AutoSOFTSSharp` | MLP | Multivariate | Direct  | - | \n",
     "|`StemGNN` | `AutoStemGNN` | GNN | Multivariate | Direct  | - | \n",
     "|`TCN` | `AutoTCN` | CNN | Univariate | Direct  | F/H/S | \n",
     "|`TFT` | `AutoTFT` | Transformer | Univariate | Direct  | F/H/S | \n",

nbs/docs/capabilities/overview.ipynb

@@ -5,8 +5,8 @@
            'AutoNBEATS', 'AutoNBEATSx', 'AutoNHITS', 'AutoDLinear', 'AutoNLinear', 'AutoTiDE', 'AutoDeepNPTS',
            'AutoKAN', 'AutoTFT', 'AutoVanillaTransformer', 'AutoInformer', 'AutoAutoformer', 'AutoFEDformer',
            'AutoPatchTST', 'AutoiTransformer', 'AutoTimeXer', 'AutoTimesNet', 'AutoStemGNN', 'AutoHINT', 'AutoTSMixer',
-           'AutoTSMixerx', 'AutoMLPMultivariate', 'AutoSOFTS', 'AutoTimeMixer', 'AutoRMoK', 'AutoXLinear',
-           'RayOptions', 'OptunaOptions']
+           'AutoTSMixerx', 'AutoMLPMultivariate', 'AutoSOFTS', 'AutoSOFTSSharp', 'AutoTimeMixer', 'AutoRMoK',
+           'AutoXLinear', 'RayOptions', 'OptunaOptions']
 
 
 from ray import tune
@@ -37,6 +37,7 @@
 from .models.rmok import RMoK
 from .models.rnn import RNN
 from .models.softs import SOFTS
+from .models.softssharp import SOFTSSharp
 from .models.stemgnn import StemGNN
 from .models.tcn import TCN
 from .models.tft import TFT
@@ -2575,6 +2576,92 @@ def get_default_config(cls, h, backend, n_series):
         return config
 
 
+class AutoSOFTSSharp(BaseAuto):
+
+    default_config = {
+        "input_size_multiplier": [1, 2, 3, 4, 5],
+        "h": None,
+        "n_series": None,
+        "hidden_size": tune.choice([64, 128, 256, 512]),
+        "d_core": tune.choice([64, 128, 256, 512]),
+        "pe_keep_prob": tune.choice([0.25, 0.5, 0.75, 1.0]),
+        "learning_rate": tune.loguniform(1e-4, 1e-1),
+        "scaler_type": tune.choice([None, "robust", "standard", "identity"]),
+        "max_steps": tune.choice([500, 1000, 2000]),
+        "batch_size": tune.choice([32, 64, 128, 256]),
+        "loss": None,
+        "random_seed": tune.randint(1, 20),
+    }
+
+    def __init__(
+        self,
+        h,
+        n_series,
+        loss=MAE(),
+        valid_loss=None,
+        config=None,
+        search_alg=BasicVariantGenerator(random_state=1),
+        num_samples=10,
+        time_budget=None,
+        refit_with_val=False,
+        cpus=None,
+        gpus=None,
+        verbose=False,
+        alias=None,
+        backend="ray",
+        callbacks=None,
+        ray_options=None,
+        optuna_options=None,
+    ):
+
+        if config is None:
+            config = self.get_default_config(h=h, backend=backend, n_series=n_series)
+
+        if backend == "ray":
+            config["n_series"] = n_series
+        elif backend == "optuna":
+            mock_trial = MockTrial()
+            if (
+                "n_series" in config(mock_trial)
+                and config(mock_trial)["n_series"] != n_series
+            ) or ("n_series" not in config(mock_trial)):
+                raise Exception(f"config needs 'n_series': {n_series}")
+
+        super(AutoSOFTSSharp, self).__init__(
+            cls_model=SOFTSSharp,
+            h=h,
+            loss=loss,
+            valid_loss=valid_loss,
+            config=config,
+            search_alg=search_alg,
+            num_samples=num_samples,
+            time_budget=time_budget,
+            refit_with_val=refit_with_val,
+            cpus=cpus,
+            gpus=gpus,
+            verbose=verbose,
+            alias=alias,
+            backend=backend,
+            callbacks=callbacks,
+            ray_options=ray_options,
+            optuna_options=optuna_options,
+        )
+
+    @classmethod
+    def get_default_config(cls, h, backend, n_series):
+        config = cls.default_config.copy()
+        config["input_size"] = tune.choice(
+            [h * x for x in config["input_size_multiplier"]]
+        )
+        config["step_size"] = tune.choice([1, h])
+        del config["input_size_multiplier"]
+        if backend == "optuna":
+            config["n_series"] = n_series
+            config = cls._ray_config_to_optuna(config)
+
+        return config
+
+
 class AutoTimeMixer(BaseAuto):
 
     default_config = {

neuralforecast/auto.py

@@ -32,6 +32,7 @@
     NHITS,
     RNN,
     SOFTS,
+    SOFTSSharp,
     TCN,
     TFT,
     Autoformer,
@@ -196,6 +197,8 @@ def _insample_times(
     "autodeepnpts": DeepNPTS,
     "softs": SOFTS,
     "autosofts": SOFTS,
+    "softssharp": SOFTSSharp,
+    "autosoftssharp": SOFTSSharp,
     "timemixer": TimeMixer,
     "autotimemixer": TimeMixer,
     "kan": KAN,

neuralforecast/core.py

@@ -2,7 +2,7 @@
            'MLP', 'NHITS', 'NBEATS', 'NBEATSx', 'DLinear', 'NLinear',
            'TFT', 'VanillaTransformer', 'Informer', 'Autoformer', 'PatchTST', 'FEDformer',
            'StemGNN', 'HINT', 'TimesNet', 'TimeLLM', 'TSMixer', 'TSMixerx', 'MLPMultivariate',
-           'iTransformer', 'BiTCN', 'TiDE', 'DeepNPTS', 'SOFTS', 'TimeMixer', 'KAN', 'RMoK',
+           'iTransformer', 'BiTCN', 'TiDE', 'DeepNPTS', 'SOFTS', 'SOFTSSharp', 'TimeMixer', 'KAN', 'RMoK',
            'TimeXer', 'xLSTM', 'XLinear'
            ]
 
@@ -36,6 +36,7 @@
 from .tide import TiDE
 from .deepnpts import DeepNPTS
 from .softs import SOFTS
+from .softssharp import SOFTSSharp
 from .timemixer import TimeMixer
 from .kan import KAN
 from .rmok import RMoK