migrate tests from nbs to pytest

.github/workflows/pytest.yml

@@ -0,0 +1,34 @@
+name: pytest
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  all-tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-14-xlarge, windows-latest]
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+    steps:
+      - name: Clone repo
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: pip install uv pytest && uv pip install --system ".[dev]" fugue[dask,spark]>=0.8.1 "dask<=2024.12.1"
+
+      - name: Run pytest
+        run: pytest tests

nbs/losses.numpy.ipynb

@@ -434,7 +434,7 @@
     "    smape = _divide_no_nan(delta_y, scale)\n",
     "    smape = 2 * np.average(smape, weights=weights, axis=axis)\n",
     "    \n",
-    "    if isinstance(smape, float):\n",
+    "    if np.isscalar(smape):\n",
     "        assert smape <= 2, 'SMAPE should be lower than 200'\n",
     "    else:\n",
     "        assert all(smape <= 2), 'SMAPE should be lower than 200'\n",

neuralforecast/losses/numpy.py

@@ -209,7 +209,7 @@ def smape(
     smape = _divide_no_nan(delta_y, scale)
     smape = 2 * np.average(smape, weights=weights, axis=axis)
 
-    if isinstance(smape, float):
+    if np.isscalar(smape):
         assert smape <= 2, "SMAPE should be lower than 200"
     else:
         assert all(smape <= 2), "SMAPE should be lower than 200"

scripts/cli.py

@@ -0,0 +1,110 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: cli.ipynb.
+
+# %% auto 0
+__all__ = ['tst_flags', 'to_skip', 'mapper', 'print_execs', 'print_hide', 'other_tests', 'get_markdown', 'get_code',
+           'extract_dir', 'no_dir_and_dir', 'get_all_tests2', 'get_all_tests', 'print_dir_in_nb']
+
+# %% cli.ipynb 1
+from functools import partial
+
+from execnb.nbio import read_nb
+from fastcore.script import call_parse
+from fastcore.xtras import Path, globtastic
+from nbdev import nbdev_export
+from nbdev.export import ExportModuleProc, nb_export
+from nbdev.maker import ModuleMaker
+from nbdev.processors import NBProcessor
+
+
+# %% cli.ipynb 4
+tst_flags = 'datasets distributed matplotlib polars pyarrow scipy'.split()
+to_skip = [
+    'showdoc',
+    'load_ext',
+    'from nbdev'
+]
+
+
+def print_execs(cell):
+    if 'exec' in cell.source: print(cell.source)
+
+def print_hide(cell):
+    if 'hide' in cell.directives_: print(cell.source)
+
+def other_tests(cell):
+    if len(cell.directives_) == 0:
+        print(cell.source)
+
+def get_markdown(cell):
+    if cell.cell_type == "markdown":
+        print(cell.source)
+
+def get_code(cell):
+    if cell.cell_type == "code":
+        print(cell.source)
+
+def extract_dir(cell, dirs):
+    for directive in dirs.split(','):
+        if directive in cell.directives_:
+            print(cell.source)
+
+def no_dir_and_dir(cell, dir):
+    if len(cell.directives_) == 0:
+        print(cell.source)
+
+    if dir in cell.directives_:
+        print(cell.source)
+
+def get_all_tests2(cell):
+    if cell.cell_type == "code":
+
+        if len(cell.directives_) == 0:
+            print(cell.source)
+
+
+        elif any(x in tst_flags + ['hide'] for x in cell.directives_):
+            # if not (x in cell.source for x in to_skip):
+            print(cell.source)
+
+def get_all_tests(cell):
+    if len(cell.directives_) == 0:
+        print(cell.source)
+
+    if any(x in tst_flags + ["hide"] for x in cell.directives_):
+        print(cell.source)
+
+
+
+# %% cli.ipynb 8
+mapper = {
+    'print_execs': print_execs,
+    'print_hide': print_hide,
+    'other_tests': other_tests,
+    'get_markdown': get_markdown,
+    'extract_dir': extract_dir,
+    'no_dir_and_dir': no_dir_and_dir,
+    'get_all_tests':get_all_tests,
+    'get_all_tests2':get_all_tests2
+}
+
+# %% cli.ipynb 9
+@call_parse
+def print_dir_in_nb(nb_path:str,
+                    dir:str=None,
+                    dir_name:str=None,
+                    ):
+    if dir_name not in mapper.keys():
+        raise ValueError(f'Choose processor from the the following: {mapper.keys()}')
+
+    if dir_name == 'extract_dir':
+        processor = NBProcessor(nb_path, partial(extract_dir, dir=dir))
+        processor.process()
+        return
+    elif dir_name == 'no_dir_and_dir':
+        processor = NBProcessor(nb_path, partial(no_dir_and_dir, dir=dir))
+        processor.process()
+        return
+
+    processor = NBProcessor(nb_path, mapper[dir_name])
+    processor.process()
+

scripts/cvt.py

@@ -0,0 +1,2 @@
+from nbdev.export import nb_export
+nb_export('cli.ipynb', lib_path='.', name='cli')

scripts/extract_test.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+echo Python: $(which python)
+source /Users/deven367/miniforge3/bin/activate
+conda activate nixtla
+
+echo Activated env: $(which python)
+
+
+mkdir -p ../tests
+
+tst_flags='datasets distributed matplotlib polars pyarrow scipy'
+
+nbs=$(ls ../nbs/*.ipynb)
+# echo "Available notebooks: $nbs"
+
+
+# this approach3 that was discussed on Slack
+# mkdir -p ../tests3
+# for flag in $tst_flags; do
+#     # echo "Extracting $flag"
+
+#     for nb in $nbs; do
+#         # get name of notebook without extension
+#         nb_name=$(basename "$nb" .ipynb)
+
+#         # echo "Processing notebook: $nb"
+#         # print_dir_in_nb "$nb" --dir_name no_dir_and_dir --dir "$flag" >> "../tests/test_$flag_$nb_name.py"
+#         print_dir_in_nb "$nb" --dir_name no_dir_and_dir --dir "$flag" >> "../tests3/test_${flag}_$nb_name.py"
+#     done
+# done
+
+for nb in $nbs; do
+    # get name of notebook without extension
+    nb_name=$(basename "$nb" .ipynb)
+
+    # echo "Processing notebook: $nb"
+    # print_dir_in_nb "$nb" --dir_name no_dir_and_dir --dir "$flag" >> "../tests/test_$flag_$nb_name.py"
+    python cli.py "$nb" --dir_name get_all_tests2 >> "../tests/test_$nb_name.py"
+done

tests/conftest.py

@@ -0,0 +1,51 @@
+import logging
+import os
+import sys
+import warnings
+
+import pytest
+
+from neuralforecast.tsdataset import TimeSeriesDataset
+from neuralforecast.utils import AirPassengersDF as Y_df
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup_test_environment():
+    """Configure the environment for all neural forecast tests."""
+    is_mac_ci = sys.platform.startswith('darwin') and os.getenv("CI", "false").lower() == "true"
+
+    if is_mac_ci:
+        # Disable memory cap (careful!) or force CPU mode
+        os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+        os.environ["PYTORCH_MPS_HIGH_WATERMARK_RATIO"] = "0.0"  # Removes MPS memory ceiling
+        os.environ["DEVICE"] = "cpu"  # Optional: if your code uses this
+        logging.warning("Running on macOS CI – forcing MPS fallback settings")
+
+    logging.getLogger("pytorch_lightning").setLevel(logging.ERROR)
+    logging.getLogger("lightning_fabric").setLevel(logging.ERROR)
+    yield
+
+
+@pytest.fixture
+def suppress_warnings():
+    """Suppress warnings for individual tests."""
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        yield
+
+
+@pytest.fixture
+def setup_dataset():
+    """Setup train/test dataset for model testing."""
+    Y_train_df = Y_df[Y_df.ds <= "1959-12-31"]  # 132 train
+    _ = Y_df[Y_df.ds > "1959-12-31"]  # 12 test
+    dataset, *_ = TimeSeriesDataset.from_df(Y_train_df)
+    return dataset
+
+
+@pytest.fixture
+def full_dataset_split():
+    """Provide both train and test dataframes if needed."""
+    Y_train_df = Y_df[Y_df.ds <= "1959-12-31"]  # 132 train
+    Y_test_df = Y_df[Y_df.ds > "1959-12-31"]  # 12 test
+    return Y_train_df, Y_test_df