Merge pull request #1212 from nix-community/nix-github-actions

Redefine Github Actions in Nix using nix-github-actions

Author: adisbladis

.github/ci.nix

@@ -0,0 +1,43 @@
+let
+  sources = import ../nix/sources.nix;
+
+  inherit (import sources.nix-github-actions) mkGithubMatrix;
+
+  mkPkgs = system: import sources.nixpkgs {
+    config = {
+      allowAliases = false;
+      allowInsecurePredicate = x: true;
+    };
+    overlays = [
+      (import ../overlay.nix)
+    ];
+    inherit system;
+  };
+
+in
+mkGithubMatrix {
+  attrPrefix = "";
+  checks = {
+    x86_64-linux =
+      let
+        pkgs = mkPkgs "x86_64-linux";
+      in
+      import ../tests { inherit pkgs; };
+
+    x86_64-darwin =
+      let
+        pkgs = mkPkgs "x86_64-darwin";
+        inherit (pkgs) lib;
+
+
+        tests = import ../tests { inherit pkgs; };
+      in
+      {
+        # Aggregate all tests into one derivation so that only one GHA runner is scheduled for all darwin jobs
+        aggregate = pkgs.runCommand "darwin-aggregate"
+          {
+            env.TEST_INPUTS = (lib.concatStringsSep " " (lib.attrValues (lib.filterAttrs (n: v: lib.isDerivation v) tests)));
+          } "touch $out";
+      };
+  };
+}

.github/workflows/ci.yml

@@ -43,58 +43,39 @@ jobs:
     - name: Check format
       run: nix-shell --arg packages 'pkgs:[ pkgs.p2nix-tools.env ]' --run 'black --check .'
 
-  matrix_generate:
+  nix-matrix:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - uses: actions/checkout@v3
       - uses: cachix/install-nix-action@v22
-        with:
-          nix_path: nixpkgs=channel:nixos-unstable
       - id: set-matrix
+        name: Generate Nix Matrix
         run: |
-          set -euo pipefail
-
-          matrix="$(nix-instantiate --eval --json --expr 'builtins.attrNames (import ./tests {})' | jq -rcM '{attr: .}')"
-          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+          set -Eeu
+          echo "matrix=$(nix eval --json -f ./.github/ci.nix matrix)" >> "$GITHUB_OUTPUT"
 
-  builds-linux:
-    needs: matrix_generate
-    runs-on: ubuntu-latest
+  nix-build:
+    needs: nix-matrix
+    runs-on: ${{ matrix.os }}
     strategy:
-      matrix: ${{fromJSON(needs.matrix_generate.outputs.matrix)}}
+      matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}}
     steps:
       - uses: actions/checkout@v3
       - uses: cachix/install-nix-action@v22
-        with:
-          nix_path: nixpkgs=channel:nixos-unstable
-      - uses: cachix/cachix-action@v12
-        with:
-          name: poetry2nix
-          signingKey: "VhaWuN3IyJVpWg+aZvTocVB+W8ziZKKRGLKR53Pkld3YRZxYOUfXZf0fvqF+LkqVW0eA60trVd5vsqNONpX9Hw=="
-      - run: nix-build --keep-going --show-trace tests/default.nix -A '${{ matrix.attr }}'
-
-  builds-macos:
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: cachix/install-nix-action@v22
-        with:
-          nix_path: nixpkgs=channel:nixos-unstable
       - uses: cachix/cachix-action@v12
         with:
           name: poetry2nix
           signingKey: "VhaWuN3IyJVpWg+aZvTocVB+W8ziZKKRGLKR53Pkld3YRZxYOUfXZf0fvqF+LkqVW0eA60trVd5vsqNONpX9Hw=="
-      - run: nix-build --keep-going --show-trace tests/default.nix -A '${{ matrix.attr }}'
+      - run: nix-build --keep-going --show-trace .github/ci.nix -A '${{ matrix.attr }}'
 
   collect:
     runs-on: ubuntu-latest
     needs:
       - nixpkgs-fmt
       - black-fmt
-      - builds-linux
-      - builds-macos
+      - nix-build
       - sort-build-systems
     steps:
-      - run: exit 0
+      - run: true

nix/sources.json

@@ -1,4 +1,16 @@
 {
+    "nix-github-actions": {
+        "branch": "master",
+        "description": "A library to turn Nix Flake attribute sets into Github Actions matrices [maintainer=@adisbladis]",
+        "homepage": "",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301",
+        "sha256": "0rkh2dcnl4rna4snqd7cdnvddjyqrgj3rais0dkhs4ygz64j8ip1",
+        "type": "tarball",
+        "url": "https://github.com/nix-community/nix-github-actions/archive/165b1650b753316aa7f1787f3005a8d2da0f5301.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
     "nixpkgs": {
         "branch": "master",
         "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",

tests/default.nix

@@ -18,14 +18,11 @@ let
   pep425 = pkgs.callPackage ../pep425.nix { inherit poetryLib; python = pkgs.python3; };
   pep425PythonOldest = pkgs.callPackage ../pep425.nix { inherit poetryLib; python = pkgs.python38; };
   pep425OSX = pkgs.callPackage ../pep425.nix { inherit poetryLib; isLinux = false; python = pkgs.python3; };
-  skipTests = builtins.filter (t: builtins.typeOf t != "list") (builtins.split "," (builtins.getEnv "SKIP_TESTS"));
   callTest = test: attrs: pkgs.callPackage test ({ inherit poetry2nix; } // attrs);
 
-  # HACK: Return null on MacOS since the test in question fails
-  skipOSX = drv: if pkgs.stdenv.isDarwin then builtins.trace "Note: Skipping ${drv.name} on OSX" (pkgs.runCommand drv.name { } "touch $out") else drv;
+  inherit (pkgs) lib stdenv;
 
 in
-builtins.removeAttrs
 {
   trivial = callTest ./trivial { };
 
@@ -39,7 +36,6 @@ builtins.removeAttrs
   common-pkgs-1 = callTest ./common-pkgs-1 { };
   common-pkgs-2 = callTest ./common-pkgs-2 { };
   pep425 = pkgs.callPackage ./pep425 { inherit pep425; inherit pep425OSX; inherit pep425PythonOldest; };
-  pep600 = skipOSX (callTest ./pep600 { });
   env = callTest ./env { };
   pytest-metadata = callTest ./pytest-metadata { };
   pytest-randomly = callTest ./pytest-randomly { };
@@ -62,25 +58,16 @@ builtins.removeAttrs
     inherit poetry;
     inherit (pkgs) postgresql;
   };
-  # pyqt5 = skipOSX (callTest ./pyqt5 { });
   extras = callTest ./extras { };
   source-filter = callTest ./source-filter { };
   canonical-module-names = callTest ./canonical-module-names { };
   wandb = callTest ./wandb { };
   utf8-pyproject = callTest ./utf8-pyproject { };
 
-  # Test deadlocks on darwin, sandboxing issue?
-  dependency-environment = skipOSX (callTest ./dependency-environment { });
-
-  # Editable tests fails on Darwin because of sandbox paths
-  editable = skipOSX (callTest ./editable { });
-  editable-egg = skipOSX (callTest ./editable-egg { });
-
   ansible-molecule = callTest ./ansible-molecule { };
   bcrypt = callTest ./bcrypt { };
   mk-poetry-packages = callTest ./mk-poetry-packages { };
   markupsafe2 = callTest ./markupsafe2 { };
-  pendulum = skipOSX (callTest ./pendulum { });
   # uwsgi = callTest ./uwsgi { };  # Commented out because build is flaky (unrelated to poetry2nix)
   jq = callTest ./jq { };
   ubersmith = callTest ./ubersmith { };
@@ -94,12 +81,6 @@ builtins.removeAttrs
   watchfiles = callTest ./watchfiles { };
   sqlalchemy = callTest ./sqlalchemy { };
   tzlocal = callTest ./tzlocal { };
-  text-generation-webui = skipOSX (callTest ./text-generation-webui { });
-
-  # Cross tests fail on darwin for some strange reason:
-  # ERROR: MarkupSafe-2.0.1-cp39-cp39-linux_aarch64.whl is not a supported wheel on this platform.
-  extended-cross = skipOSX (callTest ./extended-cross { });
-  trivial-cross = skipOSX (callTest ./trivial-cross { });
 
   ml-stack = callTest ./ml-stack { };
 
@@ -171,11 +152,23 @@ builtins.removeAttrs
   cairocffi-no-wheel = callTest ./cairocffi-no-wheel { };
   rpds-py = callTest ./rpds-py { };
 
-  # Currently broken
-  # pandas = callTest ./pandas { };
-  # Inherit test cases from nixpkgs
-  # nixops = pkgs.nixops;
-  nixops_unstable = skipOSX pkgs.nixops_unstable;
+} // lib.optionalAttrs (!stdenv.isDarwin) {
+  # pyqt5 = (callTest ./pyqt5 { });
 
+  # Test deadlocks on darwin, sandboxing issue?
+  dependency-environment = (callTest ./dependency-environment { });
+
+  # Editable tests fails on Darwin because of sandbox paths
+  pep600 = (callTest ./pep600 { });
+  editable = (callTest ./editable { });
+  editable-egg = (callTest ./editable-egg { });
+  pendulum = (callTest ./pendulum { });
+
+  # Fails because of missing inputs on darwin
+  text-generation-webui = callTest ./text-generation-webui { };
+
+  # Cross tests fail on darwin for some strange reason:
+  # ERROR: MarkupSafe-2.0.1-cp39-cp39-linux_aarch64.whl is not a supported wheel on this platform.
+  extended-cross = (callTest ./extended-cross { });
+  trivial-cross = (callTest ./trivial-cross { });
 }
-  skipTests