Run tests against several PostgreSQL versions using Nix (#1492)

* Show test output with postgrest-test-spec* in nix-shell

* Test output in color

default.nix

@@ -22,12 +22,23 @@ let
 
   overlays =
     [
+      (import nix/overlays/postgresql-legacy.nix)
       (import nix/overlays/gitignore.nix)
     ];
 
   pkgs =
     import pinnedPkgs { inherit overlays; };
 
+  postgresqlVersions =
+    [
+      pkgs.postgresql_12
+      pkgs.postgresql_11
+      pkgs.postgresql_10
+      pkgs.postgresql_9_6
+      pkgs.postgresql_9_5
+      pkgs.postgresql_9_4
+    ];
+
   drv =
     pkgs.haskellPackages.callCabal2nix name src {};
 in
@@ -53,4 +64,11 @@ rec {
   # Utility for updating the pinned version of Nixpkgs.
   nixpkgsUpgrade =
     pkgs.callPackage nix/nixpkgs-upgrade.nix {};
+
+  tests =
+    pkgs.callPackage nix/tests.nix
+      {
+        inherit postgresqlVersions;
+        postgrestBuildEnv = env;
+      };
 }

nix/README.md

@@ -42,6 +42,25 @@ Within `nix-shell`, you can run Cabal commands as usual. You can also run
 stack with the `--nix` option, which causes stack to pick up the non-Haskell
 dependencies from the same pinned Nixpkgs version that the Nix builds use.
 
+## Testing
+
+In nix-shell, you'll find utility scripts that make it very easy to run the
+Haskell test suite, including setting up all required dependencies and
+temporary test databases:
+
+```bash
+# Run the tests against the most recent version of PostgreSQL:
+nix-shell --run postgrest-test-spec
+
+# Run the tests against all supported versions of PostgreSQL:
+nix-shell --run postgrest-test-spec-all
+
+# Run the tests against a specific version of PostgreSQL (use tab-completion in
+# nix-shell to see all available versions):
+nix-shell --run postgrest-test-spec-postgresql-9.5.21
+
+```
+
 ## Tour
 
 The following is not required for working on PostgREST with Nix, but it will

nix/overlays/postgresql-legacy.nix

@@ -0,0 +1,20 @@
+self: super:
+# Overlay that adds legacy versions of PostgreSQL that are supported by
+# PostgREST.
+{
+  # PostgreSQL 9.4 was removed from Nixpkgs with
+  # https://github.com/NixOS/nixpkgs/commit/8e2fc57a80d761c46702c3250e61c1bffe021e25
+  # We pin its parent commit 3b5b9a7 to get the last version that was available.
+  postgresql_9_4 =
+    let
+      rev = "3b5b9a73f59ff93d50156e250203410bdd07f4e0";
+      tarballHash = "0kr25xqbv1ldp72jbmml21pc5hl7xcfqhclv5qxa5f860jddjznk";
+
+      pinnedPkgs =
+        builtins.fetchTarball {
+          url = "https://github.com/nixos/nixpkgs/archive/${rev}.tar.gz";
+          sha256 = tarballHash;
+        };
+    in
+      (import pinnedPkgs {}).pkgs.postgresql_9_4;
+}

nix/tests.nix

@@ -0,0 +1,90 @@
+# Utilities for running the PostgREST test suite
+
+{ buildEnv
+, cabal-install
+, git
+, lib
+, postgrestBuildEnv
+, postgresqlVersions
+, runtimeShell
+, writeShellScript
+, writeShellScriptBin
+}:
+let
+  # Wrap the `test/with_tmp_db` script with the required dependencies from Nix.
+  withTmpDb =
+    postgresql:
+      writeShellScript "postgrest-test-${postgresql.name}"
+        ''
+          set -euo pipefail
+
+          export PATH=${postgresql}/bin:${git}/bin:${runtimeShell}/bin:"$PATH"
+
+          exec ${../test/with_tmp_db} "$@"
+        '';
+
+  # Script to run the Haskell test suite against a specific version of
+  # PostgreSQL.
+  testSpec =
+    name: postgresql:
+      writeShellScriptBin name
+        ''
+          set -euo pipefail
+
+          cat << EOF
+
+          Running spec against ${postgresql.name}...
+
+          EOF
+
+          # TODO: Make this work outside nix-shell when installed with nix-env.
+          # Probably using postgrestBuildEnv somehow?
+          ${withTmpDb postgresql} ${cabal-install}/bin/cabal v2-test \
+            --test-show-detail=direct
+
+          cat << EOF
+
+          Done running spec against ${postgresql.name}.
+
+          EOF
+        '';
+
+  # The PostgreSQL version that we run the tests against by default.
+  defaultPostgresql =
+    builtins.head postgresqlVersions;
+
+  defaultTestSpec =
+    testSpec "postgrest-test-spec" defaultPostgresql;
+
+  # Create a `testSpec` for each PostgreSQL version that we want to test
+  # against.
+  testSpecVersions =
+    map
+      (postgresql: testSpec "postgrest-test-spec-${postgresql.name}" postgresql)
+      postgresqlVersions;
+
+  # Helper script for running the tests against all PostgreSQL versions.
+  testSpecAllVersions =
+    let
+      testRunners =
+        map (test: "${test}/bin/${test.name}") testSpecVersions;
+    in
+      writeShellScriptBin "postgrest-test-spec-all"
+        ''
+          set -euo pipefail
+
+          ${lib.concatStringsSep "\n" testRunners}
+        '';
+in
+  # Create an environment that contains all the utility scripts for running tests
+  # that we defined above.
+buildEnv {
+  name =
+    "postgrest-tests";
+
+  paths =
+    [
+      defaultTestSpec
+      testSpecAllVersions
+    ] ++ testSpecVersions;
+}

shell.nix

@@ -8,6 +8,7 @@ pkgs.lib.overrideDerivation env (
         pkgs.cabal2nix
         pkgs.postgresql
         nixpkgsUpgrade
+        tests
       ];
 
     shellHook =