Upgrade nixpkgs to current master and add an upgrade checklist (#1579)

* simplify haskell packages overrides based on new nixpkgs
* update Nix gitignore
* simplify Haskell packages overlay

Co-authored-by: monacoremo <monacoremo>

nix/README.md

@@ -196,3 +196,8 @@ attribute with useful utilities that will be put on the PATH in `nix-shell`.
 
 Our overlays to the Nix package set are defined here. They allow us to tweak our
 `pkgs` in `default.nix` by adding new packages or overriding existing ones.
+
+## Upgrading dependencies
+
+See the [upgrading checklist](UPGRADE.md) for how to upgrade the PostgREST
+dependencies.

nix/UPGRADE.md

@@ -0,0 +1,91 @@
+# Checklist for upgrading Nix dependencies
+
+The Nix dependencies of PostgREST should be updated regularly, in most cases it
+should be a very simple operation.
+
+```bash
+# Update pinned version of Nixpkgs
+nix-shell --run nixpkgs-upgrade > nix/nixpkgs-version.nix
+
+# Verify that everything builds
+nix-build
+```
+
+The following checklist guides you through the complete process in more detail.
+
+## Upgrade the pinned version of `nixpkgs`
+
+The pinned version of [`nixpkgs`](https://github.com/NixOS/nixpkgs) is defined
+in [`nix/nixpkgs-version.nix`](nixpkgs-version.nix). The pin refers directly to
+a GitHub tarball for the given revision, which is more efficient than pulling
+the complete Git repository. To upgrade it to the current `master` of
+`nixpkgs`, you can use a small utility script defined in
+[`nix/nixpkgs-update.nix`](nixpkgs-update.nix):
+
+```bash
+# From the root of the repository, enter nix-shell
+nix-shell
+
+# Run the utility script to pin the latest revision in master
+nixpkgs-upgrade > nix/nixpkgs-version.nix
+
+# Exit the nix-shell with Ctrl-d
+
+```
+
+## Update pinned version of `static-haskell-nix`
+
+We pin [`static-haskell-nix`](https://github.com/nh2/static-haskell-nix) in
+[`nix/static-haskell-package.nix`](static-haskell-package.nix). Upgrade the
+pinned revision and the tarball hash if necessary. See
+[`nix/nixpkgs-upgrade.nix`](nixpkgs-upgrade.nix) for how to get the correct
+tarball hash, or just change the hash to an arbitrary value of correct length,
+run `nix-build` and use the expected value from the resulting error message.
+
+## Review overlays
+
+Check whether the individual [overlays](overlays) are still required.
+
+## Check if patches are still required and update them as needed
+
+We track a number of PostgREST-specific patches in [`nix/patches`](patches).
+Check whether the pull-requests/issues linked in the
+[`default.nix`](patches/default.nix) have progressed and remove/modify the
+patches if they did. If conflicting changes occurred, you might have to rebase
+the respective patches.
+
+## Build everything
+
+Using the PostgREST binary Nix cache is recommended. Install
+[Cachix](https://cachix.org/) and run `cachix use postgrest`.
+
+Run `nix-build` in the root directory of the project to build all PostgREST
+artifacts. This might take a long time, e.g. when our static GHC version needs
+to be rebuilt due to changes to some underlying package. If there are any
+errors, this is probably due to one of our patches. Try to fix them and re-run
+`nix-build` until everything builds.
+
+## Update the PostgREST binary cache
+
+If you have access to the PostgREST cachix signing key, you can push the
+artifacts that you built locally to the binary cache. This will accelerate the
+CI builds and tests, sometimes dramatically. This might sometimes even be
+required to avoid build timeouts in CI.
+
+You'll need to set the `CACHIX_SIGNING_KEY` before proceeding, e.g. by creating
+a file containing `export CACHIX_SIGNING_KEY=...` and sourcing that file, which
+avoids having the secret in you shell history.
+
+To push all new artifacts to Cachix, run:
+
+```
+nix-store -qR --include-outputs $$(nix-instantiate) | cachix push postgrest
+
+# Or, equivalently
+make cachix-push-all
+
+```
+
+The `nix-store` command will query the nix-store to list all dependencies and
+build artifacts of PostgREST. The `cachix` command will efficiently push
+everything that is not yet cached to the binary cache.

nix/docker/default.nix

@@ -57,7 +57,8 @@ let
         docker load -i ${image}
       '';
 in
-buildEnv {
+buildEnv
+  {
     name = "postgrest-docker";
     paths = [ load ];
   } // { inherit image config; }

nix/nixpkgs-version.nix

@@ -1,6 +1,6 @@
 # Pinned version of Nixpkgs, generated with nixpkgs-upgrade.
 {
-  date = "2020-07-12";
+  date = "2020-09-04";
-  rev = "c87c474b17af792e7984ef4f058291f7ce06f594";
+  rev = "7bc3a08d3a4c700b53a3b27f5acd149f24b931ec";
-  tarballHash = "1171bwg07dcaqgayacaqwk3gyq97hi261gr7a4pgbrkafqb5r3ds";
+  tarballHash = "1kiz37052zsgvw7a378zg08mpbi1wk8dkgm5j6dy0x4mxvcg8ws3";
 }

nix/overlays/default.nix

@@ -1,7 +1,7 @@
 {
   gitignore = import ./gitignore.nix;
   ghr = import ./ghr;
-  haskell-packages = import ./haskell-packages;
+  haskell-packages = import ./haskell-packages.nix;
   postgresql-default = import ./postgresql-default.nix;
   postgresql-legacy = import ./postgresql-legacy.nix;
 }

nix/overlays/ghr/ghr.nix

@@ -12,4 +12,7 @@ buildGoModule rec {
   };
 
   vendorSha256 = "14avsngzhl1b8a05i43ph6sxh9vj0jls0acxr9j7r0h3f0vpamcj";
+
+  # Disabling tests, as they require a GitHub API token
+  doCheck = false;
 }

nix/overlays/gitignore.nix

@@ -12,8 +12,8 @@ self: super:
       gitignoreSrc = super.fetchFromGitHub {
         owner = "hercules-ci";
         repo = "gitignore";
-        rev = "2ced4519f865341adcb143c5d668f955a2cb997f";
+        rev = "c4662e662462e7bf3c2a968483478a665d00e717";
-        sha256 = "sha256:0fc5bgv9syfcblp23y05kkfnpgh3gssz6vn24frs8dzw39algk2z";
+        sha256 = "1npnx0h6bd0d7ql93ka7azhj40zgjp815fw2r6smg8ch9p7mzdlx";
       };
     in
     (super.callPackage gitignoreSrc { }).gitignoreSource;

nix/overlays/haskell-packages.nix

@@ -0,0 +1,31 @@
+{ compiler, extraOverrides ? (final: prev: { }) }:
+
+self: super:
+let
+  overrides =
+    final: prev:
+    rec {
+      # To pin custom versions of Haskell packages:
+      #   protolude =
+      #     prev.callHackageDirect
+      #       {
+      #         pkg = "protolude";
+      #         ver = "0.3.0";
+      #         sha256 = "0iwh4wsjhb7pms88lw1afhdal9f86nrrkkvv65f9wxbd1b159n72";
+      #       }
+      #       { };
+      #
+      # To get the sha256:
+      #   nix-prefetch-url --unpack https://hackage.haskell.org/package/protolude-0.3.0/protolude-0.3.0.tar.gz
+      protolude = prev.protolude_0_3_0;
+    } // extraOverrides final prev;
+in
+{
+  haskell =
+    super.haskell // {
+      packages = super.haskell.packages // {
+        "${compiler}" =
+          super.haskell.packages."${compiler}".override { inherit overrides; };
+      };
+    };
+}

nix/overlays/haskell-packages/README.md

@@ -1,10 +0,0 @@
-The `.nix` files in this directory pin specific versions of Haskell packages
-from Hackage. They were generated with `cabal2nix`, e.g.:
-
-```bash
-cabal2nix cabal://Cabal-3.0.0.0 > Cabal.nix
-
-```
-
-Those overrides will likely become obsolete as the pinned versions are adopted
-into our pinned version of nixpkgs.

nix/overlays/haskell-packages/default.nix

@@ -1,38 +0,0 @@
-{ compiler, extraOverrides ? (final: prev: { }) }:
-
-self: super:
-let
-  overrides =
-    final: prev:
-    rec {
-      protolude =
-        prev.callHackageDirect
-          {
-            pkg = "protolude";
-            ver = "0.3.0";
-            sha256 = "0iwh4wsjhb7pms88lw1afhdal9f86nrrkkvv65f9wxbd1b159n72";
-          } { };
-      # To get the sha256
-      # nix-prefetch-url --unpack https://hackage.haskell.org/package/hasql-notifications-0.1.0.0/hasql-notifications-0.1.0.0.tar.gz
-      hasql-notifications =
-        self.haskell.lib.overrideCabal
-          (
-            prev.callHackageDirect
-              {
-                pkg = "hasql-notifications";
-                ver = "0.1.0.0";
-                sha256 = "1z17gsqvvzzi0yipc3qy3jz8vzpww4vsc4vaj2kbzr2mfliq6fx3";
-              } { }
-          )
-          (old: { doCheck = false; });
-    } // extraOverrides final prev;
-in
-{
-  haskell =
-    super.haskell // {
-      packages = super.haskell.packages // {
-        "${compiler}" =
-          super.haskell.packages."${compiler}".override { inherit overrides; };
-      };
-    };
-}

nix/patches/nixpkgs-revert-ghc-bootstrap.patch

@@ -1,20 +1,20 @@
-From ba48fc5bf1db795e31778fd67a9d72f28437d6c3 Mon Sep 17 00:00:00 2001
+From f9e4b7fd7a93ab643db0b1c84003987400bbe120 Mon Sep 17 00:00:00 2001
 From: Remo <remo>
-Date: Tue, 19 May 2020 13:07:54 +0200
+Date: Sun, 23 Aug 2020 12:01:48 +0200
 Subject: [PATCH] Revert "ghc: 8.6.3-binary -> 8.6.5-binary"
 
 This reverts commit 3c7ef6bcd85e3e133ac6f2ab5e238934d56d902e.
 ---
- .../ghc/{8.6.5-binary.nix => 8.6.3-binary.nix}   | 13 ++++++-------
+ .../{8.6.5-binary.nix => 8.6.3-binary.nix}    | 13 +++++-----
- pkgs/top-level/haskell-packages.nix              | 16 ++++++++--------
+ pkgs/top-level/haskell-packages.nix           | 25 ++++++++-----------
- 2 files changed, 14 insertions(+), 15 deletions(-)
+ 2 files changed, 16 insertions(+), 22 deletions(-)
- rename pkgs/development/compilers/ghc/{8.6.5-binary.nix => 8.6.3-binary.nix} (95%)
+ rename pkgs/development/compilers/ghc/{8.6.5-binary.nix => 8.6.3-binary.nix} (94%)
 
 diff --git a/pkgs/development/compilers/ghc/8.6.5-binary.nix b/pkgs/development/compilers/ghc/8.6.3-binary.nix
-similarity index 95%
+similarity index 94%
 rename from pkgs/development/compilers/ghc/8.6.5-binary.nix
 rename to pkgs/development/compilers/ghc/8.6.3-binary.nix
-index 97793d91289..9e0fd12b96c 100644
+index 41af279e83f..5853a470efc 100644
 --- a/pkgs/development/compilers/ghc/8.6.5-binary.nix
 +++ b/pkgs/development/compilers/ghc/8.6.3-binary.nix
 @@ -27,19 +27,18 @@ let
@@ -52,28 +52,29 @@ index 97793d91289..9e0fd12b96c 100644
    }.${stdenv.hostPlatform.system}
      or (throw "cannot bootstrap GHC on this platform"));
 diff --git a/pkgs/top-level/haskell-packages.nix b/pkgs/top-level/haskell-packages.nix
-index 13c5b721301..1bce80e5774 100644
+index 3e85109527e..388210b83fb 100644
 --- a/pkgs/top-level/haskell-packages.nix
 +++ b/pkgs/top-level/haskell-packages.nix
-@@ -4,7 +4,7 @@ let
+@@ -4,7 +4,8 @@ let
    # These are attributes in compiler and packages that don't support integer-simple.
    integerSimpleExcludes = [
      "ghc822Binary"
 -    "ghc865Binary"
 +    "ghc863Binary"
-     "ghc844"
++    "ghc844"
      "ghcjs"
      "ghcjs86"
-@@ -42,7 +42,7 @@ in {
+     "integer-simple"
+@@ -47,7 +48,7 @@ in {
 
      ghc822Binary = callPackage ../development/compilers/ghc/8.2.2-binary.nix { };
 
 -    ghc865Binary = callPackage ../development/compilers/ghc/8.6.5-binary.nix { };
 +    ghc863Binary = callPackage ../development/compilers/ghc/8.6.3-binary.nix { };
 
-     ghc844 = callPackage ../development/compilers/ghc/8.4.4.nix {
+     ghc865 = callPackage ../development/compilers/ghc/8.6.5.nix {
        bootPkgs = packages.ghc822Binary;
-@@ -57,19 +57,19 @@ in {
+@@ -56,31 +57,25 @@ in {
        llvmPackages = pkgs.llvmPackages_6;
      };
      ghc882 = callPackage ../development/compilers/ghc/8.8.2.nix {
@@ -85,6 +86,13 @@ index 13c5b721301..1bce80e5774 100644
      };
      ghc883 = callPackage ../development/compilers/ghc/8.8.3.nix {
 -      bootPkgs = packages.ghc865Binary;
++      bootPkgs = packages.ghc863Binary;
+       inherit (buildPackages.python3Packages) sphinx;
+       buildLlvmPackages = buildPackages.llvmPackages_7;
+       llvmPackages = pkgs.llvmPackages_7;
+     };
+     ghc884 = callPackage ../development/compilers/ghc/8.8.4.nix {
+-      bootPkgs = packages.ghc865Binary;
 +      bootPkgs = packages.ghc863Binary;
        inherit (buildPackages.python3Packages) sphinx;
        buildLlvmPackages = buildPackages.llvmPackages_7;
@@ -92,11 +100,17 @@ index 13c5b721301..1bce80e5774 100644
      };
      ghc8101 = callPackage ../development/compilers/ghc/8.10.1.nix {
 -      bootPkgs = packages.ghc865Binary;
+-      inherit (buildPackages.python3Packages) sphinx;
+-      buildLlvmPackages = buildPackages.llvmPackages_9;
+-      llvmPackages = pkgs.llvmPackages_9;
+-    };
+-    ghc8102 = callPackage ../development/compilers/ghc/8.10.2.nix {
+-      bootPkgs = packages.ghc865Binary;
 +      bootPkgs = packages.ghc863Binary;
        inherit (buildPackages.python3Packages) sphinx;
        buildLlvmPackages = buildPackages.llvmPackages_9;
        llvmPackages = pkgs.llvmPackages_9;
-@@ -112,9 +112,9 @@ in {
+@@ -133,9 +128,9 @@ in {
        compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-8.2.x.nix { };
        packageSetConfig = bootstrapPackageSet;
      };
@@ -110,5 +124,5 @@ index 13c5b721301..1bce80e5774 100644
        packageSetConfig = bootstrapPackageSet;
      };
 --
-2.25.1
+2.28.0
 

nix/tests.nix

@@ -108,7 +108,8 @@ let
 in
 # Create an environment that contains all the utility scripts for running tests
   # that we defined above.
-buildEnv {
+buildEnv
+  {
     name =
       "postgrest-tests";