From 7af54c5813c0fdfdd2b6ba36e6d04af084b96f77 Mon Sep 17 00:00:00 2001 From: Remo Rechkemmer <59358383+monacoremo@users.noreply.github.com> Date: Thu, 10 Sep 2020 02:08:16 +0200 Subject: [PATCH] Upgrade nixpkgs to current master and add an upgrade checklist (#1579) * simplify haskell packages overrides based on new nixpkgs * update Nix gitignore * simplify Haskell packages overlay Co-authored-by: monacoremo --- nix/README.md | 5 + nix/UPGRADE.md | 91 +++++++++++++++++++ nix/docker/default.nix | 9 +- nix/nixpkgs-version.nix | 6 +- nix/overlays/default.nix | 2 +- nix/overlays/ghr/ghr.nix | 3 + nix/overlays/gitignore.nix | 4 +- nix/overlays/haskell-packages.nix | 31 +++++++ nix/overlays/haskell-packages/README.md | 10 -- nix/overlays/haskell-packages/default.nix | 38 -------- .../nixpkgs-revert-ghc-bootstrap.patch | 60 +++++++----- nix/tests.nix | 19 ++-- 12 files changed, 188 insertions(+), 90 deletions(-) create mode 100644 nix/UPGRADE.md create mode 100644 nix/overlays/haskell-packages.nix delete mode 100644 nix/overlays/haskell-packages/README.md delete mode 100644 nix/overlays/haskell-packages/default.nix diff --git a/nix/README.md b/nix/README.md index 7aafed0c8..4ff1bbf22 100644 --- a/nix/README.md +++ b/nix/README.md @@ -196,3 +196,8 @@ attribute with useful utilities that will be put on the PATH in `nix-shell`. Our overlays to the Nix package set are defined here. They allow us to tweak our `pkgs` in `default.nix` by adding new packages or overriding existing ones. + +## Upgrading dependencies + +See the [upgrading checklist](UPGRADE.md) for how to upgrade the PostgREST +dependencies. diff --git a/nix/UPGRADE.md b/nix/UPGRADE.md new file mode 100644 index 000000000..8efa2c3cc --- /dev/null +++ b/nix/UPGRADE.md @@ -0,0 +1,91 @@ +# Checklist for upgrading Nix dependencies + +The Nix dependencies of PostgREST should be updated regularly, in most cases it +should be a very simple operation. + +```bash +# Update pinned version of Nixpkgs +nix-shell --run nixpkgs-upgrade > nix/nixpkgs-version.nix + +# Verify that everything builds +nix-build +``` + +The following checklist guides you through the complete process in more detail. + +## Upgrade the pinned version of `nixpkgs` + +The pinned version of [`nixpkgs`](https://github.com/NixOS/nixpkgs) is defined +in [`nix/nixpkgs-version.nix`](nixpkgs-version.nix). The pin refers directly to +a GitHub tarball for the given revision, which is more efficient than pulling +the complete Git repository. To upgrade it to the current `master` of +`nixpkgs`, you can use a small utility script defined in +[`nix/nixpkgs-update.nix`](nixpkgs-update.nix): + +```bash +# From the root of the repository, enter nix-shell +nix-shell + +# Run the utility script to pin the latest revision in master +nixpkgs-upgrade > nix/nixpkgs-version.nix + +# Exit the nix-shell with Ctrl-d + +``` + +## Update pinned version of `static-haskell-nix` + +We pin [`static-haskell-nix`](https://github.com/nh2/static-haskell-nix) in +[`nix/static-haskell-package.nix`](static-haskell-package.nix). Upgrade the +pinned revision and the tarball hash if necessary. See +[`nix/nixpkgs-upgrade.nix`](nixpkgs-upgrade.nix) for how to get the correct +tarball hash, or just change the hash to an arbitrary value of correct length, +run `nix-build` and use the expected value from the resulting error message. + +## Review overlays + +Check whether the individual [overlays](overlays) are still required. + +## Check if patches are still required and update them as needed + +We track a number of PostgREST-specific patches in [`nix/patches`](patches). +Check whether the pull-requests/issues linked in the +[`default.nix`](patches/default.nix) have progressed and remove/modify the +patches if they did. If conflicting changes occurred, you might have to rebase +the respective patches. + +## Build everything + +Using the PostgREST binary Nix cache is recommended. Install +[Cachix](https://cachix.org/) and run `cachix use postgrest`. + +Run `nix-build` in the root directory of the project to build all PostgREST +artifacts. This might take a long time, e.g. when our static GHC version needs +to be rebuilt due to changes to some underlying package. If there are any +errors, this is probably due to one of our patches. Try to fix them and re-run +`nix-build` until everything builds. + +## Update the PostgREST binary cache + +If you have access to the PostgREST cachix signing key, you can push the +artifacts that you built locally to the binary cache. This will accelerate the +CI builds and tests, sometimes dramatically. This might sometimes even be +required to avoid build timeouts in CI. + +You'll need to set the `CACHIX_SIGNING_KEY` before proceeding, e.g. by creating +a file containing `export CACHIX_SIGNING_KEY=...` and sourcing that file, which +avoids having the secret in you shell history. + +To push all new artifacts to Cachix, run: + +``` +nix-store -qR --include-outputs $$(nix-instantiate) | cachix push postgrest + +# Or, equivalently +make cachix-push-all + +``` + +The `nix-store` command will query the nix-store to list all dependencies and +build artifacts of PostgREST. The `cachix` command will efficiently push +everything that is not yet cached to the binary cache. diff --git a/nix/docker/default.nix b/nix/docker/default.nix index e71590323..79d97176e 100644 --- a/nix/docker/default.nix +++ b/nix/docker/default.nix @@ -57,7 +57,8 @@ let docker load -i ${image} ''; in -buildEnv { - name = "postgrest-docker"; - paths = [ load ]; -} // { inherit image config; } +buildEnv + { + name = "postgrest-docker"; + paths = [ load ]; + } // { inherit image config; } diff --git a/nix/nixpkgs-version.nix b/nix/nixpkgs-version.nix index 2c3fda442..8e4c2665c 100644 --- a/nix/nixpkgs-version.nix +++ b/nix/nixpkgs-version.nix @@ -1,6 +1,6 @@ # Pinned version of Nixpkgs, generated with nixpkgs-upgrade. { - date = "2020-07-12"; - rev = "c87c474b17af792e7984ef4f058291f7ce06f594"; - tarballHash = "1171bwg07dcaqgayacaqwk3gyq97hi261gr7a4pgbrkafqb5r3ds"; + date = "2020-09-04"; + rev = "7bc3a08d3a4c700b53a3b27f5acd149f24b931ec"; + tarballHash = "1kiz37052zsgvw7a378zg08mpbi1wk8dkgm5j6dy0x4mxvcg8ws3"; } diff --git a/nix/overlays/default.nix b/nix/overlays/default.nix index 70477cf29..b00ffe74d 100644 --- a/nix/overlays/default.nix +++ b/nix/overlays/default.nix @@ -1,7 +1,7 @@ { gitignore = import ./gitignore.nix; ghr = import ./ghr; - haskell-packages = import ./haskell-packages; + haskell-packages = import ./haskell-packages.nix; postgresql-default = import ./postgresql-default.nix; postgresql-legacy = import ./postgresql-legacy.nix; } diff --git a/nix/overlays/ghr/ghr.nix b/nix/overlays/ghr/ghr.nix index 470f5ce0a..37b9e6c09 100644 --- a/nix/overlays/ghr/ghr.nix +++ b/nix/overlays/ghr/ghr.nix @@ -12,4 +12,7 @@ buildGoModule rec { }; vendorSha256 = "14avsngzhl1b8a05i43ph6sxh9vj0jls0acxr9j7r0h3f0vpamcj"; + + # Disabling tests, as they require a GitHub API token + doCheck = false; } diff --git a/nix/overlays/gitignore.nix b/nix/overlays/gitignore.nix index 868a0eeea..8be3170c4 100644 --- a/nix/overlays/gitignore.nix +++ b/nix/overlays/gitignore.nix @@ -12,8 +12,8 @@ self: super: gitignoreSrc = super.fetchFromGitHub { owner = "hercules-ci"; repo = "gitignore"; - rev = "2ced4519f865341adcb143c5d668f955a2cb997f"; - sha256 = "sha256:0fc5bgv9syfcblp23y05kkfnpgh3gssz6vn24frs8dzw39algk2z"; + rev = "c4662e662462e7bf3c2a968483478a665d00e717"; + sha256 = "1npnx0h6bd0d7ql93ka7azhj40zgjp815fw2r6smg8ch9p7mzdlx"; }; in (super.callPackage gitignoreSrc { }).gitignoreSource; diff --git a/nix/overlays/haskell-packages.nix b/nix/overlays/haskell-packages.nix new file mode 100644 index 000000000..591f44b51 --- /dev/null +++ b/nix/overlays/haskell-packages.nix @@ -0,0 +1,31 @@ +{ compiler, extraOverrides ? (final: prev: { }) }: + +self: super: +let + overrides = + final: prev: + rec { + # To pin custom versions of Haskell packages: + # protolude = + # prev.callHackageDirect + # { + # pkg = "protolude"; + # ver = "0.3.0"; + # sha256 = "0iwh4wsjhb7pms88lw1afhdal9f86nrrkkvv65f9wxbd1b159n72"; + # } + # { }; + # + # To get the sha256: + # nix-prefetch-url --unpack https://hackage.haskell.org/package/protolude-0.3.0/protolude-0.3.0.tar.gz + protolude = prev.protolude_0_3_0; + } // extraOverrides final prev; +in +{ + haskell = + super.haskell // { + packages = super.haskell.packages // { + "${compiler}" = + super.haskell.packages."${compiler}".override { inherit overrides; }; + }; + }; +} diff --git a/nix/overlays/haskell-packages/README.md b/nix/overlays/haskell-packages/README.md deleted file mode 100644 index 89058d3a5..000000000 --- a/nix/overlays/haskell-packages/README.md +++ /dev/null @@ -1,10 +0,0 @@ -The `.nix` files in this directory pin specific versions of Haskell packages -from Hackage. They were generated with `cabal2nix`, e.g.: - -```bash -cabal2nix cabal://Cabal-3.0.0.0 > Cabal.nix - -``` - -Those overrides will likely become obsolete as the pinned versions are adopted -into our pinned version of nixpkgs. diff --git a/nix/overlays/haskell-packages/default.nix b/nix/overlays/haskell-packages/default.nix deleted file mode 100644 index 8588e3e35..000000000 --- a/nix/overlays/haskell-packages/default.nix +++ /dev/null @@ -1,38 +0,0 @@ -{ compiler, extraOverrides ? (final: prev: { }) }: - -self: super: -let - overrides = - final: prev: - rec { - protolude = - prev.callHackageDirect - { - pkg = "protolude"; - ver = "0.3.0"; - sha256 = "0iwh4wsjhb7pms88lw1afhdal9f86nrrkkvv65f9wxbd1b159n72"; - } { }; - # To get the sha256 - # nix-prefetch-url --unpack https://hackage.haskell.org/package/hasql-notifications-0.1.0.0/hasql-notifications-0.1.0.0.tar.gz - hasql-notifications = - self.haskell.lib.overrideCabal - ( - prev.callHackageDirect - { - pkg = "hasql-notifications"; - ver = "0.1.0.0"; - sha256 = "1z17gsqvvzzi0yipc3qy3jz8vzpww4vsc4vaj2kbzr2mfliq6fx3"; - } { } - ) - (old: { doCheck = false; }); - } // extraOverrides final prev; -in -{ - haskell = - super.haskell // { - packages = super.haskell.packages // { - "${compiler}" = - super.haskell.packages."${compiler}".override { inherit overrides; }; - }; - }; -} diff --git a/nix/patches/nixpkgs-revert-ghc-bootstrap.patch b/nix/patches/nixpkgs-revert-ghc-bootstrap.patch index 89ac41d27..415cfadef 100644 --- a/nix/patches/nixpkgs-revert-ghc-bootstrap.patch +++ b/nix/patches/nixpkgs-revert-ghc-bootstrap.patch @@ -1,31 +1,31 @@ -From ba48fc5bf1db795e31778fd67a9d72f28437d6c3 Mon Sep 17 00:00:00 2001 +From f9e4b7fd7a93ab643db0b1c84003987400bbe120 Mon Sep 17 00:00:00 2001 From: Remo -Date: Tue, 19 May 2020 13:07:54 +0200 +Date: Sun, 23 Aug 2020 12:01:48 +0200 Subject: [PATCH] Revert "ghc: 8.6.3-binary -> 8.6.5-binary" This reverts commit 3c7ef6bcd85e3e133ac6f2ab5e238934d56d902e. --- - .../ghc/{8.6.5-binary.nix => 8.6.3-binary.nix} | 13 ++++++------- - pkgs/top-level/haskell-packages.nix | 16 ++++++++-------- - 2 files changed, 14 insertions(+), 15 deletions(-) - rename pkgs/development/compilers/ghc/{8.6.5-binary.nix => 8.6.3-binary.nix} (95%) + .../{8.6.5-binary.nix => 8.6.3-binary.nix} | 13 +++++----- + pkgs/top-level/haskell-packages.nix | 25 ++++++++----------- + 2 files changed, 16 insertions(+), 22 deletions(-) + rename pkgs/development/compilers/ghc/{8.6.5-binary.nix => 8.6.3-binary.nix} (94%) diff --git a/pkgs/development/compilers/ghc/8.6.5-binary.nix b/pkgs/development/compilers/ghc/8.6.3-binary.nix -similarity index 95% +similarity index 94% rename from pkgs/development/compilers/ghc/8.6.5-binary.nix rename to pkgs/development/compilers/ghc/8.6.3-binary.nix -index 97793d91289..9e0fd12b96c 100644 +index 41af279e83f..5853a470efc 100644 --- a/pkgs/development/compilers/ghc/8.6.5-binary.nix +++ b/pkgs/development/compilers/ghc/8.6.3-binary.nix @@ -27,19 +27,18 @@ let in - + stdenv.mkDerivation rec { - version = "8.6.5"; + version = "8.6.3"; - + name = "ghc-${version}-binary"; - + - # https://downloads.haskell.org/~ghc/8.6.5/ src = fetchurl ({ i686-linux = { @@ -52,28 +52,29 @@ index 97793d91289..9e0fd12b96c 100644 }.${stdenv.hostPlatform.system} or (throw "cannot bootstrap GHC on this platform")); diff --git a/pkgs/top-level/haskell-packages.nix b/pkgs/top-level/haskell-packages.nix -index 13c5b721301..1bce80e5774 100644 +index 3e85109527e..388210b83fb 100644 --- a/pkgs/top-level/haskell-packages.nix +++ b/pkgs/top-level/haskell-packages.nix -@@ -4,7 +4,7 @@ let +@@ -4,7 +4,8 @@ let # These are attributes in compiler and packages that don't support integer-simple. integerSimpleExcludes = [ "ghc822Binary" - "ghc865Binary" + "ghc863Binary" - "ghc844" ++ "ghc844" "ghcjs" "ghcjs86" -@@ -42,7 +42,7 @@ in { - + "integer-simple" +@@ -47,7 +48,7 @@ in { + ghc822Binary = callPackage ../development/compilers/ghc/8.2.2-binary.nix { }; - + - ghc865Binary = callPackage ../development/compilers/ghc/8.6.5-binary.nix { }; + ghc863Binary = callPackage ../development/compilers/ghc/8.6.3-binary.nix { }; - - ghc844 = callPackage ../development/compilers/ghc/8.4.4.nix { + + ghc865 = callPackage ../development/compilers/ghc/8.6.5.nix { bootPkgs = packages.ghc822Binary; -@@ -57,19 +57,19 @@ in { +@@ -56,31 +57,25 @@ in { llvmPackages = pkgs.llvmPackages_6; }; ghc882 = callPackage ../development/compilers/ghc/8.8.2.nix { @@ -85,6 +86,13 @@ index 13c5b721301..1bce80e5774 100644 }; ghc883 = callPackage ../development/compilers/ghc/8.8.3.nix { - bootPkgs = packages.ghc865Binary; ++ bootPkgs = packages.ghc863Binary; + inherit (buildPackages.python3Packages) sphinx; + buildLlvmPackages = buildPackages.llvmPackages_7; + llvmPackages = pkgs.llvmPackages_7; + }; + ghc884 = callPackage ../development/compilers/ghc/8.8.4.nix { +- bootPkgs = packages.ghc865Binary; + bootPkgs = packages.ghc863Binary; inherit (buildPackages.python3Packages) sphinx; buildLlvmPackages = buildPackages.llvmPackages_7; @@ -92,11 +100,17 @@ index 13c5b721301..1bce80e5774 100644 }; ghc8101 = callPackage ../development/compilers/ghc/8.10.1.nix { - bootPkgs = packages.ghc865Binary; +- inherit (buildPackages.python3Packages) sphinx; +- buildLlvmPackages = buildPackages.llvmPackages_9; +- llvmPackages = pkgs.llvmPackages_9; +- }; +- ghc8102 = callPackage ../development/compilers/ghc/8.10.2.nix { +- bootPkgs = packages.ghc865Binary; + bootPkgs = packages.ghc863Binary; inherit (buildPackages.python3Packages) sphinx; buildLlvmPackages = buildPackages.llvmPackages_9; llvmPackages = pkgs.llvmPackages_9; -@@ -112,9 +112,9 @@ in { +@@ -133,9 +128,9 @@ in { compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-8.2.x.nix { }; packageSetConfig = bootstrapPackageSet; }; @@ -109,6 +123,6 @@ index 13c5b721301..1bce80e5774 100644 compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-8.6.x.nix { }; packageSetConfig = bootstrapPackageSet; }; --- -2.25.1 +-- +2.28.0 diff --git a/nix/tests.nix b/nix/tests.nix index 0fa263415..ee429494e 100644 --- a/nix/tests.nix +++ b/nix/tests.nix @@ -108,16 +108,17 @@ let in # Create an environment that contains all the utility scripts for running tests # that we defined above. -buildEnv { - name = - "postgrest-tests"; +buildEnv + { + name = + "postgrest-tests"; - paths = - [ - (testSpec "postgrest-test-spec" postgresql) - testSpecAllVersions - ] ++ testSpecVersions; -} + paths = + [ + (testSpec "postgrest-test-spec" postgresql) + testSpecAllVersions + ] ++ testSpecVersions; + } # The IO an memory tests have large dependencies (a static and a profiled # build of PostgREST respectively) and are run less often than the spec # tests, so we don't include them in the default test environment. We make