Nixify CircleCI setup (#1535)

* package ghr * add nix-based release scripts * refactor CI based on nix tests and release scripts * update stack version in circleci * update makefile * remove docker directory * add Docker Hub description * add README on docker
2020-05-25 11:27:35 -05:00
parent d4aba5cb08
commit 08186ea51c
20 changed files with 426 additions and 484 deletions
@@ -1,41 +1,9 @@
 version: 2

 jobs:
-  build-cache:
-    docker:
-      - image: circleci/buildpack-deps:trusty
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-          - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }}
-      - run:
-          name: install stack & dependencies
-          command: |
-            curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp
-            sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin
-            sudo apt-get update
-            sudo apt-get install -y libgmp-dev
-            sudo apt-get install -y --only-upgrade binutils
-            sudo apt-get install -y postgresql-client
-            stack setup
-      - run:
-          name: build src and tests dependencies
-          command: |
-            stack build --fast -j1 --only-dependencies
-            stack build --fast --test --no-run-tests --only-dependencies
-      - save_cache:
-          paths:
-            - "~/.stack"
-            - ".stack-work"
-          key: v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }}
-      - run:
-          name: build src and tests
-          command: |
-            stack build --fast -j1
-            stack build --fast --test --no-run-tests
-
-  style:
+  # Make sure that there are no outstanding linting hints and that
+  # auto-formatting does not result in any changes.
+  style-check:
    docker:
      - image: nixos/nix:2.3
    steps:
@@ -56,12 +24,14 @@ jobs:
            # `nix`, run `nix-shell --run postgrest-style`
            postgrest-style-check

-  build-test-9.4:
+  # Run tests based on stack and docker against the oldest PostgreSQL version
+  # that we support.
+  stack-test:
    docker:
      - image: circleci/buildpack-deps:trusty
        environment:
          - PGHOST=localhost
-      - image: circleci/postgres:9.4.26
+      - image: circleci/postgres:9.4
        environment:
          - POSTGRES_USER=circleci
          - POSTGRES_DB=circleci
@@ -71,176 +41,49 @@ jobs:
      - restore_cache:
          keys:
          - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }}
-      - run:
-          name: install ncat
-          command: |
-            # utility needed to test socket connection with curl <  7.40
-            sudo apt-get install nmap
      - run:
          name: install stack & dependencies
          command: |
-            curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp
-            sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin
+            curl -L https://github.com/commercialhaskell/stack/releases/download/v2.3.1/stack-2.3.1-linux-x86_64.tar.gz | tar zx -C /tmp
+            sudo mv /tmp/stack-2.3.1-linux-x86_64/stack /usr/bin
            sudo apt-get update
-            sudo apt-get install -y libgmp-dev
+            # ncat utility from nmap needed to test socket connection with curl <  7.40
+            sudo apt-get install -y libgmp-dev postgresql-client nmap
            sudo apt-get install -y --only-upgrade binutils
-            sudo apt-get install -y postgresql-client
            stack setup
      - run:
          name: build src and tests dependencies
          command: |
            stack build --fast -j1 --only-dependencies
            stack build --fast --test --no-run-tests --only-dependencies
+      - save_cache:
+          paths:
+            - "~/.stack"
+            - ".stack-work"
+          key: v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }}
      - run:
          name: build src and tests
          command: |
            stack build --fast -j1
            stack build --fast --test --no-run-tests
      - run:
-          name: run tests
+          name: run spec tests
+          command: |
+            POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) \
+              stack test
+      - run:
+          name: run io tests
          command: |
-            POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test
            test/io-tests.sh

-  build-test-9.6:
-    docker:
-      - image: circleci/buildpack-deps:trusty
-        environment:
-          - PGHOST=localhost
-      - image: circleci/postgres:9.6.17
-        environment:
-          - POSTGRES_USER=circleci
-          - POSTGRES_DB=circleci
-          - POSTGRES_HOST_AUTH_METHOD=trust
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-          - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }}
-      - run:
-          name: install stack & dependencies
-          command: |
-            curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp
-            sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin
-            sudo apt-get update
-            sudo apt-get install -y libgmp-dev
-            sudo apt-get install -y postgresql-client
-            stack setup
-      - run:
-          name: build src and tests
-          command: |
-            stack build --fast -j1
-            stack build --fast --test --no-run-tests
-      - run:
-          name: run tests
-          command: POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test postgrest:spec
-
-  build-test-10:
-    docker:
-      - image: circleci/buildpack-deps:trusty
-        environment:
-          - PGHOST=localhost
-      - image: circleci/postgres:10.12
-        environment:
-          - POSTGRES_USER=circleci
-          - POSTGRES_DB=circleci
-          - POSTGRES_HOST_AUTH_METHOD=trust
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-          - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }}
-      - run:
-          name: install stack & dependencies
-          command: |
-            curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp
-            sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin
-            sudo apt-get update
-            sudo apt-get install -y libgmp-dev
-            sudo apt-get install -y postgresql-client
-            stack setup
-      - run:
-          name: build src and tests
-          command: |
-            stack build --fast -j1
-            stack build --fast --test --no-run-tests
-      - run:
-          name: run tests
-          command: POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test postgrest:spec
-
-  build-test-11:
-    docker:
-      - image: circleci/buildpack-deps:trusty
-        environment:
-          - PGHOST=localhost
-      - image: circleci/postgres:11.7
-        environment:
-          - POSTGRES_USER=circleci
-          - POSTGRES_DB=circleci
-          - POSTGRES_HOST_AUTH_METHOD=trust
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-          - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }}
-      - run:
-          name: install stack & dependencies
-          command: |
-            curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp
-            sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin
-            sudo apt-get update
-            sudo apt-get install -y libgmp-dev
-            sudo apt-get install -y postgresql-client
-            stack setup
-      - run:
-          name: build src and tests
-          command: |
-            stack build --fast -j1
-            stack build --fast --test --no-run-tests
-      - run:
-          name: run tests
-          command: POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test postgrest:spec
-
-  build-test-12:
-    docker:
-      - image: circleci/buildpack-deps:trusty
-        environment:
-          - PGHOST=localhost
-      - image: circleci/postgres:12.2
-        environment:
-          - POSTGRES_USER=circleci
-          - POSTGRES_DB=circleci
-          - POSTGRES_HOST_AUTH_METHOD=trust
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-          - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }}
-      - run:
-          name: install stack & dependencies
-          command: |
-            curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp
-            sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin
-            sudo apt-get update
-            sudo apt-get install -y libgmp-dev
-            sudo apt-get install -y postgresql-client
-            stack setup
-      - run:
-          name: build src and tests
-          command: |
-            stack build --fast -j1
-            stack build --fast --test --no-run-tests
-      - run:
-          name: run tests
-          command: POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test postgrest:spec
-
-  build-prof-test:
+  # Run memory usage tests based on stack and docker.
+  stack-test-memory:
    docker:
      - image: circleci/buildpack-deps:trusty
        environment:
          - PGHOST=localhost
          - TERM=xterm
-      - image: circleci/postgres:9.6.17
+      - image: circleci/postgres:12
        environment:
          - POSTGRES_USER=circleci
          - POSTGRES_DB=circleci
@@ -253,11 +96,10 @@ jobs:
      - run:
          name: install stack & dependencies
          command: |
-            curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp
-            sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin
+            curl -L https://github.com/commercialhaskell/stack/releases/download/v2.3.1/stack-2.3.1-linux-x86_64.tar.gz | tar zx -C /tmp
+            sudo mv /tmp/stack-2.3.1-linux-x86_64/stack /usr/bin
            sudo apt-get update
-            sudo apt-get install -y libgmp-dev
-            sudo apt-get install -y postgresql-client
+            sudo apt-get install -y libgmp-dev postgresql-client
            stack setup
      - run:
          name: build dependencies with profiling enabled
@@ -278,72 +120,52 @@ jobs:
            test/create_test_db "postgres://circleci@localhost" postgrest_test
            test/memory-tests.sh

+  # Publish a new release. This only runs when a release is tagged (see
+  # workflow below).
  release:
    docker:
-      - image: circleci/golang:1.9
-    steps:
-      - attach_workspace:
-          at: /tmp/workspace
-      - checkout
-      - run:
-          name: add body and tars to github release
-          command: |
-            go get -u github.com/tcnksm/ghr
-            START=$(echo $CIRCLE_TAG | cut -c2-)
-            END='## \['
-            BODY=$(sed -n "1,/$START/d;/$END/q;p" CHANGELOG.md)
-            ghr -t $GITHUB_TOKEN -u $CIRCLE_PROJECT_USERNAME -r $CIRCLE_PROJECT_REPONAME -b "$BODY" --replace $CIRCLE_TAG /tmp/workspace/bin
-      - setup_remote_docker
-      - run:
-          name: publish docker image
-          command: |
-            docker build --build-arg POSTGREST_VERSION=$CIRCLE_TAG -t postgrest ./docker/
-            docker login -u $DOCKER_USER -p $DOCKER_PASS
-            docker tag postgrest postgrest/postgrest:$CIRCLE_TAG
-            docker push postgrest/postgrest:$CIRCLE_TAG
-            docker tag postgrest postgrest/postgrest:latest
-            docker push postgrest/postgrest:latest
-
-  build-nix:
-    machine: true
+      - image: nixos/nix:2.3
    steps:
      - checkout
-      - run:
-          name: Install Nix
-          command: |
-            curl -L https://nixos.org/nix/install | sh
-            echo "source $HOME/.nix-profile/etc/profile.d/nix.sh" >> $BASH_ENV
      - run:
          name: Install and use the Cachix binary cache
          command: |
            nix-env -iA cachix -f https://cachix.org/api/v1/install
            cachix use postgrest
      - run:
-          name: Build the dynamic PostgREST executable
-          command: |
-            # We build the dynamic postgrest executable first in order to fail fast if anything
-            # should be wrong with the source code.
-            nix-build -A postgrest
-      - restore_cache:
-          keys:
-            - v1-cabal-nix-dependencies-{{ checksum "postgrest.cabal" }}
+          name: Install release scripts
+          command: nix-env -f default.nix -iA release
      - run:
-          name: Run the spec tests against all supported PostgreSQL versions
+          name: Publish GitHub release
          command: |
-            # Our utility scripts use cabal to run the tests, which will pick up all the libraries that
-            # are built with Nix. So this should only ever build the PostgREST itself and be reasonably
-            # quick. We accelerate that part further by caching `~/.cabal` and `dist-newstyle`.
-            nix-shell --run postgrest-test-spec-all
-      - save_cache:
-          paths:
-            - "~/.cabal"
-            - "dist-newstyle"
-          key: v1-cabal-nix-dependencies-{{ checksum "postgrest.cabal" }}
+            export GITHUB_USERNAME="$CIRCLE_PROJECT_USERNAME"
+            export GITHUB_REPONAME="$CIRCLE_PROJECT_REPONAME"
+            postgrest-release-github
+      - setup_remote_docker
+      - run:
+          name: Publish Docker images
+          command: |
+            export DOCKER_REPO=postgrest
+            docker login -u $DOCKER_USER -p $DOCKER_PASS
+            postgrest-release-dockerhub
+            postgrest-release-dockerhubdescription
+
+  # Build everything in default.nix and push to the Cachix binary cache.
+  nix-build:
+    docker:
+      - image: nixos/nix:2.3
+    steps:
+      - checkout
+      - run:
+          name: Install and use the Cachix binary cache
+          command: |
+            nix-env -iA cachix -f https://cachix.org/api/v1/install
+            cachix use postgrest
      - run:
          name: Build all derivations from default.nix and push results to Cachix
          command: |
-            # Only push to the cache when CircleCI makes the CACHIX_SIGNING_KEY available (e.g. not for
-            # pull requests).
+            # Only push to the cache when CircleCI makes the CACHIX_SIGNING_KEY
+            # available (e.g. not for pull requests).
            if [ -n "${CACHIX_SIGNING_KEY:-""}" ]; then
              echo "Building and caching all derivations..."
              nix-build | cachix push postgrest
@@ -352,7 +174,8 @@ jobs:
              nix-build
            fi

-  build-static-bin:
+  # Run tests against all supported PostgreSQL versions.
+  nix-test:
    machine: true
    steps:
      - checkout
@@ -367,82 +190,75 @@ jobs:
            nix-env -iA cachix -f https://cachix.org/api/v1/install
            cachix use postgrest
      - run:
-          name: Build the PostgREST statically linked binary
-          command: |
-            nix-build -A postgrestStatic
+          name: Install testing scripts for all supported PostgreSQL versions
+          command: nix-env -f default.nix -iA tests
+      # Our utility scripts use cabal to run the tests, which will pick up all
+      # the libraries that are built with Nix. So this should only ever build
+      # the PostgREST itself and be reasonably quick. We accelerate that part
+      # further by caching `~/.cabal` and `dist-newstyle`.
+      - restore_cache:
+          keys:
+            - nix-test-{{ checksum "postgrest.cabal" }}-{{ checksum "nix/nixpkgs-version.nix" }}
+            - nix-test-{{ checksum "postgrest.cabal" }}
+            - nix-test
      - run:
-          name: compress binary
-          command: |
-            mkdir -p /tmp/workspace/bin
-            cd /tmp/workspace/bin
-            tar cvJf postgrest-$CIRCLE_TAG-linux-x64-static.tar.xz -C ~/project/result/bin postgrest
-      - persist_to_workspace:
-          root: /tmp/workspace
+          name: Run the spec tests against PostgreSQL 9.4
+          command: postgrest-test-spec-postgresql-9.4
+      - run:
+          name: Run the spec tests against PostgreSQL 9.5
+          command: postgrest-test-spec-postgresql-9.5
+      - run:
+          name: Run the spec tests against PostgreSQL 9.6
+          command: postgrest-test-spec-postgresql-9.6
+      - run:
+          name: Run the spec tests against PostgreSQL 10
+          command: postgrest-test-spec-postgresql-10
+      - run:
+          name: Run the spec tests against PostgreSQL 11
+          command: postgrest-test-spec-postgresql-11
+      - run:
+          name: Run the spec tests against PostgreSQL 12
+          command: postgrest-test-spec-postgresql-12
+      - save_cache:
          paths:
-            - bin/*
+            - "~/.cabal"
+            - "dist-newstyle"
+          key: nix-test-{{ checksum "postgrest.cabal" }}-{{ checksum "nix/nixpkgs-version.nix" }}-{{ epoch }}

 workflows:
  version: 2
  build-test-release:
    jobs:
-      - build-cache:
+      - style-check:
+          # Make sure that this job also runs when releases are tagged.
          filters:
            tags:
              only: /v[0-9]+(\.[0-9]+)*/
-      - style:
+      - stack-test:
          filters:
            tags:
              only: /v[0-9]+(\.[0-9]+)*/
-      - build-test-9.4:
+      - stack-test-memory:
          filters:
            tags:
              only: /v[0-9]+(\.[0-9]+)*/
+      - nix-build:
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
+      - nix-test:
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
+      - release:
          requires:
-            - build-cache
-      - build-test-9.6:
-          filters:
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*/
-          requires:
-            - build-cache
-      - build-test-10:
-          filters:
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*/
-          requires:
-            - build-cache
-      - build-test-11:
-          filters:
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*/
-          requires:
-            - build-cache
-      - build-test-12:
-          filters:
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*/
-          requires:
-            - build-cache
-      - build-prof-test:
-          filters:
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*/
-      - build-static-bin:
-          requires:
-            - build-nix
-            - build-prof-test
+            - style-check
+            - stack-test
+            - stack-test-memory
+            - nix-build
+            - nix-test
          filters:
            tags:
              only: /v[0-9]+(\.[0-9]+)*/
            branches:
              ignore: /.*/
-      - release:
-          requires:
-            - build-static-bin
-          filters:
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*/
-      - build-nix:
-          filters:
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*/