diff --git a/.circleci/config.yml b/.circleci/config.yml index ad57b59ec..830bf9c27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,41 +1,9 @@ version: 2 jobs: - build-cache: - docker: - - image: circleci/buildpack-deps:trusty - steps: - - checkout - - restore_cache: - keys: - - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }} - - run: - name: install stack & dependencies - command: | - curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp - sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin - sudo apt-get update - sudo apt-get install -y libgmp-dev - sudo apt-get install -y --only-upgrade binutils - sudo apt-get install -y postgresql-client - stack setup - - run: - name: build src and tests dependencies - command: | - stack build --fast -j1 --only-dependencies - stack build --fast --test --no-run-tests --only-dependencies - - save_cache: - paths: - - "~/.stack" - - ".stack-work" - key: v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }} - - run: - name: build src and tests - command: | - stack build --fast -j1 - stack build --fast --test --no-run-tests - - style: + # Make sure that there are no outstanding linting hints and that + # auto-formatting does not result in any changes. + style-check: docker: - image: nixos/nix:2.3 steps: @@ -56,12 +24,14 @@ jobs: # `nix`, run `nix-shell --run postgrest-style` postgrest-style-check - build-test-9.4: + # Run tests based on stack and docker against the oldest PostgreSQL version + # that we support. + stack-test: docker: - image: circleci/buildpack-deps:trusty environment: - PGHOST=localhost - - image: circleci/postgres:9.4.26 + - image: circleci/postgres:9.4 environment: - POSTGRES_USER=circleci - POSTGRES_DB=circleci @@ -71,176 +41,49 @@ jobs: - restore_cache: keys: - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }} - - run: - name: install ncat - command: | - # utility needed to test socket connection with curl < 7.40 - sudo apt-get install nmap - run: name: install stack & dependencies command: | - curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp - sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin + curl -L https://github.com/commercialhaskell/stack/releases/download/v2.3.1/stack-2.3.1-linux-x86_64.tar.gz | tar zx -C /tmp + sudo mv /tmp/stack-2.3.1-linux-x86_64/stack /usr/bin sudo apt-get update - sudo apt-get install -y libgmp-dev + # ncat utility from nmap needed to test socket connection with curl < 7.40 + sudo apt-get install -y libgmp-dev postgresql-client nmap sudo apt-get install -y --only-upgrade binutils - sudo apt-get install -y postgresql-client stack setup - run: name: build src and tests dependencies command: | stack build --fast -j1 --only-dependencies stack build --fast --test --no-run-tests --only-dependencies + - save_cache: + paths: + - "~/.stack" + - ".stack-work" + key: v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }} - run: name: build src and tests command: | stack build --fast -j1 stack build --fast --test --no-run-tests - run: - name: run tests + name: run spec tests + command: | + POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) \ + stack test + - run: + name: run io tests command: | - POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test test/io-tests.sh - build-test-9.6: - docker: - - image: circleci/buildpack-deps:trusty - environment: - - PGHOST=localhost - - image: circleci/postgres:9.6.17 - environment: - - POSTGRES_USER=circleci - - POSTGRES_DB=circleci - - POSTGRES_HOST_AUTH_METHOD=trust - steps: - - checkout - - restore_cache: - keys: - - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }} - - run: - name: install stack & dependencies - command: | - curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp - sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin - sudo apt-get update - sudo apt-get install -y libgmp-dev - sudo apt-get install -y postgresql-client - stack setup - - run: - name: build src and tests - command: | - stack build --fast -j1 - stack build --fast --test --no-run-tests - - run: - name: run tests - command: POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test postgrest:spec - - build-test-10: - docker: - - image: circleci/buildpack-deps:trusty - environment: - - PGHOST=localhost - - image: circleci/postgres:10.12 - environment: - - POSTGRES_USER=circleci - - POSTGRES_DB=circleci - - POSTGRES_HOST_AUTH_METHOD=trust - steps: - - checkout - - restore_cache: - keys: - - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }} - - run: - name: install stack & dependencies - command: | - curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp - sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin - sudo apt-get update - sudo apt-get install -y libgmp-dev - sudo apt-get install -y postgresql-client - stack setup - - run: - name: build src and tests - command: | - stack build --fast -j1 - stack build --fast --test --no-run-tests - - run: - name: run tests - command: POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test postgrest:spec - - build-test-11: - docker: - - image: circleci/buildpack-deps:trusty - environment: - - PGHOST=localhost - - image: circleci/postgres:11.7 - environment: - - POSTGRES_USER=circleci - - POSTGRES_DB=circleci - - POSTGRES_HOST_AUTH_METHOD=trust - steps: - - checkout - - restore_cache: - keys: - - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }} - - run: - name: install stack & dependencies - command: | - curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp - sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin - sudo apt-get update - sudo apt-get install -y libgmp-dev - sudo apt-get install -y postgresql-client - stack setup - - run: - name: build src and tests - command: | - stack build --fast -j1 - stack build --fast --test --no-run-tests - - run: - name: run tests - command: POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test postgrest:spec - - build-test-12: - docker: - - image: circleci/buildpack-deps:trusty - environment: - - PGHOST=localhost - - image: circleci/postgres:12.2 - environment: - - POSTGRES_USER=circleci - - POSTGRES_DB=circleci - - POSTGRES_HOST_AUTH_METHOD=trust - steps: - - checkout - - restore_cache: - keys: - - v1-stack-dependencies-{{ checksum "postgrest.cabal" }}-{{ checksum "stack.yaml" }} - - run: - name: install stack & dependencies - command: | - curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp - sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin - sudo apt-get update - sudo apt-get install -y libgmp-dev - sudo apt-get install -y postgresql-client - stack setup - - run: - name: build src and tests - command: | - stack build --fast -j1 - stack build --fast --test --no-run-tests - - run: - name: run tests - command: POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test postgrest:spec - - build-prof-test: + # Run memory usage tests based on stack and docker. + stack-test-memory: docker: - image: circleci/buildpack-deps:trusty environment: - PGHOST=localhost - TERM=xterm - - image: circleci/postgres:9.6.17 + - image: circleci/postgres:12 environment: - POSTGRES_USER=circleci - POSTGRES_DB=circleci @@ -253,11 +96,10 @@ jobs: - run: name: install stack & dependencies command: | - curl -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz | tar zx -C /tmp - sudo mv /tmp/stack-2.1.3-linux-x86_64/stack /usr/bin + curl -L https://github.com/commercialhaskell/stack/releases/download/v2.3.1/stack-2.3.1-linux-x86_64.tar.gz | tar zx -C /tmp + sudo mv /tmp/stack-2.3.1-linux-x86_64/stack /usr/bin sudo apt-get update - sudo apt-get install -y libgmp-dev - sudo apt-get install -y postgresql-client + sudo apt-get install -y libgmp-dev postgresql-client stack setup - run: name: build dependencies with profiling enabled @@ -278,72 +120,52 @@ jobs: test/create_test_db "postgres://circleci@localhost" postgrest_test test/memory-tests.sh + # Publish a new release. This only runs when a release is tagged (see + # workflow below). release: docker: - - image: circleci/golang:1.9 - steps: - - attach_workspace: - at: /tmp/workspace - - checkout - - run: - name: add body and tars to github release - command: | - go get -u github.com/tcnksm/ghr - START=$(echo $CIRCLE_TAG | cut -c2-) - END='## \[' - BODY=$(sed -n "1,/$START/d;/$END/q;p" CHANGELOG.md) - ghr -t $GITHUB_TOKEN -u $CIRCLE_PROJECT_USERNAME -r $CIRCLE_PROJECT_REPONAME -b "$BODY" --replace $CIRCLE_TAG /tmp/workspace/bin - - setup_remote_docker - - run: - name: publish docker image - command: | - docker build --build-arg POSTGREST_VERSION=$CIRCLE_TAG -t postgrest ./docker/ - docker login -u $DOCKER_USER -p $DOCKER_PASS - docker tag postgrest postgrest/postgrest:$CIRCLE_TAG - docker push postgrest/postgrest:$CIRCLE_TAG - docker tag postgrest postgrest/postgrest:latest - docker push postgrest/postgrest:latest - - build-nix: - machine: true + - image: nixos/nix:2.3 steps: - checkout - - run: - name: Install Nix - command: | - curl -L https://nixos.org/nix/install | sh - echo "source $HOME/.nix-profile/etc/profile.d/nix.sh" >> $BASH_ENV - run: name: Install and use the Cachix binary cache command: | nix-env -iA cachix -f https://cachix.org/api/v1/install cachix use postgrest - run: - name: Build the dynamic PostgREST executable - command: | - # We build the dynamic postgrest executable first in order to fail fast if anything - # should be wrong with the source code. - nix-build -A postgrest - - restore_cache: - keys: - - v1-cabal-nix-dependencies-{{ checksum "postgrest.cabal" }} + name: Install release scripts + command: nix-env -f default.nix -iA release - run: - name: Run the spec tests against all supported PostgreSQL versions + name: Publish GitHub release command: | - # Our utility scripts use cabal to run the tests, which will pick up all the libraries that - # are built with Nix. So this should only ever build the PostgREST itself and be reasonably - # quick. We accelerate that part further by caching `~/.cabal` and `dist-newstyle`. - nix-shell --run postgrest-test-spec-all - - save_cache: - paths: - - "~/.cabal" - - "dist-newstyle" - key: v1-cabal-nix-dependencies-{{ checksum "postgrest.cabal" }} + export GITHUB_USERNAME="$CIRCLE_PROJECT_USERNAME" + export GITHUB_REPONAME="$CIRCLE_PROJECT_REPONAME" + postgrest-release-github + - setup_remote_docker + - run: + name: Publish Docker images + command: | + export DOCKER_REPO=postgrest + docker login -u $DOCKER_USER -p $DOCKER_PASS + postgrest-release-dockerhub + postgrest-release-dockerhubdescription + + # Build everything in default.nix and push to the Cachix binary cache. + nix-build: + docker: + - image: nixos/nix:2.3 + steps: + - checkout + - run: + name: Install and use the Cachix binary cache + command: | + nix-env -iA cachix -f https://cachix.org/api/v1/install + cachix use postgrest - run: name: Build all derivations from default.nix and push results to Cachix command: | - # Only push to the cache when CircleCI makes the CACHIX_SIGNING_KEY available (e.g. not for - # pull requests). + # Only push to the cache when CircleCI makes the CACHIX_SIGNING_KEY + # available (e.g. not for pull requests). if [ -n "${CACHIX_SIGNING_KEY:-""}" ]; then echo "Building and caching all derivations..." nix-build | cachix push postgrest @@ -352,7 +174,8 @@ jobs: nix-build fi - build-static-bin: + # Run tests against all supported PostgreSQL versions. + nix-test: machine: true steps: - checkout @@ -367,82 +190,75 @@ jobs: nix-env -iA cachix -f https://cachix.org/api/v1/install cachix use postgrest - run: - name: Build the PostgREST statically linked binary - command: | - nix-build -A postgrestStatic + name: Install testing scripts for all supported PostgreSQL versions + command: nix-env -f default.nix -iA tests + # Our utility scripts use cabal to run the tests, which will pick up all + # the libraries that are built with Nix. So this should only ever build + # the PostgREST itself and be reasonably quick. We accelerate that part + # further by caching `~/.cabal` and `dist-newstyle`. + - restore_cache: + keys: + - nix-test-{{ checksum "postgrest.cabal" }}-{{ checksum "nix/nixpkgs-version.nix" }} + - nix-test-{{ checksum "postgrest.cabal" }} + - nix-test - run: - name: compress binary - command: | - mkdir -p /tmp/workspace/bin - cd /tmp/workspace/bin - tar cvJf postgrest-$CIRCLE_TAG-linux-x64-static.tar.xz -C ~/project/result/bin postgrest - - persist_to_workspace: - root: /tmp/workspace + name: Run the spec tests against PostgreSQL 9.4 + command: postgrest-test-spec-postgresql-9.4 + - run: + name: Run the spec tests against PostgreSQL 9.5 + command: postgrest-test-spec-postgresql-9.5 + - run: + name: Run the spec tests against PostgreSQL 9.6 + command: postgrest-test-spec-postgresql-9.6 + - run: + name: Run the spec tests against PostgreSQL 10 + command: postgrest-test-spec-postgresql-10 + - run: + name: Run the spec tests against PostgreSQL 11 + command: postgrest-test-spec-postgresql-11 + - run: + name: Run the spec tests against PostgreSQL 12 + command: postgrest-test-spec-postgresql-12 + - save_cache: paths: - - bin/* + - "~/.cabal" + - "dist-newstyle" + key: nix-test-{{ checksum "postgrest.cabal" }}-{{ checksum "nix/nixpkgs-version.nix" }}-{{ epoch }} workflows: version: 2 build-test-release: jobs: - - build-cache: + - style-check: + # Make sure that this job also runs when releases are tagged. filters: tags: only: /v[0-9]+(\.[0-9]+)*/ - - style: + - stack-test: filters: tags: only: /v[0-9]+(\.[0-9]+)*/ - - build-test-9.4: + - stack-test-memory: filters: tags: only: /v[0-9]+(\.[0-9]+)*/ + - nix-build: + filters: + tags: + only: /v[0-9]+(\.[0-9]+)*/ + - nix-test: + filters: + tags: + only: /v[0-9]+(\.[0-9]+)*/ + - release: requires: - - build-cache - - build-test-9.6: - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - requires: - - build-cache - - build-test-10: - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - requires: - - build-cache - - build-test-11: - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - requires: - - build-cache - - build-test-12: - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - requires: - - build-cache - - build-prof-test: - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - - build-static-bin: - requires: - - build-nix - - build-prof-test + - style-check + - stack-test + - stack-test-memory + - nix-build + - nix-test filters: tags: only: /v[0-9]+(\.[0-9]+)*/ branches: ignore: /.*/ - - release: - requires: - - build-static-bin - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - - build-nix: - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f8cc37f90..69e6a9608 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ This project adheres to [Semantic Versioning](http://semver.org/). - #1530, Fix how the PostgREST version is shown in the help text when the `.git` directory is not available - @monacoremo +### Changed + + - #1522, #1528, #1535, Docker images are now built from scratch based on a the static PostgREST executable (#1494) and with Nix instead of a `Dockerfile`. This reduces the compressed image size from over 30mb to about 4mb - @monacoremo + ## [7.0.1] - 2020-05-18 ### Fixed diff --git a/Makefile b/Makefile index 2ef0d2237..29b75dad5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: commit-check check clean lint style test test-watch coverage circleci circleci-prof-test check-dburi prompt-clean prompt-long-process +.PHONY: commit-check check clean lint style test test-watch coverage circleci circleci-memory prompt-clean prompt-long-process cachix-push-all commit-check: lint style @@ -10,32 +10,34 @@ clean: prompt-clean # For running these you'll need to install hlint and stylish-haskell first. Run: # stack install hlint stylish-haskell lint: - git ls-files | grep '\.l\?hs$$' | xargs stack exec -- hlint -X QuasiQuotes -X NoPatternSynonyms "$$@" + git ls-files | grep '\.l\?hs$$' | \ + xargs stack exec -- hlint -X QuasiQuotes -X NoPatternSynonyms "$$@" style: - git ls-files | grep '\.l\?hs$$' | xargs stack exec -- stylish-haskell -i && git diff-index --exit-code HEAD -- '*.hs' '*.lhs' + git ls-files | grep '\.l\?hs$$' | xargs stack exec -- stylish-haskell -i -test: check-dburi - stack test +test: + test/with_tmp_db stack test -test-watch: check-dburi - stack build --file-watch --test --test-arguments '--rerun --failure-report=.TESTREPORT --rerun-all-on-success' +test-watch: + test/with_tmp_db stack build --file-watch --test \ + --test-arguments '--rerun --failure-report=.TESTREPORT --rerun-all-on-success' -coverage: check-dburi clean +coverage: clean stack build --coverage - stack test --coverage + test/with_tmp_db stack test --coverage circleci: prompt-long-process - circleci local execute --job build-test-9.4 + circleci local execute --job stack-test -circleci-prof-test: prompt-long-process - circleci local execute --job build-prof-test - -check-dburi: - test -n "$(POSTGREST_TEST_CONNECTION)" # Requires POSTGREST_TEST_CONNECTION environmental variable +circleci-memory: prompt-long-process + circleci local execute --job stack-test-memory prompt-clean: @echo -n 'Are you sure? You will have to rebuild. [y/N] ' && read ans && [ $${ans:-N} = y ] prompt-long-process: @echo -n 'Are you sure? This might take a while. [y/N] ' && read ans && [ $${ans:-N} = y ] + +cachix-push-all: + nix-store -qR --include-outputs $$(nix-instantiate) | cachix push postgrest diff --git a/default.nix b/default.nix index 770920563..9db461eca 100644 --- a/default.nix +++ b/default.nix @@ -34,6 +34,7 @@ let allOverlays.postgresql-default allOverlays.postgresql-legacy allOverlays.gitignore + allOverlays.ghr (allOverlays.haskell-packages { inherit compiler; }) ]; @@ -94,12 +95,6 @@ rec { docker = pkgs.callPackage nix/docker { postgrest = postgrestStatic; }; - # Script `postgrest-docker-load` that loads the images built with Nix into - # Docker, using `docker load -i` under the hood. Referenced here from the top - # level so that it gets built by default. - dockerLoad = - docker.load; - # Environment in which PostgREST can be built with cabal, useful e.g. for # defining a shell for `nix-shell`. env = @@ -119,4 +114,11 @@ rec { # Development tools, including linting and styling scripts. devtools = pkgs.callPackage nix/devtools.nix { }; + + # Scripts for publishing new releases. + release = + pkgs.callPackage nix/release { + inherit docker; + postgrest = postgrestStatic; + }; } diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index b2d6d0611..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -# To build use: -# docker build --build-arg POSTGREST_VERSION= -t postgrest ./docker/ -# docker build --build-arg POSTGREST_VERSION=v7.0.1 -t postgrest ./docker/ - -FROM debian:buster-slim - -ARG POSTGREST_VERSION - -# Install postgrest -RUN BUILD_DEPS="curl ca-certificates xz-utils" && \ - apt-get -qq update && \ - apt-get -qq install -y --no-install-recommends $BUILD_DEPS && \ - cd /tmp && \ - curl -SLO https://github.com/PostgREST/postgrest/releases/download/${POSTGREST_VERSION}/postgrest-${POSTGREST_VERSION}-linux-x64-static.tar.xz && \ - tar -xJvf postgrest-${POSTGREST_VERSION}-linux-x64-static.tar.xz && \ - mv postgrest /usr/local/bin/postgrest && \ - cd / && \ - apt-get -qq purge --auto-remove -y $BUILD_DEPS && \ - apt-get -qq clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -COPY postgrest.conf /etc/postgrest.conf - -ENV PGRST_DB_URI= \ - PGRST_DB_SCHEMA=public \ - PGRST_DB_ANON_ROLE= \ - PGRST_DB_POOL=100 \ - PGRST_DB_EXTRA_SEARCH_PATH=public \ - PGRST_SERVER_HOST=*4 \ - PGRST_SERVER_PORT=3000 \ - PGRST_OPENAPI_SERVER_PROXY_URI= \ - PGRST_JWT_SECRET= \ - PGRST_SECRET_IS_BASE64=false \ - PGRST_JWT_AUD= \ - PGRST_MAX_ROWS= \ - PGRST_PRE_REQUEST= \ - PGRST_ROLE_CLAIM_KEY=".role" \ - PGRST_ROOT_SPEC= \ - PGRST_RAW_MEDIA_TYPES= - -RUN groupadd -g 1000 postgrest && \ - useradd -r -u 1000 -g postgrest postgrest && \ - chown postgrest:postgrest /etc/postgrest.conf - -USER 1000 - -# PostgREST reads /etc/postgrest.conf so map the configuration -# file in when you run this container -CMD exec postgrest /etc/postgrest.conf - -EXPOSE 3000 diff --git a/docker/distro_release/Dockerfile.centos7 b/docker/distro_release/Dockerfile.centos7 deleted file mode 100644 index fb38f3754..000000000 --- a/docker/distro_release/Dockerfile.centos7 +++ /dev/null @@ -1,18 +0,0 @@ -FROM centos:centos7 - -RUN yum -y update -RUN yum -y install perl make automake gcc gmp-devel libffi zlib zlib-devel xz tar -RUN yum -y install yum install https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-7-x86_64/pgdg-centos10-10-2.noarch.rpm -RUN yum -y install postgresql10-devel -RUN yum clean all -RUN curl -sSL https://get.haskellstack.org/ | sh - -ENV PATH $PATH:/usr/pgsql-10/bin - -# To disable warning when building -ENV PATH $PATH:/root/.local/bin - -RUN mkdir /source -WORKDIR /source - -ENTRYPOINT ["stack"] diff --git a/docker/distro_release/Dockerfile.ubuntu b/docker/distro_release/Dockerfile.ubuntu deleted file mode 100644 index 6a07aef5a..000000000 --- a/docker/distro_release/Dockerfile.ubuntu +++ /dev/null @@ -1,20 +0,0 @@ -FROM ubuntu:16.04 - -## TODO pin the stack version -# -RUN BUILD_DEPS="curl ca-certificates build-essential" && \ - apt-get -qq update && \ - apt-get -qqy --no-install-recommends install \ - $BUILD_DEPS \ - libpq-dev && \ - curl -sSL https://get.haskellstack.org/ | sh && \ - apt-get -qq clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# To disable warning when building -ENV PATH $PATH:/root/.local/bin - -RUN mkdir /source -WORKDIR /source - -ENTRYPOINT ["stack"] diff --git a/docker/distro_release/Dockerfile.ubuntui386 b/docker/distro_release/Dockerfile.ubuntui386 deleted file mode 100644 index 4b601e3fa..000000000 --- a/docker/distro_release/Dockerfile.ubuntui386 +++ /dev/null @@ -1,20 +0,0 @@ -FROM i386/ubuntu:16.04 - -## TODO pin the stack version - -RUN BUILD_DEPS="curl ca-certificates build-essential" && \ - apt-get -qq update && \ - apt-get -qqy --no-install-recommends install \ - $BUILD_DEPS \ - libpq-dev && \ - curl -sSL https://get.haskellstack.org/ | sh && \ - apt-get -qq clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# To disable warning when building -ENV PATH $PATH:/root/.local/bin - -RUN mkdir /source -WORKDIR /source - -ENTRYPOINT ["stack"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 46b5b9d55..000000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -stgrest: - image: pg_local - ports: - - "3000:3000" - links: - - postgres:postgres - environment: - PGRST_DB_URI: postgres://app_user:password@postgres:5432/app_db - PGRST_DB_SCHEMA: public - PGRST_DB_ANON_ROLE: app_user - -postgres: - image: postgres - ports: - - "5432:5432" - environment: - POSTGRES_DB: app_db - POSTGRES_USER: app_user - POSTGRES_PASSWORD: password diff --git a/docker/postgrest.conf b/docker/postgrest.conf deleted file mode 100644 index c1b5b9bd7..000000000 --- a/docker/postgrest.conf +++ /dev/null @@ -1,19 +0,0 @@ -db-uri = "$(PGRST_DB_URI)" -db-schema = "$(PGRST_DB_SCHEMA)" -db-anon-role = "$(PGRST_DB_ANON_ROLE)" -db-pool = "$(PGRST_DB_POOL)" -db-extra-search-path = "$(PGRST_DB_EXTRA_SEARCH_PATH)" - -server-host = "$(PGRST_SERVER_HOST)" -server-port = "$(PGRST_SERVER_PORT)" - -openapi-server-proxy-uri = "$(PGRST_OPENAPI_SERVER_PROXY_URI)" -jwt-secret = "$(PGRST_JWT_SECRET)" -secret-is-base64 = "$(PGRST_SECRET_IS_BASE64)" -jwt-aud = "$(PGRST_JWT_AUD)" -role-claim-key = "$(PGRST_ROLE_CLAIM_KEY)" - -max-rows = "$(PGRST_MAX_ROWS)" -pre-request = "$(PGRST_PRE_REQUEST)" -root-spec = "$(PGRST_ROOT_SPEC)" -raw-media-types = "$(PGRST_RAW_MEDIA_TYPES)" diff --git a/nix/docker/README.md b/nix/docker/README.md new file mode 100644 index 000000000..7a1d48276 --- /dev/null +++ b/nix/docker/README.md @@ -0,0 +1,101 @@ +# Docker image built with Nix + +In order to build an optimal PostgREST Docker image, we create the image from +scratch (i.e., without a parent image like `debian` or `alpine`), and only +include the files that are essential for running PostgREST (the static +PostgREST binary and a `postgrest.conf`). + +This is similar to what you would get with the following `Dockerfile`: + +```Dockerfile +# `scratch` is a minimal, reserved image in Docker, see +# https://docs.docker.com/develop/develop-images/baseimages/ . It essentially +# means "don't use a parent image and start with an empty one". +FROM scratch + +# The static PostgREST executable has no runtime dependencies, so it's all we +# need to include for running the application. +ADD /absolute/path/to/postgrest /bin/postgrest + +# Include a default configuration file. +ADD /absolute/path/to/postgrest.conf /etc/postgrest.conf + +ENV PGRST_DB_URI= \ + ... + +EXPOSE 3000 + +# This is the user id that Docker will run our image under by default. Note +# that we don't actually add the user to `/etc/passwd` or `/etc/shadow`. This +# means that tools like whoami would not work properly, but we don't include +# those in the image anyway. Not adding the user has the benefit that the image +# can be run under any user you specify. +USER 1000 + +CMD [ "/bin/postgrest", "/etc/postgrest.conf" ] +``` + +# Building the Docker image with Nix + +As we are building the static PostgREST executable with Nix and that's the main +input to the Docker file, we can also create the Docker image directly with Nix +using the [`dockerTools` +utilities](https://nixos.org/nixpkgs/manual/#sec-pkgs-dockerTools). Those +utilities don't actually use `Dockerfiles` or Docker to build Docker images, +but create them directly by putting together the required `json` and `tar` +files that make up an image. This is more efficient, does not rely on Docker or +root permissions and results in fully reproducible builds. See +[`nix/docker/default.nix`](./default.nix) for details how the image is built. + +# Building and loading the image + +The Nix expression provides a helper script `postgrest-docker-load` that loads +the optimized image into your local Docker instance (using `docker load -i +` under the hood). You can use it by running: + +``` +# Running from the root directory of the repository: + +# Build the `docker` attribute from `default.nix`, the result will be symlinked +# to `result`: +nix-build -A docker + +# Run the loading script: +result/bin/postgrest-docker-load +``` + +The Docker image built with Nix always has the name "postgrest:latest" when +loaded. + +# Inspecting the optimized image + +The image does not come with the usual utilities like `bash` and `ls`. + +You can, however, explore the `tar` file of the image by saving it with `docker +save postgrest:latest > image.tar`. + +[Dive](https://github.com/wagoodman/dive) is also useful for looking at the +contents of the image: + +``` +┃ ● Layers ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ │ Current Layer Contents ├──────────────────────────────────────────────────────────────────────────────── +Cmp Size Command Permission UID:GID Size Filetree + 14 MB FROM 20ee65c811575d2 dr-xr-xr-x 0:0 14 MB ├── bin + -r-xr-xr-x 0:0 14 MB │ └── postgrest +│ Layer Details ├───────────────────────────────────────────────────────────────────────────────────────── drwxr-xr-x 0:0 783 B ├── etc + -r--r--r-- 0:0 783 B │ └── postgrest.conf +Tags: (unavailable) dr-xr-xr-x 0:0 23 kB └── nix +Id: 20ee65c811575d206eb673e1887e7f7e6b7ccde902a63ccb924c5faa50b32cee dr-xr-xr-x 0:0 23 kB └── store +Digest: sha256:ece77302b83fd38fb54395dabc10c2eba06fc1d1933801d36cc2c4732d9c8f38 dr-xr-xr-x 0:0 23 kB └── s440jbrn94wmpzy7f8yfsp6jr2shllw5-openssl-1.1.1g-etc +Command: dr-xr-xr-x 0:0 23 kB └── etc + dr-xr-xr-x 0:0 23 kB └── ssl + -r--r--r-- 0:0 412 B ├── ct_log_list.cnf +│ Image Details ├───────────────────────────────────────────────────────────────────────────────────────── -r--r--r-- 0:0 412 B ├── ct_log_list.cnf.dist + dr-xr-xr-x 0:0 0 B ├── engines-1.1 + -r--r--r-- 0:0 11 kB ├── openssl.cnf +Total Image size: 14 MB -r--r--r-- 0:0 11 kB └── openssl.cnf.dist +Potential wasted space: 0 B +Image efficiency score: 100 % + +Count Total Space Path +``` diff --git a/nix/docker/default.nix b/nix/docker/default.nix index fd9df4278..913e39eb6 100644 --- a/nix/docker/default.nix +++ b/nix/docker/default.nix @@ -1,11 +1,12 @@ -{ postgrest, dockerTools, writeShellScriptBin }: +{ buildEnv, postgrest, dockerTools, writeShellScriptBin }: let - image = - tag: - dockerTools.buildImage { - inherit tag; + config = + ./postgrest.conf; - name = "postgrest/postgrest"; + image = + dockerTools.buildImage { + name = "postgrest"; + tag = "latest"; contents = postgrest; # Set the current time as the image creation date. This makes the build @@ -15,7 +16,8 @@ let extraCommands = '' mkdir etc - cp ${./postgrest.conf} etc/postgrest.conf + cp ${config} etc/postgrest.conf + rmdir share ''; config = { @@ -25,6 +27,7 @@ let "PGRST_DB_SCHEMA=public" "PGRST_DB_ANON_ROLE=" "PGRST_DB_POOL=100" + "PGRST_DB_POOL_TIMEOUT=10" "PGRST_DB_EXTRA_SEARCH_PATH=public" "PGRST_SERVER_HOST=*4" "PGRST_SERVER_PORT=3000" @@ -44,20 +47,15 @@ let }; }; }; -in -rec { - imageLatest = - image "latest"; - - imageWithVersion = - image "v${postgrest.version}"; + # Helper script for loading the image. load = writeShellScriptBin "postgrest-docker-load" '' - set -euo pipefail - - docker load -i ${imageLatest} - docker load -i ${imageWithVersion} + docker load -i ${image} ''; -} +in +buildEnv { + name = "postgrest-docker"; + paths = [ load ]; +} // { inherit image config; } diff --git a/nix/docker/postgrest.conf b/nix/docker/postgrest.conf index c1b5b9bd7..6e22c38f1 100644 --- a/nix/docker/postgrest.conf +++ b/nix/docker/postgrest.conf @@ -1,7 +1,15 @@ +# See https://postgrest.org/en/stable/configuration.html#configuration + +# Required settings + db-uri = "$(PGRST_DB_URI)" db-schema = "$(PGRST_DB_SCHEMA)" db-anon-role = "$(PGRST_DB_ANON_ROLE)" + +# Optional settings + db-pool = "$(PGRST_DB_POOL)" +db-pool-timeout = "$(PGRST_DB_POOL_TIMEOUT)" db-extra-search-path = "$(PGRST_DB_EXTRA_SEARCH_PATH)" server-host = "$(PGRST_SERVER_HOST)" diff --git a/nix/overlays/default.nix b/nix/overlays/default.nix index b52781049..70477cf29 100644 --- a/nix/overlays/default.nix +++ b/nix/overlays/default.nix @@ -1,5 +1,6 @@ { gitignore = import ./gitignore.nix; + ghr = import ./ghr; haskell-packages = import ./haskell-packages; postgresql-default = import ./postgresql-default.nix; postgresql-legacy = import ./postgresql-legacy.nix; diff --git a/nix/overlays/ghr/default.nix b/nix/overlays/ghr/default.nix new file mode 100644 index 000000000..fc7f1549e --- /dev/null +++ b/nix/overlays/ghr/default.nix @@ -0,0 +1,7 @@ +self: super: +# Overlay that adds `ghr`: Upload multiple artifacts to GitHub Release in +# parallel, http://tcnksm.github.io/ghr/ + +{ + ghr = super.callPackage ./ghr.nix { }; +} diff --git a/nix/overlays/ghr/ghr.nix b/nix/overlays/ghr/ghr.nix new file mode 100644 index 000000000..470f5ce0a --- /dev/null +++ b/nix/overlays/ghr/ghr.nix @@ -0,0 +1,15 @@ +{ buildGoModule, fetchFromGitHub }: + +buildGoModule rec { + pname = "ghr"; + version = "0.13.0"; + + src = fetchFromGitHub { + rev = "v${version}"; + owner = "tcnksm"; + repo = "ghr"; + sha256 = "1nm5kdjkqayxh06j9nr5daic9sw9nx9w06y9gaqhdrw9byvjpr1a"; + }; + + vendorSha256 = "14avsngzhl1b8a05i43ph6sxh9vj0jls0acxr9j7r0h3f0vpamcj"; +} diff --git a/nix/release/default.nix b/nix/release/default.nix new file mode 100644 index 000000000..57f2d6ef3 --- /dev/null +++ b/nix/release/default.nix @@ -0,0 +1,101 @@ +{ buildEnv +, curl +, envsubst +, ghr +, jq +, postgrest +, docker +, runCommand +, writeShellScriptBin +}: +let + # Version from the postgrest.cabal file (gotten with callCabal2nix). + version = postgrest.version; + + # Set of files that will be published in the GitHub release. + releaseFiles = + runCommand "postgrest-release-files" + { inherit postgrest version; } + '' + set -euo pipefail + + mkdir -p $out + + tar cvJf "$out"/postgrest-v"$version"-linux-x64-static.tar.xz \ + -C "$postgrest"/bin postgrest + ''; + + # Script for publishing a new release on GitHub. + github = + writeShellScriptBin "postgrest-release-github" + '' + set -euo pipefail + + changes="$(sed -n "1,/${version}/d;/## \[/q;p" ${../../CHANGELOG.md})" + + ${ghr}/bin/ghr \ + -t "$GITHUB_TOKEN" \ + -u "$GITHUB_USERNAME" \ + -r "$GITHUB_REPONAME" \ + -b "$changes" \ + --replace v${version} \ + ${releaseFiles} + ''; + + # Script for publishing a new release on Docker Hub. + dockerHub = + writeShellScriptBin "postgrest-release-dockerhub" + '' + set -euo pipefail + + docker load -i ${docker.image} + + docker tag postgrest:latest "$DOCKER_REPO"/postgrest:latest + docker tag postgrest:latest "$DOCKER_REPO"/postgrest:v${version} + + docker push "$DOCKER_REPO"/postgrest:latest + docker push "$DOCKER_REPO"/postgrest:v${version} + ''; + + # Script for updating the repository description on Docker Hub. + dockerHubDescription = + let + description = + ./docker-hub-description.md; + + fullDescription = + ./docker-hub-full-description.md; + in + writeShellScriptBin "postgrest-release-dockerhubdescription" + '' + set -euo pipefail + + # Login to Docker Hub and get a token. + token="$( + ${curl}/bin/curl -sH "Content-Type: application/json" \ + --data-urlencode "username=$DOCKER_USERNAME" \ + --data-urlencode "password=$DOCKER_PASSWORD" \ + "https://hub.docker.com/v2/users/login/" \ + | ${jq}/bin/jq -r .token + )" + + # Plug the default config file into the full description. + export DEFAULT_CONFIG="$(cat ${docker.config})" + fullDescription="$(${envsubst}/bin/envsubst < ${fullDescription})" + + # Patch the full description. + responseCode="$( + ${curl}/bin/curl -s --write-out %{response_code} --output /dev/null \ + -H "Authorization: JWT $token" -X PATCH \ + --data-urlencode description@${description} \ + --data-urlencode "full_description=$fullDescription" \ + "https://hub.docker.com/v2/repositories/$DOCKER_REPO/postgrest/" + )" + + [ "$responseCode" -eq 200 ] + ''; +in +buildEnv { + name = "postgrest-release"; + paths = [ github dockerHub dockerHubDescription ]; +} diff --git a/nix/release/docker-hub-description.md b/nix/release/docker-hub-description.md new file mode 100644 index 000000000..fc44c8542 --- /dev/null +++ b/nix/release/docker-hub-description.md @@ -0,0 +1 @@ +REST API for any Postgres database diff --git a/nix/release/docker-hub-full-description.md b/nix/release/docker-hub-full-description.md new file mode 100644 index 000000000..5a9964164 --- /dev/null +++ b/nix/release/docker-hub-full-description.md @@ -0,0 +1,33 @@ +# PostgREST + +[![Join the chat at https://gitter.im/begriffs/postgrest](https://img.shields.io/badge/gitter-join%20chat%20%E2%86%92-brightgreen.svg)](https://gitter.im/begriffs/postgrest) +[![Donate](https://img.shields.io/badge/Donate-Patreon-orange.svg?colorB=F96854)](https://www.patreon.com/postgrest) +[![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.me/postgrest) +[![Docs](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://postgrest.org) +[![Build Status](https://circleci.com/gh/PostgREST/postgrest/tree/master.svg?style=shield)](https://circleci.com/gh/PostgREST/postgrest/tree/master) + +PostgREST serves a fully RESTful API from any existing PostgreSQL database. It +provides a cleaner, more standards-compliant, faster API than you are likely to +write from scratch. + +# Usage + +To learn how to use this container, see the [PostgREST Docker +documentation](https://postgrest.com/en/stable/install.html#docker). + +You can configure the PostgREST image by setting the enviroment variables used +in the default `/etc/postgrest.conf` file or overriding that file. This is the +default configuration file: + +``` +$DEFAULT_CONFIG +``` + +# How this image is built + +The image is built from scratch using +[Nix](https://nixos.org/nixpkgs/manual/#sec-pkgs-dockerTools) instead of a +`Dockerfile`, which yields a higly secure and optimized image. This is also why +no commands are listed in the image history. See the [PostgREST +respository](https://github.com/PostgREST/postgrest/tree/master/nix/docker) for +details on the build process and how to inspect the image. diff --git a/nix/tests.nix b/nix/tests.nix index 737913276..2580c98b9 100644 --- a/nix/tests.nix +++ b/nix/tests.nix @@ -33,14 +33,14 @@ let '' set -euo pipefail + export PATH="$(cat ${postgrestBuildEnv})"/bin:"$PATH" + cat << EOF Running spec against ${postgresql.name}... EOF - # TODO: Make this work outside nix-shell when installed with nix-env. - # Probably using postgrestBuildEnv somehow? ${withTmpDb postgresql} ${cabal-install}/bin/cabal v2-test \ --test-show-detail=direct