jimmy/postgrest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add CI for ARM architectures (#2127)

Browse Source
This commit is contained in:
Laurence Isla
2022-04-18 17:40:32 +02:00
committed by GitHub
parent 7589102c6c
commit bc7d54c7d2
4 changed files with 292 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/arm/build.sh
+57
View File
@@ -0,0 +1,57 @@
#!/bin/bash
# This script builds PostgREST in a remote ARM server. It uses Docker to
# build for multiple platforms (aarch64 and armv7 on ubuntu).
# The Dockerfile is located in ./docker-env
[ -z "$1" ] && { echo "Missing 1st argument: PostgREST github commit SHA"; exit 1; }
[ -z "$2" ] && { echo "Missing 2nd argument: Docker repo"; exit 1; }
[ -z "$3" ] && { echo "Missing 3rd argument: Docker username"; exit 1; }
[ -z "$4" ] && { echo "Missing 4th argument: Docker password"; exit 1; }
[ -z "$5" ] && { echo "Missing 5th argument: Build environment directory name"; exit 1; }
PGRST_GITHUB_COMMIT="$1"
DOCKER_REPO="$2"
DOCKER_USER="$3"
DOCKER_PASS="$4"
SCRIPT_PATH="$5"
DOCKER_BUILD_PATH="$SCRIPT_PATH/docker-env"
clean_env()
{
sudo docker logout
}
# Login to Docker
sudo docker logout
{ echo $DOCKER_PASS | sudo docker login -u $DOCKER_USER --password-stdin; } || { echo "Couldn't login to docker"; exit 1; }
trap clean_env sigint sigterm exit
# Move to the docker build environment
cd ~/$DOCKER_BUILD_PATH
# Build ARM versions
sudo docker buildx build --build-arg PGRST_GITHUB_COMMIT=$PGRST_GITHUB_COMMIT \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--platform linux/arm/v7,linux/arm64 \
--cache-from $DOCKER_REPO/postgrest-build-arm \
--target=postgrest-build \
-t $DOCKER_REPO/postgrest-build-arm \
--push .
sudo docker logout
# Generate and copy binaries to the local filesystem
sudo docker buildx build --build-arg PGRST_GITHUB_COMMIT=$PGRST_GITHUB_COMMIT \
--cache-from $DOCKER_REPO/postgrest-build-arm \
--platform linux/arm/v7,linux/arm64 \
--target=postgrest-bin \
-o result .
# Compress binaries
sudo chown -R ubuntu:ubuntu ~/$DOCKER_BUILD_PATH/result
mv ~/$DOCKER_BUILD_PATH/result ~/$SCRIPT_PATH/result
cd ~/$SCRIPT_PATH
tar -cJf result.tar.xz result
.github/scripts/arm/docker-env/Dockerfile
+73
View File
@@ -0,0 +1,73 @@
# Build PostgREST for ARM architectures
FROM ubuntu:focal as postgrest-build
RUN apt-get update -y \
&& apt-get upgrade -y \
&& apt-get install -y git build-essential curl libffi-dev libffi7 libgmp-dev libgmp10 libncurses-dev libncurses5 libtinfo5 llvm libnuma-dev zlib1g-dev libpq-dev jq gcc \
&& apt-get clean
# Install ghcup
ENV BOOTSTRAP_HASKELL_NONINTERACTIVE=1
RUN bash -c "curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh"
# Add ghcup to PATH
ENV PATH=${PATH}:/root/.local/bin
ENV PATH=${PATH}:/root/.ghcup/bin
# Install cabal
RUN bash -c "ghcup upgrade"
RUN bash -c "ghcup install cabal 3.4.0.0"
RUN bash -c "ghcup set cabal 3.4.0.0"
# Install GHC
RUN bash -c "ghcup install ghc 8.10.7"
RUN bash -c "ghcup set ghc 8.10.7"
# Update Path to include Cabal and GHC exports
RUN bash -c "echo PATH="$HOME/.local/bin:$PATH" >> $HOME/.bashrc"
RUN bash -c "echo export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" >> $HOME/.bashrc"
RUN bash -c "source $HOME/.bashrc"
# Clone the repository
RUN git clone https://github.com/PostgREST/postgrest.git /postgrest
WORKDIR /postgrest
RUN cabal v2-update && cabal v2-build
# Arguments are declared here to save the above installation in cache
ARG PGRST_GITHUB_COMMIT
RUN git pull origin main \
&& git checkout $PGRST_GITHUB_COMMIT
# Build PostgREST
RUN mkdir -p /build-export
RUN cabal v2-update && cabal v2-build
RUN PGRST_BIN=$(cabal exec which postgrest | tail -1) \
&& mv $PGRST_BIN /build-export
# Simple image to generate the PostgREST binaries
FROM scratch as postgrest-bin
COPY --from=postgrest-build /build-export /
# PostgREST docker hub image
FROM ubuntu:focal AS postgrest
RUN apt-get update -y
RUN apt install libpq-dev zlib1g-dev jq gcc libnuma-dev -y
RUN apt-get clean
COPY --from=postgrest-bin postgrest /usr/bin/postgrest
EXPOSE 3000
USER 1000
CMD postgrest
.github/scripts/arm/docker-publish.sh
+52
View File
@@ -0,0 +1,52 @@
#!/bin/bash
# This script publishes the Docker ARM images to Docker Hub.
[ -z "$1" ] && { echo "Missing 1st argument: PostgREST github commit SHA"; exit 1; }
[ -z "$2" ] && { echo "Missing 2nd argument: Docker repo"; exit 1; }
[ -z "$3" ] && { echo "Missing 3rd argument: Docker username"; exit 1; }
[ -z "$4" ] && { echo "Missing 4th argument: Docker password"; exit 1; }
[ -z "$5" ] && { echo "Missing 5th argument: Build environment directory name"; exit 1; }
[ -z "$6" ] && { echo "Missing 6th argument: PostgREST version"; exit 1; }
PGRST_GITHUB_COMMIT="$1"
DOCKER_REPO="$2"
DOCKER_USER="$3"
DOCKER_PASS="$4"
SCRIPT_PATH="$5"
PGRST_VERSION="v$6"
IS_PRERELEASE="$7"
DOCKER_BUILD_PATH="$SCRIPT_PATH/docker-env"
clean_env()
{
sudo docker logout
}
# Login to Docker
sudo docker logout
{ echo $DOCKER_PASS | sudo docker login -u $DOCKER_USER --password-stdin; } || { echo "Couldn't login to docker"; exit 1; }
trap clean_env sigint sigterm exit
# Move to the docker build environment
cd ~/$DOCKER_BUILD_PATH
# Push final images to Docker hub
# NOTE: This command publishes a separate ARM image because the builds cannot
# be added to the manifest if they are not in the registry beforehand.
# This image must be manually deleted from Docker Hub at the end of the process.
sudo docker buildx build --build-arg PGRST_GITHUB_COMMIT=$PGRST_GITHUB_COMMIT \
--platform linux/arm/v7,linux/arm64 \
--cache-from $DOCKER_REPO/postgrest-build-arm \
-t $DOCKER_REPO/postgrest:$PGRST_VERSION-arm \
--push .
# Add the arm images to the manifest
# NOTE: This assumes that there already is a `postgrest:<version>` image
# for the amd64 architecture pushed to Docker Hub
sudo docker buildx imagetools create --append -t $DOCKER_REPO/postgrest:$PGRST_VERSION $DOCKER_REPO/postgrest:$PGRST_VERSION-arm
[ -z $IS_PRERELEASE ] && sudo docker buildx imagetools create --append -t $DOCKER_REPO/postgrest:latest $DOCKER_REPO/postgrest:$PGRST_VERSION-arm
sudo docker logout
.github/workflows/ci.yaml
+110 -3
View File
@@ -204,7 +204,6 @@ jobs:
result/postgrest.exe
if-no-files-found: error
Get-FreeBSD-CirrusCI:
name: Get FreeBSD build from CirrusCI
runs-on: ubuntu-latest
@@ -222,6 +221,70 @@ jobs:
path: postgrest
if-no-files-found: error
Build-Cabal-Arm:
name: Build armv7/aarch64 (Cabal)
if: ${{ github.ref == 'refs/heads/main' }}
runs-on: ubuntu-latest
outputs:
remotepath: ${{ steps.Remote-Dir.outputs.remotepath }}
env:
GITHUB_COMMIT: ${{ github.sha }}
DOCKER_REPO: "postgrest"
DOCKER_USER: "stevechavez"
DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
steps:
- uses: actions/checkout@v2.4.0
- id: Remote-Dir
name: Unique directory name for the remote build
run: echo "::set-output name=remotepath::postgrest-build-$(uuidgen)"
- name: Copy script files to the remote server
uses: appleboy/scp-action@master
with:
host: ${{ secrets.SSH_ARM_HOST }}
username: ubuntu
key: ${{ secrets.SSH_ARM_PRIVATE_KEY }}
fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
source: ".github/scripts/arm/*"
target: ${{ steps.Remote-Dir.outputs.remotepath }}
strip_components: 3
- name: Build ARM
uses: appleboy/ssh-action@master
env:
REMOTE_DIR: ${{ steps.Remote-Dir.outputs.remotepath }}
with:
host: ${{ secrets.SSH_ARM_HOST }}
username: ubuntu
key: ${{ secrets.SSH_ARM_PRIVATE_KEY }}
fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
command_timeout: 120m
script_stop: true
envs: GITHUB_COMMIT,DOCKER_REPO,DOCKER_USER,DOCKER_PASS,REMOTE_DIR
script: bash ~/$REMOTE_DIR/build.sh "$GITHUB_COMMIT" "$DOCKER_REPO" "$DOCKER_USER" "$DOCKER_PASS" "$REMOTE_DIR"
- name: Download binaries from remote server
uses: nicklasfrahm/scp-action@main
with:
direction: download
host: ${{ secrets.SSH_ARM_HOST }}
username: ubuntu
key: ${{ secrets.SSH_ARM_PRIVATE_KEY }}
fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
source: "${{ steps.Remote-Dir.outputs.remotepath }}/result.tar.xz"
target: "result.tar.xz"
- name: Extract downloaded binaries
run: tar -xvf result.tar.xz && rm result.tar.xz
- name: Save aarch64 executable as artifact
uses: actions/upload-artifact@v2.3.1
with:
name: postgrest-ubuntu-aarch64
path: result/linux_arm64/postgrest
if-no-files-found: error
- name: Save armv7 executable as artifact
uses: actions/upload-artifact@v2.3.1
with:
name: postgrest-ubuntu-armv7
path: result/linux_arm_v7/postgrest
if-no-files-found: error
Prepare-Release:
name: Prepare release
@@ -235,6 +298,7 @@ jobs:
- Build-Static-Nix
- Build-Stack
#- Get-FreeBSD-CirrusCI
- Build-Cabal-Arm
outputs:
version: ${{ steps.Identify-Version.outputs.version }}
isprerelease: ${{ steps.Identify-Version.outputs.isprerelease }}
@@ -318,6 +382,12 @@ jobs:
#tar cJvf "release-bundle/postgrest-v$VERSION-freebsd-x64.tar.xz" \
# -C artifacts/postgrest-freebsd-x64 postgrest
tar cJvf "release-bundle/postgrest-v$VERSION-ubuntu-aarch64.tar.xz" \
-C artifacts/postgrest-ubuntu-aarch64 postgrest
tar cJvf "release-bundle/postgrest-v$VERSION-ubuntu-armv7.tar.xz" \
-C artifacts/postgrest-ubuntu-armv7 postgrest
zip "release-bundle/postgrest-v$VERSION-windows-x64.zip" \
artifacts/postgrest-windows-x64/postgrest.exe
@@ -345,10 +415,14 @@ jobs:
Release-Docker:
name: Release on Docker Hub
runs-on: ubuntu-latest
needs: Prepare-Release
needs:
- Build-Cabal-Arm
- Prepare-Release
env:
GITHUB_COMMIT: ${{ github.sha }}
DOCKER_REPO: postgrest
DOCKER_USER: stevechavez
DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
VERSION: ${{ needs.Prepare-Release.outputs.version }}
ISPRERELEASE: ${{ needs.Prepare-Release.outputs.isprerelease }}
steps:
@@ -363,7 +437,7 @@ jobs:
name: postgrest-docker-x64
- name: Publish images on Docker Hub
run: |
docker login -u "$DOCKER_USER" -p "${{ secrets.DOCKER_PASS }}"
docker login -u "$DOCKER_USER" -p "$DOCKER_PASS"
docker load -i postgrest-docker.tar.gz
docker tag postgrest:latest "$DOCKER_REPO/postgrest:v$VERSION"
@@ -377,6 +451,18 @@ jobs:
else
echo "Skipping pushing to 'latest' tag for v$VERSION pre-release..."
fi
- name: Publish images for ARM builds on Docker Hub
uses: appleboy/ssh-action@master
env:
REMOTE_DIR: ${{ needs.Build-Cabal-Arm.outputs.remotepath }}
with:
host: ${{ secrets.SSH_ARM_HOST }}
username: ubuntu
key: ${{ secrets.SSH_ARM_PRIVATE_KEY }}
fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
script_stop: true
envs: GITHUB_COMMIT,DOCKER_REPO,DOCKER_USER,DOCKER_PASS,REMOTE_DIR,VERSION,ISPRERELEASE
script: bash ~/$REMOTE_DIR/docker-publish.sh "$GITHUB_COMMIT" "$DOCKER_REPO" "$DOCKER_USER" "$DOCKER_PASS" "$REMOTE_DIR" "$VERSION" "$ISPRERELEASE"
# TODO: Enable dockerhub description update again, once a solution for the permission problem is found:
# https://github.com/docker/hub-feedback/issues/1927
# - name: Update descriptions on Docker Hub
@@ -389,3 +475,24 @@ jobs:
# else
# echo "Skipping updating description for pre-release..."
# fi
Clean-Arm-Server:
name: Remove copied files from server
needs:
- Build-Cabal-Arm
- Release-Docker
if: ${{ always() && github.ref == 'refs/heads/main' }}
runs-on: ubuntu-latest
env:
REMOTE_DIR: ${{ needs.Build-Cabal-Arm.outputs.remotepath }}
steps:
- uses: actions/checkout@v2.4.0
- name: Remove uploaded files from server
uses: appleboy/ssh-action@master
with:
host: ${{ secrets.SSH_ARM_HOST }}
username: ubuntu
key: ${{ secrets.SSH_ARM_PRIVATE_KEY }}
fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
envs: REMOTE_DIR
script: rm -rf $REMOTE_DIR