diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index db6f09122..71615ac02 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -344,6 +344,18 @@ jobs:
           fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
           source: "${{ steps.Remote-Dir.outputs.remotepath }}/result.tar.xz"
           target: "result.tar.xz"
+      - name: Remove uploaded files from server
+        if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        uses: appleboy/ssh-action@master
+        env:
+          REMOTE_DIR: ${{ steps.Remote-Dir.outputs.remotepath }}
+        with:
+          host: ${{ secrets.SSH_ARM_HOST }}
+          username: ubuntu
+          key: ${{ secrets.SSH_ARM_PRIVATE_KEY }}
+          fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
+          envs: REMOTE_DIR
+          script: rm -rf $REMOTE_DIR
       - name: Extract downloaded binaries
         run: tar -xvf result.tar.xz && rm result.tar.xz
       - name: Save aarch64 executable as artifact
@@ -560,10 +572,7 @@ jobs:
     needs:
       - Build-Cabal-Arm
       - Release-Docker-Arm
-    if: success() ||
-      needs.Build-Cabal-Arm.result == 'failure' ||
-      needs.Build-Cabal-Arm.result == 'cancelled' ||
-      (needs.Build-Cabal-Arm.result == 'success' && !startsWith(github.ref, 'refs/tags/v'))
+    if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     env:
       REMOTE_DIR: ${{ needs.Build-Cabal-Arm.outputs.remotepath }}