Use cross-compilation for CI, build the Docker image from source (#85)

This commit is contained in:
mdecimus
2024-04-12 16:35:00 +02:00
parent 4ba65d6676
commit d876033c1a
17 changed files with 391 additions and 628 deletions

View File

@@ -1,351 +1,230 @@
# Credits: https://github.com/33KK
name: Build
on:
workflow_dispatch:
pull_request:
push:
tags:
- "v*.*.*"
env:
REGISTRY_IMAGE: stalwartlabs/mail-server
tags: ["v*.*.*"]
branches: ["main"]
jobs:
build:
name: Building for ${{ matrix.target }} on ${{ matrix.host_os }}
runs-on: ${{ matrix.host_os }}
if: '!cancelled()'
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
host_os: ubuntu-20.04
- target: x86_64-apple-darwin
host_os: macos-latest
- target: x86_64-pc-windows-msvc
host_os: windows-2022
#- target: aarch64-unknown-linux-gnu
# host_os: ubuntu-20.04
#- target: x86_64-unknown-linux-musl
# host_os: ubuntu-20.04
#- target: aarch64-unknown-linux-musl
# host_os: ubuntu-20.04
#- target: aarch64-apple-darwin
# host_os: macos-latest
#- target: aarch64-pc-windows-msvc
# host_os: windows-2022
#- target: aarch64-pc-windows-msvc
# host_os: windows-2022
#- target: arm-unknown-linux-musleabihf
# host_os: ubuntu-20.04
#- target: arm-unknown-linux-gnueabihf
# host_os: ubuntu-20.04
#- target: armv7-unknown-linux-musleabihf
# host_os: ubuntu-20.04
#- target: armv7-unknown-linux-gnueabihf
# host_os: ubuntu-20.04
name: Build / ${{matrix.target}}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
persist-credentials: false
runs-on: ${{matrix.host_os}}
- name: Install dependencies (Linux)
if: ${{ contains(matrix.host_os, 'ubuntu') }}
run: |
sudo apt-get update -y
sudo apt-get install -yq protobuf-compiler wget
wget https://github.com/apple/foundationdb/releases/download/7.1.0/foundationdb-clients_7.1.0-1_amd64.deb
sudo dpkg -i --force-architecture foundationdb-clients_7.1.0-1_amd64.deb
- name: Install dependencies (MacOs)
if: ${{ contains(matrix.host_os, 'macos') }}
run: |
brew install protobuf
brew install wget
wget https://github.com/apple/foundationdb/releases/download/7.1.32/FoundationDB-7.1.32_x86_64.pkg
sudo installer -allowUntrusted -dumplog -pkg FoundationDB-7.1.32_x86_64.pkg -target /
- name: Install dependencies (Windows)
if: ${{ contains(matrix.host_os, 'windows') }}
uses: arduino/setup-protoc@v1
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
override: true
target: ${{ matrix.target }}
toolchain: stable
- name: Rust Cache
uses: Swatinem/rust-cache@v2
with:
key: ${{ matrix.host_os }}-${{ matrix.target }}-mail
- name: Building binary (Unix version)
if: ${{ !contains(matrix.host_os, 'windows') }}
run: |
cargo build -p mail-server --target=${{ matrix.target }} --no-default-features --features "foundationdb elastic s3 redis" --release
cd target/${{ matrix.target }}/release && tar czvf ../../../stalwart-mail-foundationdb-${{ matrix.target }}.tar.gz stalwart-mail && cd -
cargo build -p mail-server -p stalwart-cli --target=${{ matrix.target }} --release
cd target/${{ matrix.target }}/release
tar czvf ../../../stalwart-mail-${{ matrix.target }}.tar.gz stalwart-mail
tar czvf ../../../stalwart-cli-${{ matrix.target }}.tar.gz stalwart-cli
cd -
- name: Building binary (Windows version)
if: ${{ contains(matrix.host_os, 'windows') }}
run: |
cargo build -p mail-server -p stalwart-cli --target=${{ matrix.target }} --release
cd target/${{ matrix.target }}/release
7z a ../../../stalwart-mail-${{ matrix.target }}.zip stalwart-mail.exe
7z a ../../../stalwart-cli-${{ matrix.target }}.zip stalwart-cli.exe
cd -
- name: Publish Release
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v1
with:
files: 'stalwart-*'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
cross_build_tools:
runs-on: ubuntu-latest
name: Building tools for ${{ matrix.target }} on ${{ matrix.distro }}
if: '!cancelled()'
strategy:
matrix:
include:
- arch: aarch64
distro: ubuntu20.04
target: aarch64-unknown-linux-gnu
steps:
- uses: actions/checkout@v3
- uses: uraimo/run-on-arch-action@v2
name: Build artifact
id: build
with:
arch: ${{ matrix.arch }}
distro: ${{ matrix.distro }}
# Not required, but speeds up builds
githubToken: ${{ github.token }}
# Create an artifacts directory
setup: |
mkdir -p "${PWD}/artifacts"
# Mount the artifacts directory as /artifacts in the container
dockerRunArgs: |
--volume "${PWD}/artifacts:/artifacts"
# Pass some environment variables to the container
env: |
target: ${{ matrix.target }}
# The shell to run commands with in the container
shell: /bin/sh
install: |
apt-get update -yq
apt-get install -yq build-essential cmake wget curl
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
run: |
export PATH="$HOME/.cargo/bin:$PATH"
cargo build -p stalwart-cli --target=${target} --release
cd target/${target}/release
tar czvf /artifacts/stalwart-cli-${target}.tar.gz stalwart-cli
cd -
- name: Move packages
run: |
mv ${PWD}/artifacts/* .
- name: Publish Release
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v1
with:
files: 'stalwart-*'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
cross_build:
runs-on: ubuntu-latest
name: Building for ${{ matrix.target }} on ${{ matrix.distro }}
if: '!cancelled()'
strategy:
matrix:
include:
- arch: aarch64
distro: ubuntu20.04
target: aarch64-unknown-linux-gnu
steps:
- uses: actions/checkout@v3
- uses: uraimo/run-on-arch-action@v2
name: Build artifact
id: build
with:
arch: ${{ matrix.arch }}
distro: ${{ matrix.distro }}
# Not required, but speeds up builds
githubToken: ${{ github.token }}
# Create an artifacts directory
setup: |
mkdir -p "${PWD}/artifacts"
# Mount the artifacts directory as /artifacts in the container
dockerRunArgs: |
--volume "${PWD}/artifacts:/artifacts"
# Pass some environment variables to the container
env: |
target: ${{ matrix.target }}
# The shell to run commands with in the container
shell: /bin/sh
install: |
apt-get update -yq
apt-get install -yq build-essential cmake protobuf-compiler wget curl clang libclang-dev
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
run: |
export PATH="$HOME/.cargo/bin:$PATH"
cargo build -p mail-server --target=${target} --release
cd target/${target}/release
tar czvf /artifacts/stalwart-mail-${target}.tar.gz stalwart-mail
cd -
- name: Move packages
run: |
mv ${PWD}/artifacts/* .
- name: Publish Release
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v1
with:
files: 'stalwart-*'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
build_docker:
needs:
- build
- cross_build
- cross_build_tools
name: Build Docker image for ${{ matrix.platform }}
runs-on: ubuntu-latest
if: '!cancelled()'
strategy:
fail-fast: false
matrix:
platform:
- linux/amd64
- linux/arm64
include:
- target: aarch64-unknown-linux-gnu
host_os: ubuntu-20.04
use_cross: true
- target: armv7-unknown-linux-gnueabihf
host_os: ubuntu-20.04
use_cross: true
#- target: arm-unknown-linux-gnueabihf
# host_os: ubuntu-20.04
# use_cross: true
- target: x86_64-unknown-linux-gnu
host_os: ubuntu-20.04
use_cross: false
- target: aarch64-unknown-linux-musl
host_os: ubuntu-20.04
use_cross: true
#- target: armv7-unknown-linux-musleabihf
# host_os: ubuntu-20.04
# use_cross: true
#- target: arm-unknown-linux-musleabihf
# host_os: ubuntu-20.04
# use_cross: true
- target: x86_64-unknown-linux-musl
host_os: ubuntu-20.04
use_cross: true
- target: aarch64-apple-darwin
host_os: macos-latest
use_cross: false
- target: x86_64-apple-darwin
host_os: macos-latest
use_cross: false
# FIXME: waiting for ldap3 and hickory-resolver bump to ring 0.17.x
#- target: aarch64-pc-windows-msvc
# host_os: windows-2022
# use_cross: false
- target: x86_64-pc-windows-msvc
host_os: windows-2022
use_cross: false
steps:
-
name: Prepare
- name: Checkout
uses: actions/checkout@v4
- name: Install Dependencies (Linux)
if: startsWith(matrix.host_os, 'ubuntu')
shell: bash
run: |
platform=${{ matrix.platform }}
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
-
name: Checkout
uses: actions/checkout@v3
-
name: Docker meta
sudo apt-get update
sudo apt-get install -yq build-essential
- name: Install FoundationDB (x86_64-unknown-linux-gnu)
if: matrix.target == 'x86_64-unknown-linux-gnu'
run: |
curl -LO https://github.com/apple/foundationdb/releases/download/7.1.0/foundationdb-clients_7.1.0-1_amd64.deb
sudo dpkg -i --force-architecture foundationdb-clients_7.1.0-1_amd64.deb
echo "USE_FOUNDATIONDB=1" >> "$GITHUB_ENV"
- name: Install FoundationDB (x86_64-apple-darwin)
if: matrix.target == 'x86_64-apple-darwin'
run: |
curl -LO https://github.com/apple/foundationdb/releases/download/7.1.34/FoundationDB-7.1.34_x86_64.pkg
sudo installer -allowUntrusted -dumplog -pkg FoundationDB-7.1.34_x86_64.pkg -target /
echo "USE_FOUNDATIONDB=1" >> "$GITHUB_ENV"
- name: Rust Cache
uses: Swatinem/rust-cache@v2
with:
key: ${{matrix.host_os}}-${{matrix.target}}-mail
- name: Install Cross
if: matrix.use_cross == true
uses: baptiste0928/cargo-install@v2
with:
crate: cross
git: https://github.com/cross-rs/cross
- name: Build
shell: bash
run: |
set -eux
target="${{matrix.target}}"
rustup target add "${target}"
root="${PWD}"
mkdir artifacts archives
ext="${{startsWith(matrix.host_os, 'windows') == true && '.exe' || ''}}"
# Workaround a Windows moment
export PATH="/c/Strawberry/c/bin:/c/Strawberry/perl/site/bin:/c/Strawberry/perl/bin:$PATH"
build() {
${{matrix.env}} ${{matrix.use_cross == true && 'cross' || 'cargo'}} build --release --target "${target}" "$@"
}
artifact() {
local file="${1}${ext}"
local name="${root}/archives/${2:-$1}-${target}"
if [ "${ext}" = ".exe" ]; then
7z a "${name}.zip" "${file}"
else
tar czvf "${name}.tar.gz" "${file}"
fi
mv "${file}" "${root}/artifacts/${2:-$1}"
}
mkdir -p "${root}/target/${target}/release" && cd "$_"
if [ "${USE_FOUNDATIONDB:-0}" = 1 ]; then
build -p mail-server --features foundationdb
else
build -p mail-server
fi
build -p stalwart-cli
artifact stalwart-mail
artifact stalwart-cli
- name: Upload Archives
uses: actions/upload-artifact@v3
with:
name: archives
path: ./archives
- name: Upload Artifacts
uses: actions/upload-artifact@v3
with:
name: ${{matrix.target}}
path: ./artifacts
release:
name: Release
if: github.event_name == 'push'
needs: build
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Download Artifacts
uses: actions/download-artifact@v3
with:
name: archives
path: ./archives
- name: Release
uses: softprops/action-gh-release@v1
with:
files: ./archives/*
prerelease: ${{!startsWith(github.ref, 'refs/tags/') == true && true || null}}
tag_name: ${{!startsWith(github.ref, 'refs/tags/') == true && 'nightly' || null}}
docker_build:
name: Docker Build
if: github.event_name == 'push'
runs-on: ubuntu-latest
permissions:
packages: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Log In to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{github.repository_owner}}
password: ${{github.token}}
- name: Log In to DockerHub
uses: docker/login-action@v2
with:
username: ${{secrets.DOCKERHUB_USERNAME}}
password: ${{secrets.DOCKERHUB_TOKEN}}
- name: Extract Metadata for Docker
id: meta
uses: docker/metadata-action@v4
with:
images: ${{ env.REGISTRY_IMAGE }}
images: |
${{github.repository}}
ghcr.io/${{github.repository}}
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
-
name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Build and push by digest
type=ref,event=tag
type=edge,branch=main
- name: Build and Push Docker Images
id: build
uses: docker/build-push-action@v4
with:
context: .
platforms: ${{ matrix.platform }}
labels: ${{ steps.meta.outputs.labels }}
outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
#cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache-${{ env.PLATFORM_PAIR }}
#cache-to: type=registry,ref=s${{ env.REGISTRY_IMAGE }}:buildcache-${{ env.PLATFORM_PAIR }},mode=max
cache-from: type=gha,scope=build-${{ env.PLATFORM_PAIR }}
cache-to: type=gha,scope=build-${{ env.PLATFORM_PAIR }}
-
name: Export digest
run: |
mkdir -p /tmp/digests
digest="${{ steps.build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"
-
name: Upload digest
uses: actions/upload-artifact@v3
with:
name: digests
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
merge_docker:
name: Merge and push Docker manifest
runs-on: ubuntu-latest
needs:
- build_docker
steps:
-
name: Download digests
uses: actions/download-artifact@v3
with:
name: digests
path: /tmp/digests
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
-
name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
images: ${{ env.REGISTRY_IMAGE }}
-
name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Create manifest list and push
working-directory: /tmp/digests
run: |
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
-
name: Inspect image
run: |
docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
push: true
platforms: linux/amd64,linux/arm64
tags: ${{steps.meta.outputs.tags}}
labels: ${{steps.meta.outputs.labels}}

View File

@@ -1,120 +0,0 @@
name: Docker Build
on:
workflow_dispatch:
pull_request:
env:
REGISTRY_IMAGE: stalwartlabs/mail-server
jobs:
build_docker:
name: Build Docker image for ${{ matrix.platform }}
runs-on: ubuntu-latest
if: '!cancelled()'
strategy:
fail-fast: false
matrix:
platform:
- linux/amd64
- linux/arm64
steps:
-
name: Prepare
run: |
platform=${{ matrix.platform }}
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
-
name: Checkout
uses: actions/checkout@v3
-
name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
images: ${{ env.REGISTRY_IMAGE }}
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
-
name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Build and push by digest
id: build
uses: docker/build-push-action@v4
with:
context: .
platforms: ${{ matrix.platform }}
labels: ${{ steps.meta.outputs.labels }}
outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
#cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache-${{ env.PLATFORM_PAIR }}
#cache-to: type=registry,ref=s${{ env.REGISTRY_IMAGE }}:buildcache-${{ env.PLATFORM_PAIR }},mode=max
cache-from: type=gha,scope=build-${{ env.PLATFORM_PAIR }}
cache-to: type=gha,scope=build-${{ env.PLATFORM_PAIR }}
-
name: Export digest
run: |
mkdir -p /tmp/digests
digest="${{ steps.build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"
-
name: Upload digest
uses: actions/upload-artifact@v3
with:
name: digests
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
merge_docker:
name: Merge and push Docker manifest
runs-on: ubuntu-latest
needs:
- build_docker
steps:
-
name: Download digests
uses: actions/download-artifact@v3
with:
name: digests
path: /tmp/digests
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
-
name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
images: ${{ env.REGISTRY_IMAGE }}
-
name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Create manifest list and push
working-directory: /tmp/digests
run: |
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
-
name: Inspect image
run: |
docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}

View File

@@ -4,8 +4,8 @@ on:
workflow_dispatch:
pull_request:
push:
tags:
- '*'
tags: ["v*.*.*"]
branches: ["*"]
jobs:
style:
@@ -13,98 +13,55 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v1
uses: actions/checkout@v4
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
components: rustfmt
profile: minimal
override: true
- name: cargo fmt -- --check
uses: actions-rs/cargo@v1
with:
command: fmt
args: --all -- --check
- name: Check Style
run: cargo fmt --all --check
test:
name: Test
needs: [style]
needs: style
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v1
uses: actions/checkout@v4
- name: Install dependencies
run: |
sudo apt-get update -y
sudo apt-get install -yq protobuf-compiler
wget https://github.com/glauth/glauth/releases/download/v2.2.0/glauth-linux-arm64
curl -LO https://github.com/glauth/glauth/releases/download/v2.2.0/glauth-linux-arm64
chmod a+rx glauth-linux-arm64
nohup ./glauth-linux-arm64 -c tests/resources/ldap.cfg &
wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20230629051228.0.0_amd64.deb -O minio.deb
curl -Lo minio.deb https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20230629051228.0.0_amd64.deb
sudo dpkg -i minio.deb
mkdir ~/minio
nohup minio server ~/minio --console-address :9090 &
wget https://dl.min.io/client/mc/release/linux-amd64/mc
curl -LO https://dl.min.io/client/mc/release/linux-amd64/mc
chmod a+rx mc
./mc alias set myminio http://localhost:9000 minioadmin minioadmin
./mc mb tmp
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
profile: minimal
override: true
- name: Rust Cache
uses: Swatinem/rust-cache@v2
- name: JMAP Protocol Tests
uses: actions-rs/cargo@v1
with:
command: test
args: --manifest-path=crates/jmap-proto/Cargo.toml
run: cargo test -p jmap_proto -- --nocapture
- name: IMAP Protocol Tests
uses: actions-rs/cargo@v1
with:
command: test
args: --manifest-path=crates/imap-proto/Cargo.toml
run: cargo test -p imap_proto -- --nocapture
- name: Full-text search Tests
uses: actions-rs/cargo@v1
with:
command: test
args: --manifest-path=crates/store/Cargo.toml
#- name: Store Tests
# uses: actions-rs/cargo@v1
# with:
# command: test
# args: --manifest-path=tests/Cargo.toml store -- --nocapture
run: cargo test -p store -- --nocapture
- name: Directory Tests
uses: actions-rs/cargo@v1
with:
command: test
args: --manifest-path=tests/Cargo.toml directory -- --nocapture
run: cargo test -p tests directory -- --nocapture
- name: SMTP Tests
uses: actions-rs/cargo@v1
with:
command: test
args: --manifest-path=tests/Cargo.toml smtp -- --nocapture
run: cargo test -p tests smtp -- --nocapture
- name: IMAP Tests
uses: actions-rs/cargo@v1
with:
command: test
args: --manifest-path=tests/Cargo.toml imap -- --nocapture
run: cargo test -p tests imap -- --nocapture
- name: JMAP Tests
uses: actions-rs/cargo@v1
with:
command: test
args: --manifest-path=tests/Cargo.toml jmap -- --nocapture
run: cargo test -p tests jmap -- --nocapture