Merge pull request #1835 from crazy-max/ghactions-release

Refactor release for GitHub Actions
v0.8
Tõnis Tiigi 2020-11-25 15:11:06 -08:00 committed by GitHub
commit a632fc5409
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 233 additions and 223 deletions

View File

@ -1,6 +1,8 @@
name: build
on:
schedule:
- cron: '0 10 * * *' # everyday at 10am
workflow_dispatch:
push:
branches:
@ -14,9 +16,11 @@ on:
env:
REPO_SLUG_ORIGIN: "moby/buildkit:latest"
REPO_SLUG_TARGET: "moby/buildkit"
PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le"
CACHEKEY_INTEGRATION_TESTS: "integration-tests"
CACHEKEY_BINARIES: "binaries"
CACHEKEY_CROSS: "cross"
jobs:
base:
@ -104,6 +108,14 @@ jobs:
key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_INTEGRATION_TESTS }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildkit-${{ env.CACHEKEY_INTEGRATION_TESTS }}-
-
name: Cache ${{ env.CACHEKEY_BINARIES }}
uses: actions/cache@v2
with:
path: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
@ -116,21 +128,19 @@ jobs:
name: Test ${{ matrix.pkg }} ${{ matrix.typ }}
run: |
./hack/test ${{ matrix.typ }}
mv ./coverage/coverage.txt ./coverage/coverage-${{ github.job }}-$(echo "${{ matrix.coverage-pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.typ }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]').txt
env:
TEST_COVERAGE: 1
TESTPKGS: ${{ matrix.pkg }}
TESTFLAGS: -v --parallel=6 --timeout=20m
SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }}
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }} /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
-
name: Upload coverage
uses: codecov/codecov-action@v1
name: Upload coverage file
uses: actions/upload-artifact@v2
with:
file: ./coverage/coverage.txt
-
name: Dump context
if: always()
uses: crazy-max/ghaction-dump-context@v1
name: coverage
path: ./coverage
test-os:
runs-on: ${{ matrix.os }}
@ -168,17 +178,45 @@ jobs:
TMPDIR: ${{ runner.temp }}
SKIP_INTEGRATION_TESTS: 1
run: |
go test -coverprofile=coverage.txt -covermode=atomic ./...
mkdir -p ./coverage
go test -coverprofile=./coverage/coverage-${{ github.job }}-${{ matrix.os }}.txt -covermode=atomic ./...
shell: bash
-
name: Upload coverage
name: Upload coverage file
uses: actions/upload-artifact@v2
with:
name: coverage
path: ./coverage
upload-coverage:
runs-on: ubuntu-latest
needs: [test, test-os]
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Download coverage files
uses: actions/download-artifact@v2
with:
name: coverage
path: ./coverage
-
name: List coverage files
uses: actions/github-script@v3
id: files
with:
result-encoding: string
script: |
return require('fs').readdirSync('./coverage', {withFileTypes: true})
.filter(item => !item.isDirectory())
.map(item => `./coverage/${item.name}`)
.join(',');
-
name: Send to Codecov
uses: codecov/codecov-action@v1
with:
file: coverage.txt
-
name: Dump context
if: always()
uses: crazy-max/ghaction-dump-context@v1
files: ${{ steps.files.outputs.result }}
cross:
runs-on: ubuntu-latest
@ -187,13 +225,13 @@ jobs:
name: Checkout
uses: actions/checkout@v2
-
name: Cache cross
name: Cache ${{ env.CACHEKEY_CROSS }}
uses: actions/cache@v2
with:
path: /tmp/.buildkit-cache/cross
key: ${{ runner.os }}-buildkit-cross-${{ github.sha }}
path: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildkit-cross-
${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
@ -209,15 +247,88 @@ jobs:
env:
PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,windows/amd64
RUNC_PLATFORMS: ${{ env.PLATFORMS }}
CACHE_DIR: /tmp/.buildkit-cache/cross
-
name: Dump context
if: always()
uses: crazy-max/ghaction-dump-context@v1
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
buildkit-buildkitd:
release-base:
runs-on: ubuntu-latest
needs: [base]
outputs:
tag: ${{ steps.prep.outputs.tag }}
push: ${{ steps.prep.outputs.push }}
steps:
- name: Prepare
id: prep
run: |
TAG=local
PUSH=false
if [ "${{ github.event_name }}" = "schedule" ]; then
TAG=nightly
PUSH=push
elif [[ $GITHUB_REF == refs/tags/* ]]; then
TAG=${GITHUB_REF#refs/tags/}
PUSH=push
elif [[ $GITHUB_REF == refs/heads/* ]]; then
TAG=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
PUSH=push
fi
PUSH=false # FIXME: Remove when migration to GitHub Actions completed
echo ::set-output name=tag::${TAG}
echo ::set-output name=push::${PUSH}
image:
runs-on: ubuntu-latest
needs: [release-base, test, cross]
strategy:
fail-fast: false
matrix:
include:
- target-stage: ''
- target-stage: rootless
steps:
-
name: Checkout${{ env.CACHEKEY_BINARIES }}
uses: actions/checkout@v2
-
name: Cache image${{ matrix.target-stage }}
uses: actions/cache@v2
with:
path: /tmp/.buildkit-cache/image${{ matrix.target-stage }}
key: ${{ runner.os }}-buildkit-image${{ matrix.target-stage }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildkit-image${{ matrix.target-stage }}-
-
name: Cache ${{ env.CACHEKEY_CROSS }}
uses: actions/cache@v2
with:
path: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
with:
driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-
name: Login to DockerHub
if: needs.release-base.outputs.push == 'push'
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Build ${{ needs.release-base.outputs.tag }}
run: |
./hack/images "${{ needs.release-base.outputs.tag }}" "$REPO_SLUG_TARGET" "${{ needs.release-base.outputs.push }}"
env:
TARGET: ${{ matrix.target-stage }}
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }} /tmp/.buildkit-cache/image${{ matrix.target-stage }}
binaries:
runs-on: ubuntu-latest
needs: [release-base, test, cross]
steps:
-
name: Checkout
@ -230,6 +341,14 @@ jobs:
key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-
-
name: Cache ${{ env.CACHEKEY_CROSS }}
uses: actions/cache@v2
with:
path: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildkit-${{ env.CACHEKEY_CROSS }}-
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
@ -239,13 +358,30 @@ jobs:
with:
driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-
name: Build
name: Build ${{ needs.release-base.outputs.tag }}
run: |
./hack/buildkit-buildkitd
./hack/release-tar "${{ needs.release-base.outputs.tag }}" release-out
env:
PLATFORMS: windows/amd64
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,windows/amd64
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }} /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
-
name: Dump context
if: always()
uses: crazy-max/ghaction-dump-context@v1
name: Move artifacts
run: |
mv ./release-out/**/* ./release-out/
-
name: Upload artifacts
uses: actions/upload-artifact@v2
with:
name: buildkit
path: ./release-out/*
if-no-files-found: error
-
name: GitHub Release
if: needs.release-base.outputs.push == 'push'
uses: softprops/action-gh-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
draft: true
files: ./release-out/*
name: ${{ needs.release-base.outputs.tag }}

View File

@ -1,27 +0,0 @@
name: release
on:
workflow_dispatch:
workflow_run:
workflows:
- 'build'
branches:
- 'master'
types:
- 'completed'
env:
REPO_SLUG_ORIGIN: "moby/buildkit:latest"
PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le"
jobs:
image:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Dump context
if: always()
uses: crazy-max/ghaction-dump-context@v1

View File

@ -55,11 +55,11 @@ jobs:
name: "Dockerfile integration tests"
- script: TESTPKGS=./frontend/dockerfile ./hack/test dockerfile
name: "External Dockerfile tests"
- script:
- RUNC_PLATFORMS=$PLATFORMS PLATFORMS="${PLATFORMS},darwin/amd64,windows/amd64" ./hack/cross
- PLATFORMS="windows/amd64" ./hack/buildkit-buildkitd
- script: RUNC_PLATFORMS=$PLATFORMS PLATFORMS="${PLATFORMS},darwin/amd64,windows/amd64" ./hack/cross
name: "Cross"
- script: ./hack/images local $REPO_SLUG_TARGET
- script:
- ./hack/images local $REPO_SLUG_TARGET
- TARGET=rootless ./hack/images local $REPO_SLUG_TARGET
name: "Build image"
if: type == cron
- stage: deploy
@ -68,13 +68,13 @@ jobs:
if: type != pull_request
deploy:
- provider: script
script: ./hack/images master $REPO_SLUG_TARGET push
script: ./hack/images master $REPO_SLUG_TARGET push && TARGET=rootless ./hack/images master $REPO_SLUG_TARGET push
on:
repo: moby/buildkit
branch: master
condition: $TRAVIS_EVENT_TYPE != "cron"
- provider: script
script: ./hack/images $TRAVIS_TAG $REPO_SLUG_TARGET push && PLATFORMS="${PLATFORMS},darwin/amd64,windows/amd64" ./hack/release-tar $TRAVIS_TAG release-out
script: ./hack/images $TRAVIS_TAG $REPO_SLUG_TARGET push && TARGET=rootless ./hack/images $TRAVIS_TAG $REPO_SLUG_TARGET push && PLATFORMS="${PLATFORMS},darwin/amd64,windows/amd64" ./hack/release-tar $TRAVIS_TAG release-out
on:
repo: moby/buildkit
tags: true

View File

@ -1,60 +1,22 @@
#!/usr/bin/env bash
. $(dirname $0)/util
set -eu
: ${TARGETPLATFORM=}
: ${CONTINUOUS_INTEGRATION=}
: ${BUILDKITD_TAGS=}
set -ex
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
platformFlag="--platform local"
if [ -n "$TARGETPLATFORM" ]; then
platformFlag="--platform $TARGETPLATFORM"
fi
buildkitdTagsFlags=""
if [ -n "$BUILDKITD_TAGS" ]; then
buildkitdTagsFlags="--opt build-arg:BUILDKITD_TAGS=\"$BUILDKITD_TAGS\""
buildkitdTagsFlags="--build-arg=BUILDKITD_TAGS=\"$BUILDKITD_TAGS\""
fi
binariesDocker() {
mkdir -p bin/tmp
export DOCKER_BUILDKIT=1
iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
target=$(echo "$TARGETPLATFORM" | cut -d"/" -f1)
if [ -z "$target" ]; then
target="linux"
fi
docker build $platformFlag --build-arg BUILDKITD_TAGS="${BUILDKITD_TAGS}" --target binaries-$target --iidfile $iidfile --force-rm .
iid=$(cat $iidfile)
containerID=$(docker create $iid copy)
docker cp $containerID:/ bin/tmp
mv bin/tmp/build* bin/
rm -rf bin/tmp
docker rm $containerID
docker rmi -f $iid
rm -f $iidfile
}
binaries() {
platformFlag=""
if [ ! -z "$TARGETPLATFORM" ]; then
platformFlag="--opt platform=$TARGETPLATFORM"
fi
buildctl build $progressFlag --frontend=dockerfile.v0 \
--local context=. --local dockerfile=. \
$buildkitdTagsFlags --opt target=binaries $platformFlag \
--output type=local,dest=./bin/
}
case $buildmode in
"buildkit")
binaries
;;
"docker-buildkit")
binariesDocker
;;
*)
echo "Unsupported build mode: $buildmode" >&2
exit 1
;;
esac
buildxCmd build $platformFlag $buildkitdTagsFlags \
--target "binaries" \
--output "type=local,dest=./bin/" \
.

View File

@ -35,7 +35,6 @@ if [ "$TRAVIS" = "true" ]; then
exportFlags="$exportFlags:integration-tests"
fi
elif [ "$GITHUB_ACTIONS" = "true" ]; then
currentcontext="."
if [ -n "$cacheref" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheref"
exportCacheFlags="--cache-to=type=local,dest=$cacheref"

View File

@ -1,21 +0,0 @@
#!/usr/bin/env bash
. $(dirname $0)/util
set -e
: ${PLATFORMS=windows/amd64}
importCacheFlags=""
if [ -n "$cacheref" ]; then
if [ "$cachetype" = "local" ]; then
currentcontext="."
importCacheFlags="--cache-from=type=local,src=$cacheref"
else
importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries"
fi
fi
buildxCmd build $importCacheFlags \
--target "buildkit-buildkitd" \
--platform "$PLATFORMS" \
$currentcontext

View File

@ -14,7 +14,6 @@ if [ -n "$MASTERCACHE_PASSWORD" ]; then
exportCacheFlags="--cache-to=type=inline"
outputFlags="--output=type=image,push=true,name=cicache.buildk.it/moby/buildkit/master:binaries-cross-helper"
elif [ "$GITHUB_ACTIONS" = "true" ]; then
currentcontext="."
if [ -n "$cacheref" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheref"
exportCacheFlags="--cache-to=type=local,dest=$cacheref"

View File

@ -1,18 +1,14 @@
#!/usr/bin/env bash
. $(dirname $0)/util
TAG=$1
REPO=$2
PUSH=$3
. $(dirname $0)/util
set -eu -o pipefail
: ${PLATFORMS=linux/amd64}
: ${CONTINUOUS_INTEGRATION=}
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
: ${TARGET=}
versionTag=$(git describe --always --tags --match "v[0-9]*")
@ -29,76 +25,39 @@ if [ -z "$TAG" ] || [ -z "$REPO" ]; then
usage
fi
imageDocker() {
if echo $PLATFORMS | grep , >/dev/null; then
echo "PREFER_BUILDCTL=1 needs to be set to build images for $PLATFORMS"
exit 1
fi
set -x
DOCKER_BUILDKIT=1 docker build $progressFlag --platform=$PLATFORMS -t $REPO:$TAG .
DOCKER_BUILDKIT=1 docker build $progressFlag --platform=$PLATFORMS -t $REPO:$TAG-rootless --target rootless .
set +x
pushFlag="push=false"
if [ "$PUSH" = "push" ]; then
pushFlag="push=true"
fi
if [ "$PUSH" = "push" ]; then
set -x
docker push $REPO:$TAG
docker push $REPO:$TAG-rootless
set +x
fi
if [[ "$versionTag" == "$TAG" ]]; then
set -x
docker tag $REPO:$TAG $REPO:latest
docker tag $REPO:$TAG-rootless $REPO:rootless
set +x
if [ "$PUSH" = "push" ]; then
set -x
docker push $REPO:latest
docker push $REPO:rootless
set +x
fi
fi
}
targetFlag=""
if [ -n "$TARGET" ]; then
targetFlag="--target=$TARGET"
fi
image() {
pushFlag="push=false"
if [ "$PUSH" = "push" ]; then
pushFlag="push=true"
fi
tagLatest=""
tagLatestRootless=""
if [[ "$versionTag" == "$TAG" ]]; then
tagLatest=""
if [[ "$versionTag" == "$TAG" ]]; then
if [ -n "$TARGET" ]; then
tagLatest=",$REPO:$TARGET"
else
tagLatest=",$REPO:latest"
tagLatestRootless=",$REPO:rootless"
fi
fi
set -x
importCacheFlags="--cache-from=type=registry,ref=$REPO:$TAG$tagLatest "
if [[ -n "$cacheref" ]] && [[ "$cachetype" = "local" ]]; then
for ref in $cacheref; do
importCacheFlags="$importCacheFlags--cache-from=type=local,src=$ref "
done
fi
buildctl build $progressFlag --frontend=dockerfile.v0 \
--local context=. --local dockerfile=. \
--opt platform=$PLATFORMS \
--export-cache type=inline \
--import-cache type=registry,ref=$REPO:$TAG$tagLatest \
--output type=image,\"name=$REPO:$TAG$tagLatest\",$pushFlag
exportCacheFlags=""
if [ "$PUSH" = "push" ]; then
exportCacheFlags="--cache-to=type=inline "
fi
buildctl build $progressFlag --frontend=dockerfile.v0 \
--local context=. --local dockerfile=. \
--opt target=rootless \
--opt platform=$PLATFORMS \
--export-cache type=inline \
--import-cache type=registry,ref=$REPO:$TAG-rootless$tagLatestRootless \
--output type=image,\"name=$REPO:$TAG-rootless$tagLatestRootless\",$pushFlag
}
case $buildmode in
"buildkit")
image
;;
"docker-buildkit")
imageDocker
;;
*)
echo "Unsupported build mode: $buildmode" >&2
exit 1
;;
esac
buildxCmd build $targetFlag $importCacheFlags $exportCacheFlags \
--platform "$PLATFORMS" \
--tag "$REPO:$TAG$tagLatest" \
--output "type=image,$pushFlag" \
$currentref

View File

@ -3,13 +3,10 @@
TAG=$1
OUT=$2
. $(dirname $0)/util
set -eu -o pipefail
: ${PLATFORMS=linux/amd64}
: ${CONTINUOUS_INTEGRATION=}
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
usage() {
echo "usage: ./hack/release-tar <tag> <out>"
@ -20,11 +17,15 @@ if [ -z "$TAG" ] || [ -z "$OUT" ]; then
usage
fi
set -x
importCacheFlags=""
if [[ -n "$cacheref" ]] && [[ "$cachetype" = "local" ]]; then
for ref in $cacheref; do
importCacheFlags="$importCacheFlags--cache-from=type=local,src=$ref "
done
fi
buildctl build $progressFlag --frontend=dockerfile.v0 \
--local context=. --local dockerfile=. \
--opt target=release \
--opt platform=$PLATFORMS \
--exporter local \
--exporter-opt output=$OUT
buildxCmd build $importCacheFlags \
--target release \
--platform "$PLATFORMS" \
--output "type=local,dest=$OUT" \
$currentcontext

View File

@ -26,7 +26,9 @@ fi
importCacheFlags=""
if [ -n "$cacheref" ]; then
if [ "$cachetype" = "local" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheref "
for ref in $cacheref; do
importCacheFlags="$importCacheFlags--cache-from=type=local,src=$ref "
done
else
importCacheFlags="--cache-from=type=registry,ref=$cacheref:integration-tests "
fi
@ -102,7 +104,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
--build-arg "BUILDTAGS=$buildtags" \
--file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
--output "type=oci,dest=$tarout" \
.
$currentcontext
if [ -s $tarout ]; then
if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$GITHUB_ACTIONS" = "true" ]; then