From d754dda292aed44bbd227247620c44fc35d69047 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 26 Nov 2020 02:13:12 +0100 Subject: [PATCH] Refactor frontend for GitHub Actions Signed-off-by: CrazyMax --- .github/workflows/build.yml | 77 ++++++++++++++- .../cmd/dockerfile-frontend/Dockerfile | 2 +- .../cmd/dockerfile-frontend/hack/release | 98 ++++++++++--------- hack/util | 2 - 4 files changed, 130 insertions(+), 49 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 880783c6..749127a1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,6 +17,7 @@ on: env: REPO_SLUG_ORIGIN: "moby/buildkit:latest" REPO_SLUG_TARGET: "moby/buildkit" + DF_REPO_SLUG_TARGET: "docker/dockerfile-upstream" PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le" CACHEKEY_INTEGRATION_TESTS: "integration-tests" CACHEKEY_BINARIES: "binaries" @@ -263,7 +264,7 @@ jobs: if [ "${{ github.event_name }}" = "schedule" ]; then TAG=nightly PUSH=push - elif [[ $GITHUB_REF == refs/tags/* ]]; then + elif [[ $GITHUB_REF == refs/tags/v* ]]; then TAG=${GITHUB_REF#refs/tags/} PUSH=push elif [[ $GITHUB_REF == refs/heads/* ]]; then @@ -385,3 +386,77 @@ jobs: draft: true files: ./release-out/* name: ${{ needs.release-base.outputs.tag }} + + frontend-base: + runs-on: ubuntu-latest + outputs: + typ: ${{ steps.prep.outputs.typ }} + tag: ${{ steps.prep.outputs.tag }} + push: ${{ steps.prep.outputs.push }} + steps: + - + name: Prepare + id: prep + run: | + TYP=master + TAG=mainline + PUSH=false + if [ "${{ github.event_name }}" = "schedule" ]; then + TYP=daily + TAG=_ + PUSH=push + elif [[ $GITHUB_REF == refs/tags/dockerfile/* ]]; then + TYP=tag + TAG=${GITHUB_REF#refs/tags/} + PUSH=push + elif [[ $GITHUB_REF == refs/heads/* ]]; then + PUSH=push + fi + PUSH=false # FIXME: Remove when migration to GitHub Actions completed + echo ::set-output name=typ::${TYP} + echo ::set-output name=tag::${TAG} + echo ::set-output name=push::${PUSH} + + frontend-image: + runs-on: ubuntu-latest + needs: [frontend-base, test] + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + - + name: Cache layers + uses: actions/cache@v2 + with: + path: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }} + key: ${{ runner.os }}-buildkit-frontend-${{ needs.frontend-base.outputs.typ }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildkit-frontend-${{ needs.frontend-base.outputs.typ }}- + - + name: Login to DockerHub + uses: docker/login-action@v1 + if: needs.frontend-base.outputs.push == 'push' + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build ${{ needs.frontend-base.outputs.typ }}/${{ needs.frontend-base.outputs.tag }} + run: | + ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" "${{ needs.frontend-base.outputs.tag }}" "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" + env: + CACHE_DIR: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }} + - + name: Build ${{ needs.frontend-base.outputs.typ }}/labs + if: needs.frontend-base.outputs.typ == 'master' + run: | + ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" labs "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" + env: + CACHE_DIR: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }} diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile index bb850183..914e1ac7 100644 --- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile +++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile @@ -1,4 +1,4 @@ -# syntax = docker/dockerfile:1.0-experimental +# syntax = docker/dockerfile:1.1-experimental FROM --platform=$BUILDPLATFORM tonistiigi/xx:golang@sha256:6f7d999551dd471b58f70716754290495690efa8421e0a1fcf18eb11d0c0a537 AS xgo diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release index ecabc7bf..b0e54be2 100755 --- a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release +++ b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release @@ -1,11 +1,10 @@ #!/usr/bin/env bash -: ${PLATFORMS=linux/amd64} -: ${CONTINUOUS_INTEGRATION=} -: ${DAILY_TARGETS=} +. $(dirname $0)/../../../../../hack/util +set -e -progressFlag="" -if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi +: ${PLATFORMS=linux/amd64} +: ${DAILY_TARGETS=} usage() { echo "$0 (master|tag|daily) (tag|channel) [push]" @@ -61,11 +60,20 @@ TAG=$2 REPO=$3 PUSH=$4 -pushFlag="" +pushFlag="push=false" if [ "$PUSH" = "push" ]; then pushFlag="push=true" fi +importCacheFlags="" +exportCacheFlags="" +if [ "$GITHUB_ACTIONS" = "true" ]; then + if [ -n "$cacheref" ]; then + importCacheFlags="--cache-from=type=local,src=$cacheref" + exportCacheFlags="--cache-to=type=local,dest=$cacheref" + fi +fi + case $TYP in "master") tagf=./frontend/dockerfile/release/$TAG/tags @@ -73,19 +81,20 @@ case $TYP in echo "invalid release $TAG" exit 1 fi + buildTags=$(cat $tagf) pushTag="master" if [ "$TAG" != "mainline" ]; then pushTag=${pushTag}-$TAG fi - set -x - buildctl build $progressFlag --frontend=dockerfile.v0 \ - --local context=. --local dockerfile=. \ - --opt filename=./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile \ - --opt platform=$PLATFORMS \ - --opt "build-arg:CHANNEL=$TAG" \ - --opt "build-arg:BUILDTAGS=$buildTags" \ - --output type=image,name=$REPO:$pushTag,$pushFlag + + buildxCmd build $importCacheFlags $exportCacheFlags \ + --platform "$PLATFORMS" \ + --build-arg "CHANNEL=$TAG" \ + --build-arg "BUILDTAGS=$buildTags" \ + --output "type=image,name=$REPO:$pushTag,$pushFlag" \ + --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ + $currentcontext ;; "tag") publishedNames="" @@ -97,56 +106,55 @@ case $TYP in fi buildTags=$(cat $tagf) - set -x - buildctl build $progressFlag --frontend=dockerfile.v0 \ - --local context=. --local dockerfile=. \ - --opt filename=./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile \ - --opt platform=$PLATFORMS \ - --opt "build-arg:CHANNEL=$TAG" \ - --opt "build-arg:BUILDTAGS=$buildTags" \ - --output type=image,\"name=$publishedNames\",$pushFlag + buildxCmd build $importCacheFlags $exportCacheFlags \ + --platform "$PLATFORMS" \ + --build-arg "CHANNEL=$TAG" \ + --build-arg "BUILDTAGS=$buildTags" \ + --output "type=image,name=\"$publishedNames\",$pushFlag" \ + --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ + $currentcontext ;; "daily") # if [ -z $DAILY_TARGETS ]; then # DAILY_TARGETS="" # fi - + for TAG in $DAILY_TARGETS; do - + tagf=./frontend/dockerfile/release/$TAG/tags if [ ! -f $tagf ]; then echo "invalid release $TAG" exit 1 fi buildTags=$(cat $tagf) - + # find the buildID of the last pushed image # returns a BuildID if rebuild needed - + tmp=$(mktemp -d -t buildid.XXXXXXXXXX) - set -x dt=$(date +%Y%m%d) - buildctl build $progressFlag --progress=plain --frontend=dockerfile.v0 \ - --local context=. --local dockerfile=. \ - --opt filename=./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile \ - --opt "build-arg:CHANNEL=$TAG" \ - --opt "build-arg:REPO=$REPO" \ - --opt "build-arg:DATE=$dt" \ - --opt "build-arg:BUILDTAGS=$buildTags" \ - --opt target=buildid\ - --output type=local,dest=$tmp - + buildxCmd build $importCacheFlags $exportCacheFlags \ + --platform "$PLATFORMS" \ + --target "buildid" \ + --build-arg "CHANNEL=$TAG" \ + --build-arg "BUILDTAGS=$buildTags" \ + --build-arg "REPO=$REPO" \ + --build-arg "DATE=$dt" \ + --output "type=local,dest=$tmp" \ + --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ + $currentcontext + if [ -f $tmp/buildid ]; then buildid=$(cat $tmp/buildid) echo "buildid: $buildid" - - buildctl build $progressFlag --frontend=dockerfile.v0 \ - --local context=. --local dockerfile=. \ - --opt filename=./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile \ - --opt platform=$PLATFORMS \ - --opt "build-arg:CHANNEL=$TAG" \ - --opt "build-arg:BUILDTAGS=$buildTags" \ - --output type=image,name=$REPO:$dt-$TAG,$pushFlag + + buildxCmd build $importCacheFlags $exportCacheFlags \ + --platform "$PLATFORMS" \ + --build-arg "CHANNEL=$TAG" \ + --build-arg "BUILDTAGS=$buildTags" \ + --output "type=image,name=$REPO:$dt-$TAG,$pushFlag" \ + --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ + $currentcontext rm $tmp/buildid fi rm -r $tmp diff --git a/hack/util b/hack/util index f659992f..f9c5e014 100755 --- a/hack/util +++ b/hack/util @@ -68,8 +68,6 @@ elif [ -n "$TRAVIS_BRANCH" ]; then fi currentcontext="." -currentcontextBuildctl="--local context=. --local dockerfile=." if [ -n "$currentref" ]; then currentcontext="--build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 $currentref" - currentcontextBuildctl="--opt context=$currentref --opt build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" fi