Merge pull request #1840 from crazy-max/ghactions-frontend

Refactor frontend for GitHub Actions
2020-11-25 18:06:45 -08:00 · 2020-11-25 18:06:45 -08:00 · 7437b4777c
parent 4ed96790e5 d754dda292
commit 7437b4777c
4 changed files with 130 additions and 49 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -17,6 +17,7 @@ on:
 env:
  REPO_SLUG_ORIGIN: "moby/buildkit:latest"
  REPO_SLUG_TARGET: "moby/buildkit"
+  DF_REPO_SLUG_TARGET: "docker/dockerfile-upstream"
  PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le"
  CACHEKEY_INTEGRATION_TESTS: "integration-tests"
  CACHEKEY_BINARIES: "binaries"
@ -263,7 +264,7 @@ jobs:
          if [ "${{ github.event_name }}" = "schedule" ]; then
            TAG=nightly
            PUSH=push
-          elif [[ $GITHUB_REF == refs/tags/* ]]; then
+          elif [[ $GITHUB_REF == refs/tags/v* ]]; then
            TAG=${GITHUB_REF#refs/tags/}
            PUSH=push
          elif [[ $GITHUB_REF == refs/heads/* ]]; then
@ -385,3 +386,77 @@ jobs:
          draft: true
          files: ./release-out/*
          name: ${{ needs.release-base.outputs.tag }}
+
+  frontend-base:
+    runs-on: ubuntu-latest
+    outputs:
+      typ: ${{ steps.prep.outputs.typ }}
+      tag: ${{ steps.prep.outputs.tag }}
+      push: ${{ steps.prep.outputs.push }}
+    steps:
+      -
+        name: Prepare
+        id: prep
+        run: |
+          TYP=master
+          TAG=mainline
+          PUSH=false
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            TYP=daily
+            TAG=_
+            PUSH=push
+          elif [[ $GITHUB_REF == refs/tags/dockerfile/* ]]; then
+            TYP=tag
+            TAG=${GITHUB_REF#refs/tags/}
+            PUSH=push
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            PUSH=push
+          fi
+          PUSH=false # FIXME: Remove when migration to GitHub Actions completed
+          echo ::set-output name=typ::${TYP}
+          echo ::set-output name=tag::${TAG}
+          echo ::set-output name=push::${PUSH}
+
+  frontend-image:
+    runs-on: ubuntu-latest
+    needs: [frontend-base, test]
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
+      -
+        name: Cache layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
+          key: ${{ runner.os }}-buildkit-frontend-${{ needs.frontend-base.outputs.typ }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildkit-frontend-${{ needs.frontend-base.outputs.typ }}-
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        if: needs.frontend-base.outputs.push == 'push'
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build ${{ needs.frontend-base.outputs.typ }}/${{ needs.frontend-base.outputs.tag }}
+        run: |
+          ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" "${{ needs.frontend-base.outputs.tag }}" "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}"
+        env:
+          CACHE_DIR: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
+      -
+        name: Build ${{ needs.frontend-base.outputs.typ }}/labs
+        if: needs.frontend-base.outputs.typ == 'master'
+        run: |
+          ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" labs "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}"
+        env:
+          CACHE_DIR: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
--- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile
+++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile
@ -1,4 +1,4 @@
-# syntax = docker/dockerfile:1.0-experimental
+# syntax = docker/dockerfile:1.1-experimental

 FROM --platform=$BUILDPLATFORM tonistiigi/xx:golang@sha256:6f7d999551dd471b58f70716754290495690efa8421e0a1fcf18eb11d0c0a537 AS xgo

--- a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release
+++ b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release
@ -1,11 +1,10 @@
 #!/usr/bin/env bash

-: ${PLATFORMS=linux/amd64}
-: ${CONTINUOUS_INTEGRATION=}
-: ${DAILY_TARGETS=}
+. $(dirname $0)/../../../../../hack/util
+set -e

-progressFlag=""
-if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
+: ${PLATFORMS=linux/amd64}
+: ${DAILY_TARGETS=}

 usage() {
  echo "$0 (master|tag|daily) (tag|channel) <repo> [push]"
@ -61,11 +60,20 @@ TAG=$2
 REPO=$3
 PUSH=$4

-pushFlag=""
+pushFlag="push=false"
 if [ "$PUSH" = "push" ]; then
  pushFlag="push=true"
 fi

+importCacheFlags=""
+exportCacheFlags=""
+if [ "$GITHUB_ACTIONS" = "true" ]; then
+  if [ -n "$cacheref" ]; then
+    importCacheFlags="--cache-from=type=local,src=$cacheref"
+    exportCacheFlags="--cache-to=type=local,dest=$cacheref"
+  fi
+fi
+
 case $TYP in
 "master")
  tagf=./frontend/dockerfile/release/$TAG/tags
@ -73,19 +81,20 @@ case $TYP in
    echo "invalid release $TAG"
    exit 1
  fi
+
  buildTags=$(cat $tagf)
  pushTag="master"
  if [ "$TAG" != "mainline" ]; then
    pushTag=${pushTag}-$TAG
  fi
-  set -x
-  buildctl build $progressFlag --frontend=dockerfile.v0 \
-    --local context=. --local dockerfile=. \
-    --opt filename=./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile \
-    --opt platform=$PLATFORMS \
-    --opt "build-arg:CHANNEL=$TAG" \
-    --opt "build-arg:BUILDTAGS=$buildTags" \
-    --output type=image,name=$REPO:$pushTag,$pushFlag
+
+  buildxCmd build $importCacheFlags $exportCacheFlags \
+    --platform "$PLATFORMS" \
+    --build-arg "CHANNEL=$TAG" \
+    --build-arg "BUILDTAGS=$buildTags" \
+    --output "type=image,name=$REPO:$pushTag,$pushFlag" \
+    --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
+    $currentcontext
  ;;
 "tag")
  publishedNames=""
@ -97,56 +106,55 @@ case $TYP in
  fi
  buildTags=$(cat $tagf)

-  set -x
-  buildctl build $progressFlag --frontend=dockerfile.v0 \
-    --local context=. --local dockerfile=. \
-    --opt filename=./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile \
-    --opt platform=$PLATFORMS \
-    --opt "build-arg:CHANNEL=$TAG" \
-    --opt "build-arg:BUILDTAGS=$buildTags" \
-    --output type=image,\"name=$publishedNames\",$pushFlag
+  buildxCmd build $importCacheFlags $exportCacheFlags \
+    --platform "$PLATFORMS" \
+    --build-arg "CHANNEL=$TAG" \
+    --build-arg "BUILDTAGS=$buildTags" \
+    --output "type=image,name=\"$publishedNames\",$pushFlag" \
+    --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
+    $currentcontext
  ;;
 "daily")
  # if [ -z $DAILY_TARGETS ]; then
  #   DAILY_TARGETS=""
  # fi
-  
+
  for TAG in $DAILY_TARGETS; do
-  
+
    tagf=./frontend/dockerfile/release/$TAG/tags
    if [ ! -f $tagf ]; then
      echo "invalid release $TAG"
      exit 1
    fi
    buildTags=$(cat $tagf)
-  
+
    # find the buildID of the last pushed image
    # returns a BuildID if rebuild needed
-  
+
    tmp=$(mktemp -d -t buildid.XXXXXXXXXX)
-    set -x
    dt=$(date +%Y%m%d)
-    buildctl build $progressFlag --progress=plain --frontend=dockerfile.v0 \
-      --local context=. --local dockerfile=. \
-      --opt filename=./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile \
-      --opt "build-arg:CHANNEL=$TAG" \
-      --opt "build-arg:REPO=$REPO" \
-      --opt "build-arg:DATE=$dt" \
-      --opt "build-arg:BUILDTAGS=$buildTags" \
-      --opt target=buildid\
-      --output type=local,dest=$tmp
-    
+    buildxCmd build $importCacheFlags $exportCacheFlags \
+      --platform "$PLATFORMS" \
+      --target "buildid" \
+      --build-arg "CHANNEL=$TAG" \
+      --build-arg "BUILDTAGS=$buildTags" \
+      --build-arg "REPO=$REPO" \
+      --build-arg "DATE=$dt" \
+      --output "type=local,dest=$tmp" \
+      --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
+      $currentcontext
+
    if [ -f $tmp/buildid ]; then
      buildid=$(cat $tmp/buildid)
      echo "buildid: $buildid"
-      
-      buildctl build $progressFlag --frontend=dockerfile.v0 \
-        --local context=. --local dockerfile=. \
-        --opt filename=./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile \
-        --opt platform=$PLATFORMS \
-        --opt "build-arg:CHANNEL=$TAG" \
-        --opt "build-arg:BUILDTAGS=$buildTags" \
-        --output type=image,name=$REPO:$dt-$TAG,$pushFlag
+
+      buildxCmd build $importCacheFlags $exportCacheFlags \
+        --platform "$PLATFORMS" \
+        --build-arg "CHANNEL=$TAG" \
+        --build-arg "BUILDTAGS=$buildTags" \
+        --output "type=image,name=$REPO:$dt-$TAG,$pushFlag" \
+        --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
+        $currentcontext
      rm $tmp/buildid
    fi
    rm -r $tmp
--- a/hack/util
+++ b/hack/util
@ -68,8 +68,6 @@ elif [ -n "$TRAVIS_BRANCH" ]; then
 fi

 currentcontext="."
-currentcontextBuildctl="--local context=. --local dockerfile=."
 if [ -n "$currentref" ]; then
  currentcontext="--build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 $currentref"
-  currentcontextBuildctl="--opt context=$currentref --opt build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR=1"
 fi