Merge pull request #1831 from crazy-max/ghactions-test

Refactor test for GitHub Actions

.github/workflows/test.yml

@@ -0,0 +1,164 @@
+name: test
+
+on:
+  push:
+    branches:
+      - "ghactions-test"  # remove when merged
+      - 'master'
+    tags:
+      - 'v*'
+      - 'dockerfile/*'
+  pull_request:
+    branches:
+      - 'master'
+
+env:
+  REPO_SLUG_ORIGIN: "moby/buildkit:latest"
+
+jobs:
+  test-base:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Cache integration-tests
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildkit-cache/integration-tests
+          key: ${{ runner.os }}-buildkit-integration-tests-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildkit-integration-tests-
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
+      -
+        name: Build integration-tests
+        run: |
+          ./hack/build_ci_first_pass integration-tests
+        env:
+          CACHE_DIR: /tmp/.buildkit-cache/integration-tests
+      -
+        name: Dump context
+        if: always()
+        uses: crazy-max/ghaction-dump-context@v1
+
+  test:
+    runs-on: ubuntu-latest
+    needs: [test-base]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          -
+            pkg: ./client
+            typ: integration
+          -
+            pkg: ./cmd/buildctl ./worker/containerd
+            typ: integration
+          -
+            pkg: ''
+            skip-integration-tests: 1
+            typ: integration gateway
+          -
+            pkg: ./frontend
+            typ: ''
+          -
+            pkg: ./frontend/dockerfile
+            typ: ''
+          -
+            pkg: ./frontend/dockerfile
+            typ: dockerfile
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Cache integration-tests
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildkit-cache/integration-tests
+          key: ${{ runner.os }}-buildkit-integration-tests-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildkit-integration-tests-
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
+      -
+        name: Test ${{ matrix.pkg }} ${{ matrix.typ }}
+        run: |
+          ./hack/test ${{ matrix.typ }}
+        env:
+          TEST_COVERAGE: 1
+          TESTPKGS: ${{ matrix.pkg }}
+          TESTFLAGS: -v --parallel=6 --timeout=20m
+          SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }}
+          CACHE_DIR: /tmp/.buildkit-cache/integration-tests
+      -
+        name: Upload coverage
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./coverage/coverage.txt
+      -
+        name: Dump context
+        if: always()
+        uses: crazy-max/ghaction-dump-context@v1
+
+  os:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          # - ubuntu-latest
+          # - macOS-latest
+          - windows-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.13
+      -
+        name: Cache Go modules
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      -
+        name: Go mod
+        run: |
+          go mod download
+      -
+        name: Test
+        env:
+          TMPDIR: ${{ runner.temp }}
+          SKIP_INTEGRATION_TESTS: 1
+        run: |
+          go test -coverprofile=coverage.txt -covermode=atomic ./...
+        shell: bash
+      -
+        name: Upload coverage
+        uses: codecov/codecov-action@v1
+        with:
+          file: coverage.txt
+      -
+        name: Dump context
+        if: always()
+        uses: crazy-max/ghaction-dump-context@v1

.github/workflows/windows.yaml

@@ -1,43 +0,0 @@
-# This workflow is for builds on Windows. We cannot use the existing
-# Travis-CI workflow as that depends on already having functioning
-# BuildKit to run the v1.1-experimental Dockerfile that builds BuildKit.
-name: Windows
-
-on: [push, pull_request]
-
-jobs:
-  build:
-    name: Build BuildKit
-    runs-on: windows-2019
-    steps:
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.13
-      - name: Checkout
-        uses: actions/checkout@v2
-        # Is this needed? Didn't Go Modules fix this?
-        with:
-          path: src/github.com/moby/buildkit
-      - name: Compile
-        run: go install -mod=vendor ./cmd/...
-        working-directory: src/github.com/moby/buildkit
-
-  unit_tests:
-    name: Unit Tests
-    runs-on: windows-2019
-    steps:
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.13
-      - name: Checkout
-        uses: actions/checkout@v2
-        # Is this needed? Didn't Go Modules fix this?
-        with:
-          path: src/github.com/moby/buildkit
-      - name: Unit test
-        env:
-          SKIP_INTEGRATION_TESTS: 1
-        run: go test -mod=vendor -v ./...
-        working-directory: src/github.com/moby/buildkit

.gitignore

@@ -1,4 +1,5 @@
 bin
+coverage
 release-out
 .certs
 .tmp

.travis.yml

@@ -19,7 +19,8 @@ before_install:
   - docker info
 
 install:
-  - docker run --rm --privileged linuxkit/binfmt:v0.8
+  - docker buildx create --name builder --driver docker-container --driver-opt image=$REPO_SLUG_ORIGIN --use
+  - docker run --rm --privileged tonistiigi/binfmt:latest --install all
   - docker run --name buildkit -d --privileged -p 1234:1234 $REPO_SLUG_ORIGIN --debug --addr tcp://0.0.0.0:1234 --oci-worker-gc=false
   - sudo docker cp buildkit:/usr/bin/buildctl /usr/bin/
   - export BUILDKIT_HOST=tcp://0.0.0.0:1234
@@ -31,7 +32,10 @@ jobs:
   include:
     - stage: building
       name: "Build"
-      script: ./hack/login_ci_cache && ./hack/build_ci_first_pass
+      script:
+        - ./hack/login_ci_cache
+        - ./hack/build_ci_first_pass binaries
+        - ./hack/build_ci_first_pass integration-tests
     - stage: testing
       name: "Client integration tests"
       script:
@@ -39,15 +43,15 @@ jobs:
         - TESTPKGS=./cmd/buildctl ./hack/test integration
         - TESTPKGS=./worker/containerd ./hack/test integration
     - script:
-       - ./hack/lint
+        - ./hack/lint
-       - SKIP_INTEGRATION_TESTS=1 ./hack/test integration gateway
+        - SKIP_INTEGRATION_TESTS=1 ./hack/test integration gateway
-       - ./hack/validate-vendor
+        - ./hack/validate-vendor
-       - ./hack/validate-generated-files
+        - ./hack/validate-generated-files
-       - ./hack/validate-shfmt
+        - ./hack/validate-shfmt
-       - TESTPKGS=./frontend ./hack/test
+        - TESTPKGS=./frontend ./hack/test
       name: "Unit Tests & Lint & Vendor & Proto"
     - script:
-       - TESTPKGS=./frontend/dockerfile TESTFLAGS='-v --parallel=5 --timeout=30m' ./hack/test
+        - TESTPKGS=./frontend/dockerfile TESTFLAGS='-v --parallel=5 --timeout=30m' ./hack/test
       name: "Dockerfile integration tests"
     - script: TESTPKGS=./frontend/dockerfile ./hack/test dockerfile
       name: "External Dockerfile tests"
@@ -80,9 +84,9 @@ jobs:
           skip_cleanup: true
           file_glob: true
           on:
-           repo: moby/buildkit
+            repo: moby/buildkit
-           tags: true
+            tags: true
-           condition: $TRAVIS_TAG =~ ^v[0-9]
+            condition: $TRAVIS_TAG =~ ^v[0-9]
         - provider: script
           script: ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release master mainline $DF_REPO_SLUG_TARGET push
           on:
@@ -108,7 +112,5 @@ jobs:
             branch: master
             condition: $TRAVIS_EVENT_TYPE == "cron"
 
-
 before_deploy:
   - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-

README.md

@@ -5,6 +5,7 @@
 [![GoDoc](https://godoc.org/github.com/moby/buildkit?status.svg)](https://godoc.org/github.com/moby/buildkit/client/llb)
 [![Build Status](https://travis-ci.com/moby/buildkit.svg?branch=master)](https://travis-ci.com/moby/buildkit)
 [![Go Report Card](https://goreportcard.com/badge/github.com/moby/buildkit)](https://goreportcard.com/report/github.com/moby/buildkit)
+[![codecov](https://codecov.io/gh/moby/buildkit/branch/master/graph/badge.svg)](https://codecov.io/gh/moby/buildkit)
 
 BuildKit is a toolkit for converting source code to build artifacts in an efficient, expressive and repeatable manner.
 

hack/build_ci_first_pass

@@ -1,35 +1,60 @@
 #!/usr/bin/env bash
 
-. $(dirname $0)/util
+TYP=$1
 
+. $(dirname $0)/util
 set -e
 
-if [ -z "$TRAVIS_COMMIT" ]; then
+usage() {
-  echo "TRAVIS_COMMIT_REQURED"
+  echo "usage: ./hack/build_ci_first_pass <typ>"
-  exit 0
+  exit 1
+}
+
+if [ -z "$TYP" ]; then
+  usage
 fi
 
-progressFlag=""
+importCacheFlags=""
-if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
-
 exportCacheFlags=""
 exportFlags=""
 
-if [ -f /tmp/buildkit-ci-cache-repo ]; then
+if [ "$TRAVIS" = "true" ]; then
-  exportCacheFlags="--export-cache=type=inline"
+  if [ -z "$TRAVIS_COMMIT" ]; then
-  exportFlags="--output=type=image,push=true,name=$(cat /tmp/buildkit-ci-cache-repo)"
+    echo "Travis commit required"
+    exit 0
+  fi
+  if [ -f /tmp/buildkit-ci-cache-repo ]; then
+    exportCacheFlags="--cache-to=type=inline"
+    exportFlags="--output=type=image,push=true,name=$(cat /tmp/buildkit-ci-cache-repo)"
+  fi
+  if [ "$TYP" = "binaries" ]; then
+    importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries --cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests"
+    exportFlags="$exportFlags:binaries"
+  elif [ "$TYP" = "integration-tests" ]; then
+    importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests"
+    exportFlags="$exportFlags:integration-tests"
+  fi
+elif [ "$GITHUB_ACTIONS" = "true" ]; then
+  context="."
+  if [ -n "$cacheref" ]; then
+    importCacheFlags="--cache-from=type=local,src=$cacheref"
+    exportCacheFlags="--cache-to=type=local,dest=$cacheref"
+  fi
 fi
 
-set -x
+case $TYP in
-buildctl build $progressFlag --frontend=dockerfile.v0 \
+  "binaries")
-  $currentcontextBuildctl \
+    buildxCmd build $importCacheFlags $exportCacheFlags $exportFlags \
-  --opt target=binaries \
+      --target "binaries" \
-  --import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries \
+      $currentcontext
-  --import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests \
+    ;;
-  $exportCacheFlags $exportFlags:binaries
+  "integration-tests")
-
+    buildxCmd build $importCacheFlags $exportCacheFlags $exportFlags \
-buildctl build $progressFlag --frontend=dockerfile.v0 \
+      --target "integration-tests-base" \
-  $currentcontextBuildctl \
+      $currentcontext
-  --opt target=integration-tests-base \
+    ;;
-  --import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests \
+  *)
-  $exportCacheFlags $exportFlags:integration-tests
+    echo >&2 "Unknown type $TYP"
+    exit 1
+    ;;
+esac

hack/test

@@ -8,8 +8,8 @@ set -eu -o pipefail
 : ${TEST_DOCKERFILE=}
 : ${TEST_DOCKERD=}
 : ${TEST_DOCKERD_BINARY=$(which dockerd)}
+: ${TEST_COVERAGE=}
 : ${DOCKERFILE_RELEASES=}
-: ${CONTINUOUS_INTEGRATION=}
 : ${BUILDKIT_REGISTRY_MIRROR_DIR=}
 
 if [ "$TEST_DOCKERD" == "1" ] && ! file $TEST_DOCKERD_BINARY | grep "statically linked" >/dev/null; then
@@ -19,12 +19,13 @@ fi
 
 importCacheFlags=""
 if [ -n "$cacheref" ]; then
-  importCacheFlags="--import-cache=type=registry,ref=$cacheref:integration-tests"
+  if [ "$cachetype" = "local" ]; then
+    importCacheFlags="--cache-from=type=local,src=$cacheref"
+  else
+    importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests --cache-from=type=registry,ref=$cacheref:integration-tests"
+  fi
 fi
 
-progressFlag=""
-if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
-
 if [ "$#" == 0 ]; then TEST_INTEGRATION=1; fi
 
 while test $# -gt 0; do
@@ -47,33 +48,25 @@ done
 
 iid="buildkit-tests"
 iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
-set -x
 
-case $buildmode in
+coverageVol=""
-  "buildkit")
+coverageFlags=""
-    tmpfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
+if [ "$TEST_COVERAGE" = "1" ]; then
-    buildctl build $progressFlag --frontend=dockerfile.v0 $currentcontextBuildctl \
+  covdir="$(pwd)/coverage"
-      --opt target=integration-tests \
+  mkdir -p "$covdir"
-      --output type=docker,name=$iid,dest=$tmpfile \
+  coverageVol="-v $covdir:/coverage"
-      --import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests $importCacheFlags
+  coverageFlags="-coverprofile=/coverage/coverage.txt -covermode=atomic"
-    docker load -i $tmpfile
+fi
-    rm $tmpfile
+
-    ;;
+buildxCmd build $importCacheFlags \
-  "docker-buildkit")
+  --target "integration-tests" \
-    export DOCKER_BUILDKIT=1
+  --output "type=docker,name=$iid" \
-    docker build --iidfile $iidfile --target integration-tests --force-rm .
+  $currentcontext
-    iid=$(cat $iidfile)
-    ;;
-  *)
-    echo "Unsupported build mode: $buildmode" >&2
-    exit 1
-    ;;
-esac
 
 cacheVolume=$(docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod alpine)
 
 if [ "$TEST_INTEGRATION" == 1 ]; then
-  cid=$(docker create --rm -v /tmp --volumes-from=$cacheVolume -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test ${TESTFLAGS:--v} ${TESTPKGS:-./...})
+  cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...})
   if [ "$TEST_DOCKERD" = "1" ]; then
     docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/
   fi
@@ -98,26 +91,16 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
   for release in $DOCKERFILE_RELEASES; do
     buildtags=$(cat ./frontend/dockerfile/release/$release/tags)
     tarout=$(mktemp -t dockerfile-frontend.XXXXXXXXXX)
-    case $buildmode in
+
-      "buildkit")
+    buildxCmd build $importCacheFlags \
-        buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \
+      --build-arg "BUILDTAGS=$buildtags" \
-          --opt filename=./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile \
+      --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
-          --opt build-arg:BUILDTAGS="$buildtags" \
+      --output "type=oci,dest=$tarout" \
-          --output type=oci,dest=$tarout
+      .
-        ;;
-      "docker-buildkit")
-        dfiidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
-        docker build --iidfile=$dfiidfile -f ./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile --build-arg BUILDTAGS="$buildtags" .
-        dfiid=$(cat $dfiidfile)
-        docker save -o $tarout $dfiid
-        docker rmi $dfiid
-        rm $dfiidfile
-        ;;
-    esac
 
     if [ -s $tarout ]; then
-      if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
+      if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$GITHUB_ACTIONS" = "true" ]; then
-        cid=$(docker create -v /tmp --rm --privileged --volumes-from=$cacheVolume -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM=1 -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile)
+        cid=$(docker create -v /tmp $coverageVol --rm --privileged --volumes-from=$cacheVolume -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM=1 -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test $coverageFlags --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile)
         docker cp $tarout $cid:/$release.tar
         if [ "$TEST_DOCKERD" = "1" ]; then
           docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/
@@ -130,10 +113,3 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
 fi
 
 docker rm -v $cacheVolume
-
-case $buildmode in
-  "docker-buildkit")
-    rm "$iidfile"
-    docker rmi $iid
-    ;;
-esac

hack/util

@@ -57,7 +57,7 @@ cachetype=""
 cacheref=""
 currentref=""
 if [ "$GITHUB_ACTIONS" = "true" ]; then
-  currentref="git://github.com/moby/buildkit#$GITHUB_REF"
+  currentref="git://github.com/$GITHUB_REPOSITORY#$GITHUB_REF"
   cachetype="local"
   cacheref="$CACHE_DIR"
 elif [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then