Merge pull request #1831 from crazy-max/ghactions-test

Refactor test for GitHub Actions
2020-11-22 10:49:35 -08:00 · 2020-11-22 10:49:35 -08:00 · 2a25df5e5b
parent f30fab2532 135782149d
commit 2a25df5e5b
8 changed files with 260 additions and 134 deletions
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -0,0 +1,164 @@
+name: test
+
+on:
+  push:
+    branches:
+      - "ghactions-test"  # remove when merged
+      - 'master'
+    tags:
+      - 'v*'
+      - 'dockerfile/*'
+  pull_request:
+    branches:
+      - 'master'
+
+env:
+  REPO_SLUG_ORIGIN: "moby/buildkit:latest"
+
+jobs:
+  test-base:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Cache integration-tests
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildkit-cache/integration-tests
+          key: ${{ runner.os }}-buildkit-integration-tests-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildkit-integration-tests-
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
+      -
+        name: Build integration-tests
+        run: |
+          ./hack/build_ci_first_pass integration-tests
+        env:
+          CACHE_DIR: /tmp/.buildkit-cache/integration-tests
+      -
+        name: Dump context
+        if: always()
+        uses: crazy-max/ghaction-dump-context@v1
+
+  test:
+    runs-on: ubuntu-latest
+    needs: [test-base]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          -
+            pkg: ./client
+            typ: integration
+          -
+            pkg: ./cmd/buildctl ./worker/containerd
+            typ: integration
+          -
+            pkg: ''
+            skip-integration-tests: 1
+            typ: integration gateway
+          -
+            pkg: ./frontend
+            typ: ''
+          -
+            pkg: ./frontend/dockerfile
+            typ: ''
+          -
+            pkg: ./frontend/dockerfile
+            typ: dockerfile
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Cache integration-tests
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildkit-cache/integration-tests
+          key: ${{ runner.os }}-buildkit-integration-tests-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildkit-integration-tests-
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
+      -
+        name: Test ${{ matrix.pkg }} ${{ matrix.typ }}
+        run: |
+          ./hack/test ${{ matrix.typ }}
+        env:
+          TEST_COVERAGE: 1
+          TESTPKGS: ${{ matrix.pkg }}
+          TESTFLAGS: -v --parallel=6 --timeout=20m
+          SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }}
+          CACHE_DIR: /tmp/.buildkit-cache/integration-tests
+      -
+        name: Upload coverage
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./coverage/coverage.txt
+      -
+        name: Dump context
+        if: always()
+        uses: crazy-max/ghaction-dump-context@v1
+
+  os:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          # - ubuntu-latest
+          # - macOS-latest
+          - windows-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.13
+      -
+        name: Cache Go modules
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      -
+        name: Go mod
+        run: |
+          go mod download
+      -
+        name: Test
+        env:
+          TMPDIR: ${{ runner.temp }}
+          SKIP_INTEGRATION_TESTS: 1
+        run: |
+          go test -coverprofile=coverage.txt -covermode=atomic ./...
+        shell: bash
+      -
+        name: Upload coverage
+        uses: codecov/codecov-action@v1
+        with:
+          file: coverage.txt
+      -
+        name: Dump context
+        if: always()
+        uses: crazy-max/ghaction-dump-context@v1
--- a/.github/workflows/windows.yaml
+++ b/.github/workflows/windows.yaml
@ -1,43 +0,0 @@
-# This workflow is for builds on Windows. We cannot use the existing
-# Travis-CI workflow as that depends on already having functioning
-# BuildKit to run the v1.1-experimental Dockerfile that builds BuildKit.
-name: Windows
-
-on: [push, pull_request]
-
-jobs:
-  build:
-    name: Build BuildKit
-    runs-on: windows-2019
-    steps:
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.13
-      - name: Checkout
-        uses: actions/checkout@v2
-        # Is this needed? Didn't Go Modules fix this?
-        with:
-          path: src/github.com/moby/buildkit
-      - name: Compile
-        run: go install -mod=vendor ./cmd/...
-        working-directory: src/github.com/moby/buildkit
-
-  unit_tests:
-    name: Unit Tests
-    runs-on: windows-2019
-    steps:
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.13
-      - name: Checkout
-        uses: actions/checkout@v2
-        # Is this needed? Didn't Go Modules fix this?
-        with:
-          path: src/github.com/moby/buildkit
-      - name: Unit test
-        env:
-          SKIP_INTEGRATION_TESTS: 1
-        run: go test -mod=vendor -v ./...
-        working-directory: src/github.com/moby/buildkit
--- a/.gitignore
+++ b/.gitignore
@ -1,4 +1,5 @@
 bin
+coverage
 release-out
 .certs
 .tmp
--- a/.travis.yml
+++ b/.travis.yml
@ -19,7 +19,8 @@ before_install:
  - docker info

 install:
-  - docker run --rm --privileged linuxkit/binfmt:v0.8
+  - docker buildx create --name builder --driver docker-container --driver-opt image=$REPO_SLUG_ORIGIN --use
+  - docker run --rm --privileged tonistiigi/binfmt:latest --install all
  - docker run --name buildkit -d --privileged -p 1234:1234 $REPO_SLUG_ORIGIN --debug --addr tcp://0.0.0.0:1234 --oci-worker-gc=false
  - sudo docker cp buildkit:/usr/bin/buildctl /usr/bin/
  - export BUILDKIT_HOST=tcp://0.0.0.0:1234
@ -31,23 +32,26 @@ jobs:
  include:
    - stage: building
      name: "Build"
-      script: ./hack/login_ci_cache && ./hack/build_ci_first_pass
+      script:
+        - ./hack/login_ci_cache
+        - ./hack/build_ci_first_pass binaries
+        - ./hack/build_ci_first_pass integration-tests
    - stage: testing
      name: "Client integration tests"
-      script: 
+      script:
        - TESTPKGS=./client TESTFLAGS='-v --timeout=20m' ./hack/test integration
        - TESTPKGS=./cmd/buildctl ./hack/test integration
        - TESTPKGS=./worker/containerd ./hack/test integration
    - script:
-       - ./hack/lint
-       - SKIP_INTEGRATION_TESTS=1 ./hack/test integration gateway
-       - ./hack/validate-vendor
-       - ./hack/validate-generated-files
-       - ./hack/validate-shfmt
-       - TESTPKGS=./frontend ./hack/test
+        - ./hack/lint
+        - SKIP_INTEGRATION_TESTS=1 ./hack/test integration gateway
+        - ./hack/validate-vendor
+        - ./hack/validate-generated-files
+        - ./hack/validate-shfmt
+        - TESTPKGS=./frontend ./hack/test
      name: "Unit Tests & Lint & Vendor & Proto"
    - script:
-       - TESTPKGS=./frontend/dockerfile TESTFLAGS='-v --parallel=5 --timeout=30m' ./hack/test
+        - TESTPKGS=./frontend/dockerfile TESTFLAGS='-v --parallel=5 --timeout=30m' ./hack/test
      name: "Dockerfile integration tests"
    - script: TESTPKGS=./frontend/dockerfile ./hack/test dockerfile
      name: "External Dockerfile tests"
@ -80,9 +84,9 @@ jobs:
          skip_cleanup: true
          file_glob: true
          on:
-           repo: moby/buildkit
-           tags: true
-           condition: $TRAVIS_TAG =~ ^v[0-9]
+            repo: moby/buildkit
+            tags: true
+            condition: $TRAVIS_TAG =~ ^v[0-9]
        - provider: script
          script: ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release master mainline $DF_REPO_SLUG_TARGET push
          on:
@ -107,8 +111,6 @@ jobs:
            repo: moby/buildkit
            branch: master
            condition: $TRAVIS_EVENT_TYPE == "cron"
-      

 before_deploy:
  - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-
--- a/README.md
+++ b/README.md
@ -5,6 +5,7 @@
 [![GoDoc](https://godoc.org/github.com/moby/buildkit?status.svg)](https://godoc.org/github.com/moby/buildkit/client/llb)
 [![Build Status](https://travis-ci.com/moby/buildkit.svg?branch=master)](https://travis-ci.com/moby/buildkit)
 [![Go Report Card](https://goreportcard.com/badge/github.com/moby/buildkit)](https://goreportcard.com/report/github.com/moby/buildkit)
+[![codecov](https://codecov.io/gh/moby/buildkit/branch/master/graph/badge.svg)](https://codecov.io/gh/moby/buildkit)

 BuildKit is a toolkit for converting source code to build artifacts in an efficient, expressive and repeatable manner.

--- a/hack/build_ci_first_pass
+++ b/hack/build_ci_first_pass
@ -1,35 +1,60 @@
 #!/usr/bin/env bash

-. $(dirname $0)/util
+TYP=$1

+. $(dirname $0)/util
 set -e

-if [ -z "$TRAVIS_COMMIT" ]; then
-  echo "TRAVIS_COMMIT_REQURED"
-  exit 0
+usage() {
+  echo "usage: ./hack/build_ci_first_pass <typ>"
+  exit 1
+}
+
+if [ -z "$TYP" ]; then
+  usage
 fi

-progressFlag=""
-if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
-
+importCacheFlags=""
 exportCacheFlags=""
 exportFlags=""

-if [ -f /tmp/buildkit-ci-cache-repo ]; then
-  exportCacheFlags="--export-cache=type=inline"
-  exportFlags="--output=type=image,push=true,name=$(cat /tmp/buildkit-ci-cache-repo)"
+if [ "$TRAVIS" = "true" ]; then
+  if [ -z "$TRAVIS_COMMIT" ]; then
+    echo "Travis commit required"
+    exit 0
+  fi
+  if [ -f /tmp/buildkit-ci-cache-repo ]; then
+    exportCacheFlags="--cache-to=type=inline"
+    exportFlags="--output=type=image,push=true,name=$(cat /tmp/buildkit-ci-cache-repo)"
+  fi
+  if [ "$TYP" = "binaries" ]; then
+    importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries --cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests"
+    exportFlags="$exportFlags:binaries"
+  elif [ "$TYP" = "integration-tests" ]; then
+    importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests"
+    exportFlags="$exportFlags:integration-tests"
+  fi
+elif [ "$GITHUB_ACTIONS" = "true" ]; then
+  context="."
+  if [ -n "$cacheref" ]; then
+    importCacheFlags="--cache-from=type=local,src=$cacheref"
+    exportCacheFlags="--cache-to=type=local,dest=$cacheref"
+  fi
 fi

-set -x
-buildctl build $progressFlag --frontend=dockerfile.v0 \
-  $currentcontextBuildctl \
-  --opt target=binaries \
-  --import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries \
-  --import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests \
-  $exportCacheFlags $exportFlags:binaries
-
-buildctl build $progressFlag --frontend=dockerfile.v0 \
-  $currentcontextBuildctl \
-  --opt target=integration-tests-base \
-  --import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests \
-  $exportCacheFlags $exportFlags:integration-tests
+case $TYP in
+  "binaries")
+    buildxCmd build $importCacheFlags $exportCacheFlags $exportFlags \
+      --target "binaries" \
+      $currentcontext
+    ;;
+  "integration-tests")
+    buildxCmd build $importCacheFlags $exportCacheFlags $exportFlags \
+      --target "integration-tests-base" \
+      $currentcontext
+    ;;
+  *)
+    echo >&2 "Unknown type $TYP"
+    exit 1
+    ;;
+esac
--- a/hack/test
+++ b/hack/test
@ -8,8 +8,8 @@ set -eu -o pipefail
 : ${TEST_DOCKERFILE=}
 : ${TEST_DOCKERD=}
 : ${TEST_DOCKERD_BINARY=$(which dockerd)}
+: ${TEST_COVERAGE=}
 : ${DOCKERFILE_RELEASES=}
-: ${CONTINUOUS_INTEGRATION=}
 : ${BUILDKIT_REGISTRY_MIRROR_DIR=}

 if [ "$TEST_DOCKERD" == "1" ] && ! file $TEST_DOCKERD_BINARY | grep "statically linked" >/dev/null; then
@ -19,12 +19,13 @@ fi

 importCacheFlags=""
 if [ -n "$cacheref" ]; then
-  importCacheFlags="--import-cache=type=registry,ref=$cacheref:integration-tests"
+  if [ "$cachetype" = "local" ]; then
+    importCacheFlags="--cache-from=type=local,src=$cacheref"
+  else
+    importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests --cache-from=type=registry,ref=$cacheref:integration-tests"
+  fi
 fi

-progressFlag=""
-if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
-
 if [ "$#" == 0 ]; then TEST_INTEGRATION=1; fi

 while test $# -gt 0; do
@ -47,33 +48,25 @@ done

 iid="buildkit-tests"
 iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
-set -x

-case $buildmode in
-  "buildkit")
-    tmpfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
-    buildctl build $progressFlag --frontend=dockerfile.v0 $currentcontextBuildctl \
-      --opt target=integration-tests \
-      --output type=docker,name=$iid,dest=$tmpfile \
-      --import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests $importCacheFlags
-    docker load -i $tmpfile
-    rm $tmpfile
-    ;;
-  "docker-buildkit")
-    export DOCKER_BUILDKIT=1
-    docker build --iidfile $iidfile --target integration-tests --force-rm .
-    iid=$(cat $iidfile)
-    ;;
-  *)
-    echo "Unsupported build mode: $buildmode" >&2
-    exit 1
-    ;;
-esac
+coverageVol=""
+coverageFlags=""
+if [ "$TEST_COVERAGE" = "1" ]; then
+  covdir="$(pwd)/coverage"
+  mkdir -p "$covdir"
+  coverageVol="-v $covdir:/coverage"
+  coverageFlags="-coverprofile=/coverage/coverage.txt -covermode=atomic"
+fi
+
+buildxCmd build $importCacheFlags \
+  --target "integration-tests" \
+  --output "type=docker,name=$iid" \
+  $currentcontext

 cacheVolume=$(docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod alpine)

 if [ "$TEST_INTEGRATION" == 1 ]; then
-  cid=$(docker create --rm -v /tmp --volumes-from=$cacheVolume -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test ${TESTFLAGS:--v} ${TESTPKGS:-./...})
+  cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...})
  if [ "$TEST_DOCKERD" = "1" ]; then
    docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/
  fi
@ -98,26 +91,16 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
  for release in $DOCKERFILE_RELEASES; do
    buildtags=$(cat ./frontend/dockerfile/release/$release/tags)
    tarout=$(mktemp -t dockerfile-frontend.XXXXXXXXXX)
-    case $buildmode in
-      "buildkit")
-        buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \
-          --opt filename=./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile \
-          --opt build-arg:BUILDTAGS="$buildtags" \
-          --output type=oci,dest=$tarout
-        ;;
-      "docker-buildkit")
-        dfiidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
-        docker build --iidfile=$dfiidfile -f ./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile --build-arg BUILDTAGS="$buildtags" .
-        dfiid=$(cat $dfiidfile)
-        docker save -o $tarout $dfiid
-        docker rmi $dfiid
-        rm $dfiidfile
-        ;;
-    esac
+
+    buildxCmd build $importCacheFlags \
+      --build-arg "BUILDTAGS=$buildtags" \
+      --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
+      --output "type=oci,dest=$tarout" \
+      .

    if [ -s $tarout ]; then
-      if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
-        cid=$(docker create -v /tmp --rm --privileged --volumes-from=$cacheVolume -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM=1 -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile)
+      if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$GITHUB_ACTIONS" = "true" ]; then
+        cid=$(docker create -v /tmp $coverageVol --rm --privileged --volumes-from=$cacheVolume -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM=1 -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test $coverageFlags --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile)
        docker cp $tarout $cid:/$release.tar
        if [ "$TEST_DOCKERD" = "1" ]; then
          docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/
@ -130,10 +113,3 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
 fi

 docker rm -v $cacheVolume
-
-case $buildmode in
-  "docker-buildkit")
-    rm "$iidfile"
-    docker rmi $iid
-    ;;
-esac
--- a/hack/util
+++ b/hack/util
@ -57,7 +57,7 @@ cachetype=""
 cacheref=""
 currentref=""
 if [ "$GITHUB_ACTIONS" = "true" ]; then
-  currentref="git://github.com/moby/buildkit#$GITHUB_REF"
+  currentref="git://github.com/$GITHUB_REPOSITORY#$GITHUB_REF"
  cachetype="local"
  cacheref="$CACHE_DIR"
 elif [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then