JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor test for GitHub Actions 

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
v0.8
CrazyMax 2020-11-22 04:03:55 +01:00
parent dec4da0920
commit e22c19d77f
No known key found for this signature in database
GPG Key ID: 3248E46B6BB8C7F7
8 changed files with 276 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

180
.github/workflows/test.yml vendored Normal file
View File

@ -0,0 +1,180 @@
name: test
on:
push:
branches:
- "ghactions-test" #remove when merged
- 'master'
tags:
- 'v*'
- 'dockerfile/*'
pull_request:
branches:
- 'master'
env:
REPO_SLUG_ORIGIN: "moby/buildkit:latest"
jobs:
test-base:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Cache integration-tests
uses: actions/cache@v2
with:
path: /tmp/.buildkit-cache/integration-tests
key: ${{ runner.os }}-buildkit-integration-tests-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildkit-integration-tests-
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
with:
driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-
name: Build integration-tests
run: |
./hack/build_ci_first_pass integration-tests
env:
CACHE_DIR: /tmp/.buildkit-cache/integration-tests
-
name: Dump context
if: always()
uses: crazy-max/ghaction-dump-context@v1
test:
runs-on: ubuntu-latest
needs: [ test-base ]
strategy:
fail-fast: false
matrix:
include:
-
pkg: ./client
flags: -v --timeout=20m
skip-it: 0
typ: integration
-
pkg: ./cmd/buildctl
flags: -v
skip-it: 0
typ: integration
-
pkg: ./worker/containerd
flags: -v
skip-it: 0
typ: integration
-
pkg: ''
flags: -v
skip-it: 1
typ: integration gateway
-
pkg: ./frontend
flags: -v
skip-it: 0
typ: ''
-
pkg: ./frontend/dockerfile
flags: -v --parallel=6 --timeout=20m
skip-it: 0
typ: ''
-
pkg: ./frontend/dockerfile
flags: -v
skip-it: 0
typ: dockerfile
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Cache integration-tests
uses: actions/cache@v2
with:
path: /tmp/.buildkit-cache/integration-tests
key: ${{ runner.os }}-buildkit-integration-tests-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildkit-integration-tests-
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
with:
driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-
name: Test ${{ matrix.pkg }} ${{ matrix.typ }}
run: |
./hack/test ${{ matrix.typ }}
env:
TEST_COVERAGE: 1
TESTPKGS: ${{ matrix.pkg }}
TESTFLAGS: ${{ matrix.flags }}
SKIP_INTEGRATION_TESTS: ${{ matrix.skip-it }}
CACHE_DIR: /tmp/.buildkit-cache/integration-tests
-
name: Upload coverage
uses: codecov/codecov-action@v1
with:
file: ./coverage/coverage.txt
-
name: Dump context
if: always()
uses: crazy-max/ghaction-dump-context@v1
os:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
#- ubuntu-latest
#- macOS-latest
- windows-latest
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.13
-
name: Cache Go modules
uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
-
name: Go mod
run: |
go mod download
-
name: Test
env:
TMPDIR: ${{ runner.temp }}
SKIP_INTEGRATION_TESTS: 1
run: |
go test -coverprofile=coverage.txt -covermode=atomic ./...
shell: bash
-
name: Upload coverage
uses: codecov/codecov-action@v1
with:
file: coverage.txt
-
name: Dump context
if: always()
uses: crazy-max/ghaction-dump-context@v1

43
.github/workflows/windows.yaml vendored
View File

@ -1,43 +0,0 @@
# This workflow is for builds on Windows. We cannot use the existing
# Travis-CI workflow as that depends on already having functioning
# BuildKit to run the v1.1-experimental Dockerfile that builds BuildKit.
name: Windows
on: [push, pull_request]
jobs:
build:
name: Build BuildKit
runs-on: windows-2019
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.13
- name: Checkout
uses: actions/checkout@v2
# Is this needed? Didn't Go Modules fix this?
with:
path: src/github.com/moby/buildkit
- name: Compile
run: go install -mod=vendor ./cmd/...
working-directory: src/github.com/moby/buildkit
unit_tests:
name: Unit Tests
runs-on: windows-2019
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.13
- name: Checkout
uses: actions/checkout@v2
# Is this needed? Didn't Go Modules fix this?
with:
path: src/github.com/moby/buildkit
- name: Unit test
env:
SKIP_INTEGRATION_TESTS: 1
run: go test -mod=vendor -v ./...
working-directory: src/github.com/moby/buildkit

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
bin
coverage
release-out
.certs
.tmp

33
.travis.yml
View File

@ -4,6 +4,7 @@ language: minimal
env:
global:
- DOCKER_CHANNEL="stable"
- DOCKER_CLI_EXPERIMENTAL="enabled"
- PLATFORMS="linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le"
- PREFER_BUILDCTL="1"
@ -18,7 +19,8 @@ before_install:
- docker info
install:
- docker run --rm --privileged linuxkit/binfmt:v0.8
- docker buildx create --name builder --driver docker-container --driver-opt image=$REPO_SLUG_ORIGIN --use
- docker run --rm --privileged tonistiigi/binfmt:latest --install all
- docker run --name buildkit -d --privileged -p 1234:1234 $REPO_SLUG_ORIGIN --debug --addr tcp://0.0.0.0:1234 --oci-worker-gc=false
- sudo docker cp buildkit:/usr/bin/buildctl /usr/bin/
- export BUILDKIT_HOST=tcp://0.0.0.0:1234
@ -30,23 +32,26 @@ jobs:
include:
- stage: building
name: "Build"
script: ./hack/login_ci_cache && ./hack/build_ci_first_pass
script:
- ./hack/login_ci_cache
- ./hack/build_ci_first_pass binaries
- ./hack/build_ci_first_pass integration-tests
- stage: testing
name: "Client integration tests"
script:
script:
- TESTPKGS=./client TESTFLAGS='-v --timeout=20m' ./hack/test integration
- TESTPKGS=./cmd/buildctl ./hack/test integration
- TESTPKGS=./worker/containerd ./hack/test integration
- script:
- ./hack/lint
- SKIP_INTEGRATION_TESTS=1 ./hack/test integration gateway
- ./hack/validate-vendor
- ./hack/validate-generated-files
- ./hack/validate-shfmt
- TESTPKGS=./frontend ./hack/test
- ./hack/lint
- SKIP_INTEGRATION_TESTS=1 ./hack/test integration gateway
- ./hack/validate-vendor
- ./hack/validate-generated-files
- ./hack/validate-shfmt
- TESTPKGS=./frontend ./hack/test
name: "Unit Tests & Lint & Vendor & Proto"
- script:
- TESTPKGS=./frontend/dockerfile TESTFLAGS='-v --parallel=6 --timeout=20m' ./hack/test
- TESTPKGS=./frontend/dockerfile TESTFLAGS='-v --parallel=5 --timeout=30m' ./hack/test
name: "Dockerfile integration tests"
- script: TESTPKGS=./frontend/dockerfile ./hack/test dockerfile
name: "External Dockerfile tests"
@ -79,9 +84,9 @@ jobs:
skip_cleanup: true
file_glob: true
on:
repo: moby/buildkit
tags: true
condition: $TRAVIS_TAG =~ ^v[0-9]
repo: moby/buildkit
tags: true
condition: $TRAVIS_TAG =~ ^v[0-9]
- provider: script
script: ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release master mainline $DF_REPO_SLUG_TARGET push
on:
@ -106,8 +111,6 @@ jobs:
repo: moby/buildkit
branch: master
condition: $TRAVIS_EVENT_TYPE == "cron"
before_deploy:
- echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin

1
README.md
View File

@ -5,6 +5,7 @@
[![GoDoc](https://godoc.org/github.com/moby/buildkit?status.svg)](https://godoc.org/github.com/moby/buildkit/client/llb)
[![Build Status](https://travis-ci.com/moby/buildkit.svg?branch=master)](https://travis-ci.com/moby/buildkit)
[![Go Report Card](https://goreportcard.com/badge/github.com/moby/buildkit)](https://goreportcard.com/report/github.com/moby/buildkit)
[![codecov](https://codecov.io/gh/moby/buildkit/branch/master/graph/badge.svg)](https://codecov.io/gh/moby/buildkit)
BuildKit is a toolkit for converting source code to build artifacts in an efficient, expressive and repeatable manner.

71
hack/build_ci_first_pass
View File

@ -1,35 +1,60 @@
#!/usr/bin/env bash
. $(dirname $0)/util
TYP=$1
. $(dirname $0)/util
set -e
if [ -z "$TRAVIS_COMMIT" ]; then
echo "TRAVIS_COMMIT_REQURED"
exit 0
usage() {
echo "usage: ./hack/build_ci_first_pass <typ>"
exit 1
}
if [ -z "$TYP" ]; then
usage
fi
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
importCacheFlags=""
exportCacheFlags=""
exportFlags=""
if [ -f /tmp/buildkit-ci-cache-repo ]; then
exportCacheFlags="--export-cache=type=inline"
exportFlags="--output=type=image,push=true,name=$(cat /tmp/buildkit-ci-cache-repo)"
if [ "$TRAVIS" = "true" ]; then
if [ -z "$TRAVIS_COMMIT" ]; then
echo "Travis commit required"
exit 0
fi
if [ -f /tmp/buildkit-ci-cache-repo ]; then
exportCacheFlags="--cache-to=type=inline"
exportFlags="--output=type=image,push=true,name=$(cat /tmp/buildkit-ci-cache-repo)"
fi
if [ "$TYP" = "binaries" ]; then
importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries --cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests"
exportFlags="$exportFlags:binaries"
elif [ "$TYP" = "integration-tests" ]; then
importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests"
exportFlags="$exportFlags:integration-tests"
fi
elif [ "$GITHUB_ACTIONS" = "true" ]; then
context="."
if [ -n "$cacheref" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheref"
exportCacheFlags="--cache-to=type=local,dest=$cacheref"
fi
fi
set -x
buildctl build $progressFlag --frontend=dockerfile.v0 \
$currentcontextBuildctl \
--opt target=binaries \
--import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries \
--import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests \
$exportCacheFlags $exportFlags:binaries
buildctl build $progressFlag --frontend=dockerfile.v0 \
$currentcontextBuildctl \
--opt target=integration-tests-base \
--import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests \
$exportCacheFlags $exportFlags:integration-tests
case $TYP in
"binaries")
buildxCmd build $importCacheFlags $exportCacheFlags $exportFlags \
--target "binaries" \
$currentcontext
;;
"integration-tests")
buildxCmd build $importCacheFlags $exportCacheFlags $exportFlags \
--target "integration-tests-base" \
$currentcontext
;;
*)
echo >&2 "Unknown type $TYP"
exit 1
;;
esac

79
hack/test
View File

@ -8,8 +8,8 @@ set -eu -o pipefail
: ${TEST_DOCKERFILE=}
: ${TEST_DOCKERD=}
: ${TEST_DOCKERD_BINARY=$(which dockerd)}
: ${TEST_COVERAGE=}
: ${DOCKERFILE_RELEASES=}
: ${CONTINUOUS_INTEGRATION=}
: ${BUILDKIT_REGISTRY_MIRROR_DIR=}
if [ "$TEST_DOCKERD" == "1" ] && ! file $TEST_DOCKERD_BINARY | grep "statically linked" >/dev/null; then
@ -19,12 +19,13 @@ fi
importCacheFlags=""
if [ -n "$cacheref" ]; then
importCacheFlags="--import-cache=type=registry,ref=$cacheref:integration-tests"
if [ "$cachetype" = "local" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheref"
else
importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests --cache-from=type=registry,ref=$cacheref:integration-tests"
fi
fi
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
if [ "$#" == 0 ]; then TEST_INTEGRATION=1; fi
while test $# -gt 0; do
@ -47,33 +48,25 @@ done
iid="buildkit-tests"
iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
set -x
case $buildmode in
"buildkit")
tmpfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
buildctl build $progressFlag --frontend=dockerfile.v0 $currentcontextBuildctl \
--opt target=integration-tests \
--output type=docker,name=$iid,dest=$tmpfile \
--import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:integration-tests $importCacheFlags
docker load -i $tmpfile
rm $tmpfile
;;
"docker-buildkit")
export DOCKER_BUILDKIT=1
docker build --iidfile $iidfile --target integration-tests --force-rm .
iid=$(cat $iidfile)
;;
*)
echo "Unsupported build mode: $buildmode" >&2
exit 1
;;
esac
coverageVol=""
coverageFlags=""
if [ "$TEST_COVERAGE" = "1" ]; then
covdir="$(pwd)/coverage"
mkdir -p "$covdir"
coverageVol="-v $covdir:/coverage"
coverageFlags="-coverprofile=/coverage/coverage.txt -covermode=atomic"
fi
buildxCmd build $importCacheFlags \
--target "integration-tests" \
--output "type=docker,name=$iid" \
$currentcontext
cacheVolume=$(docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod alpine)
if [ "$TEST_INTEGRATION" == 1 ]; then
cid=$(docker create --rm -v /tmp --volumes-from=$cacheVolume -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test ${TESTFLAGS:--v} ${TESTPKGS:-./...})
cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...})
if [ "$TEST_DOCKERD" = "1" ]; then
docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/
fi
@ -98,26 +91,15 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
for release in $DOCKERFILE_RELEASES; do
buildtags=$(cat ./frontend/dockerfile/release/$release/tags)
tarout=$(mktemp -t dockerfile-frontend.XXXXXXXXXX)
case $buildmode in
"buildkit")
buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \
--opt filename=./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile \
--opt build-arg:BUILDTAGS="$buildtags" \
--output type=oci,dest=$tarout
;;
"docker-buildkit")
dfiidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
docker build --iidfile=$dfiidfile -f ./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile --build-arg BUILDTAGS="$buildtags" .
dfiid=$(cat $dfiidfile)
docker save -o $tarout $dfiid
docker rmi $dfiid
rm $dfiidfile
;;
esac
buildxCmd build $importCacheFlags \
--build-arg "BUILDTAGS=$buildtags" \
--file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
.
if [ -s $tarout ]; then
if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
cid=$(docker create -v /tmp --rm --privileged --volumes-from=$cacheVolume -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM=1 -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile)
if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$GITHUB_ACTIONS" = "true" ]; then
cid=$(docker create -v /tmp $coverageVol --rm --privileged --volumes-from=$cacheVolume -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM=1 -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test $coverageFlags --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile)
docker cp $tarout $cid:/$release.tar
if [ "$TEST_DOCKERD" = "1" ]; then
docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/
@ -130,10 +112,3 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
fi
docker rm -v $cacheVolume
case $buildmode in
"docker-buildkit")
rm "$iidfile"
docker rmi $iid
;;
esac

2
hack/util
View File

@ -35,7 +35,7 @@ cachetype=""
cacheref=""
currentref=""
if [ "$GITHUB_ACTIONS" = "true" ]; then
currentref="git://github.com/moby/buildkit#$GITHUB_REF"
currentref="git://github.com/$GITHUB_REPOSITORY#$GITHUB_REF"
cachetype="local"
cacheref="$CACHE_DIR"
elif [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then