diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..855bd44f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,251 @@ +name: build + +on: + workflow_dispatch: + push: + branches: + - 'master' + tags: + - 'v*' + - 'dockerfile/*' + pull_request: + branches: + - 'master' + +env: + REPO_SLUG_ORIGIN: "moby/buildkit:latest" + PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le" + CACHEKEY_INTEGRATION_TESTS: "integration-tests" + CACHEKEY_BINARIES: "binaries" + +jobs: + base: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Cache ${{ env.CACHEKEY_INTEGRATION_TESTS }} + uses: actions/cache@v2 + with: + path: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }} + key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_INTEGRATION_TESTS }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildkit-${{ env.CACHEKEY_INTEGRATION_TESTS }}- + - + name: Cache ${{ env.CACHEKEY_BINARIES }} + uses: actions/cache@v2 + with: + path: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }} + key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}- + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + - + name: Build ${{ env.CACHEKEY_BINARIES }} + run: | + ./hack/build_ci_first_pass binaries + env: + CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }} + - + name: Build ${{ env.CACHEKEY_INTEGRATION_TESTS }} + run: | + ./hack/build_ci_first_pass integration-tests + env: + CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }} + - + name: Dump context + if: always() + uses: crazy-max/ghaction-dump-context@v1 + + test: + runs-on: ubuntu-latest + needs: [base] + strategy: + fail-fast: false + matrix: + include: + - + pkg: ./client + typ: integration + - + pkg: ./cmd/buildctl ./worker/containerd + typ: integration + - + pkg: '' + skip-integration-tests: 1 + typ: integration gateway + - + pkg: ./frontend + typ: '' + - + pkg: ./frontend/dockerfile + typ: '' + - + pkg: ./frontend/dockerfile + typ: dockerfile + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Cache ${{ env.CACHEKEY_INTEGRATION_TESTS }} + uses: actions/cache@v2 + with: + path: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }} + key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_INTEGRATION_TESTS }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildkit-${{ env.CACHEKEY_INTEGRATION_TESTS }}- + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + - + name: Test ${{ matrix.pkg }} ${{ matrix.typ }} + run: | + ./hack/test ${{ matrix.typ }} + env: + TEST_COVERAGE: 1 + TESTPKGS: ${{ matrix.pkg }} + TESTFLAGS: -v --parallel=6 --timeout=20m + SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }} + CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }} + - + name: Upload coverage + uses: codecov/codecov-action@v1 + with: + file: ./coverage/coverage.txt + - + name: Dump context + if: always() + uses: crazy-max/ghaction-dump-context@v1 + + test-os: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + # - ubuntu-latest + # - macOS-latest + - windows-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.13 + - + name: Cache Go modules + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - + name: Go mod + run: | + go mod download + - + name: Test + env: + TMPDIR: ${{ runner.temp }} + SKIP_INTEGRATION_TESTS: 1 + run: | + go test -coverprofile=coverage.txt -covermode=atomic ./... + shell: bash + - + name: Upload coverage + uses: codecov/codecov-action@v1 + with: + file: coverage.txt + - + name: Dump context + if: always() + uses: crazy-max/ghaction-dump-context@v1 + + cross: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Cache cross + uses: actions/cache@v2 + with: + path: /tmp/.buildkit-cache/cross + key: ${{ runner.os }}-buildkit-cross-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildkit-cross- + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + - + name: Cross + run: | + ./hack/cross + env: + PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,windows/amd64 + RUNC_PLATFORMS: ${{ env.PLATFORMS }} + CACHE_DIR: /tmp/.buildkit-cache/cross + - + name: Dump context + if: always() + uses: crazy-max/ghaction-dump-context@v1 + + buildkit-buildkitd: + runs-on: ubuntu-latest + needs: [base] + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Cache ${{ env.CACHEKEY_BINARIES }} + uses: actions/cache@v2 + with: + path: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }} + key: ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildkit-${{ env.CACHEKEY_BINARIES }}- + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + - + name: Build + run: | + ./hack/buildkit-buildkitd + env: + PLATFORMS: windows/amd64 + CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }} + - + name: Dump context + if: always() + uses: crazy-max/ghaction-dump-context@v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..3ed6189f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,27 @@ +name: release + +on: + workflow_dispatch: + workflow_run: + workflows: + - 'build' + branches: + - 'master' + types: + - 'completed' + +env: + REPO_SLUG_ORIGIN: "moby/buildkit:latest" + PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le" + +jobs: + image: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Dump context + if: always() + uses: crazy-max/ghaction-dump-context@v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index bc8ad864..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,164 +0,0 @@ -name: test - -on: - push: - branches: - - "ghactions-test" # remove when merged - - 'master' - tags: - - 'v*' - - 'dockerfile/*' - pull_request: - branches: - - 'master' - -env: - REPO_SLUG_ORIGIN: "moby/buildkit:latest" - -jobs: - test-base: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Cache integration-tests - uses: actions/cache@v2 - with: - path: /tmp/.buildkit-cache/integration-tests - key: ${{ runner.os }}-buildkit-integration-tests-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildkit-integration-tests- - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} - - - name: Build integration-tests - run: | - ./hack/build_ci_first_pass integration-tests - env: - CACHE_DIR: /tmp/.buildkit-cache/integration-tests - - - name: Dump context - if: always() - uses: crazy-max/ghaction-dump-context@v1 - - test: - runs-on: ubuntu-latest - needs: [test-base] - strategy: - fail-fast: false - matrix: - include: - - - pkg: ./client - typ: integration - - - pkg: ./cmd/buildctl ./worker/containerd - typ: integration - - - pkg: '' - skip-integration-tests: 1 - typ: integration gateway - - - pkg: ./frontend - typ: '' - - - pkg: ./frontend/dockerfile - typ: '' - - - pkg: ./frontend/dockerfile - typ: dockerfile - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Cache integration-tests - uses: actions/cache@v2 - with: - path: /tmp/.buildkit-cache/integration-tests - key: ${{ runner.os }}-buildkit-integration-tests-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildkit-integration-tests- - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} - - - name: Test ${{ matrix.pkg }} ${{ matrix.typ }} - run: | - ./hack/test ${{ matrix.typ }} - env: - TEST_COVERAGE: 1 - TESTPKGS: ${{ matrix.pkg }} - TESTFLAGS: -v --parallel=6 --timeout=20m - SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }} - CACHE_DIR: /tmp/.buildkit-cache/integration-tests - - - name: Upload coverage - uses: codecov/codecov-action@v1 - with: - file: ./coverage/coverage.txt - - - name: Dump context - if: always() - uses: crazy-max/ghaction-dump-context@v1 - - os: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: - # - ubuntu-latest - # - macOS-latest - - windows-latest - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.13 - - - name: Cache Go modules - uses: actions/cache@v2 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Go mod - run: | - go mod download - - - name: Test - env: - TMPDIR: ${{ runner.temp }} - SKIP_INTEGRATION_TESTS: 1 - run: | - go test -coverprofile=coverage.txt -covermode=atomic ./... - shell: bash - - - name: Upload coverage - uses: codecov/codecov-action@v1 - with: - file: coverage.txt - - - name: Dump context - if: always() - uses: crazy-max/ghaction-dump-context@v1 diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e54d535a..e7c2d2de 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1,6 +1,7 @@ name: validate on: + workflow_dispatch: push: branches: - 'master' diff --git a/.travis.yml b/.travis.yml index 82f722bd..4f84cac1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,9 @@ jobs: name: "Dockerfile integration tests" - script: TESTPKGS=./frontend/dockerfile ./hack/test dockerfile name: "External Dockerfile tests" - - script: RUNC_PLATFORMS=$PLATFORMS PLATFORMS="${PLATFORMS},darwin/amd64,windows/amd64" ./hack/cross + - script: + - RUNC_PLATFORMS=$PLATFORMS PLATFORMS="${PLATFORMS},darwin/amd64,windows/amd64" ./hack/cross + - PLATFORMS="windows/amd64" ./hack/buildkit-buildkitd name: "Cross" - script: ./hack/images local $REPO_SLUG_TARGET name: "Build image" diff --git a/hack/build_ci_first_pass b/hack/build_ci_first_pass index ff022e7d..05cd5a3a 100755 --- a/hack/build_ci_first_pass +++ b/hack/build_ci_first_pass @@ -6,7 +6,7 @@ TYP=$1 set -e usage() { - echo "usage: ./hack/build_ci_first_pass " + echo "usage: ./hack/build_ci_first_pass " exit 1 } @@ -35,7 +35,7 @@ if [ "$TRAVIS" = "true" ]; then exportFlags="$exportFlags:integration-tests" fi elif [ "$GITHUB_ACTIONS" = "true" ]; then - context="." + currentcontext="." if [ -n "$cacheref" ]; then importCacheFlags="--cache-from=type=local,src=$cacheref" exportCacheFlags="--cache-to=type=local,dest=$cacheref" diff --git a/hack/buildkit-buildkitd b/hack/buildkit-buildkitd new file mode 100755 index 00000000..11e6099a --- /dev/null +++ b/hack/buildkit-buildkitd @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +. $(dirname $0)/util +set -e + +: ${PLATFORMS=windows/amd64} + +importCacheFlags="" +if [ -n "$cacheref" ]; then + if [ "$cachetype" = "local" ]; then + currentcontext="." + importCacheFlags="--cache-from=type=local,src=$cacheref" + else + importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries" + fi +fi + +buildxCmd build $importCacheFlags \ + --target "buildkit-buildkitd" \ + --platform "$PLATFORMS" \ + $currentcontext diff --git a/hack/cross b/hack/cross index ee8e1c43..ed9fb4c6 100755 --- a/hack/cross +++ b/hack/cross @@ -1,33 +1,33 @@ #!/usr/bin/env bash . $(dirname $0)/util +set -e : ${PLATFORMS=linux/arm} importCacheFlags="" -if [ -n "$cacheref" ]; then - importCacheFlags="--import-cache=type=registry,ref=$cacheref:binaries" -fi - -set -ex - -if [ -n "$MASTERCACHE_PASSWORD" ] && [ -n "$RUNC_PLATFORMS" ]; then - repo="cicache.buildk.it/moby/buildkit/master" +exportCacheFlags="" +outputFlags="" +if [ -n "$MASTERCACHE_PASSWORD" ]; then $(dirname $0)/login_ci_cache - buildctl build $progressFlag --frontend=dockerfile.v0 \ - $currentcontextBuildctl \ - --opt target=binaries-linux-helper \ - --opt platform=$RUNC_PLATFORMS \ - --import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries-cross-helper \ - --export-cache type=inline --output type=image,push=true,name=$repo:binaries-cross-helper + importCacheFlags="--cache-from=type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries-cross-helper" + exportCacheFlags="--cache-to=type=inline" + outputFlags="--output=type=image,push=true,name=cicache.buildk.it/moby/buildkit/master:binaries-cross-helper" +elif [ "$GITHUB_ACTIONS" = "true" ]; then + currentcontext="." + if [ -n "$cacheref" ]; then + importCacheFlags="--cache-from=type=local,src=$cacheref" + exportCacheFlags="--cache-to=type=local,dest=$cacheref" + fi fi -buildctl build --progress=plain --frontend=dockerfile.v0 \ - $currentcontextBuildctl \ - --opt platform=$PLATFORMS \ - --import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries-cross-helper +if [ -n "$RUNC_PLATFORMS" ]; then + buildxCmd build $importCacheFlags $exportCacheFlags $outputFlags \ + --target "binaries-linux-helper" \ + --platform "$RUNC_PLATFORMS" \ + $currentcontext +fi -buildctl build --progress=plain --frontend=dockerfile.v0 \ - $currentcontextBuildctl \ - --opt platform=windows/amd64 --opt target=buildkit-buildkitd \ - --import-cache type=registry,ref=cicache.buildk.it/moby/buildkit/master:binaries +buildxCmd build $importCacheFlags \ + --platform "$PLATFORMS" \ + $currentcontext diff --git a/hack/util b/hack/util index 9a43750d..f659992f 100755 --- a/hack/util +++ b/hack/util @@ -3,7 +3,6 @@ export BUILDX_NO_DEFAULT_LOAD=true : ${PREFER_BUILDCTL=} : ${PREFER_LEGACY=} -: ${CONTINUOUS_INTEGRATION=} : ${CI=} : ${GITHUB_ACTIONS=} : ${DOCKER_BUILDKIT=} @@ -12,7 +11,7 @@ export BUILDX_NO_DEFAULT_LOAD=true : ${CACHE_DIR=} progressFlag="" -if [ "$CONTINUOUS_INTEGRATION" = "true" ] || [ "$CI" = "true" ]; then +if [ "$CI" = "true" ]; then progressFlag="--progress=plain" fi