JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix CI cache 

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
v0.8
CrazyMax 2020-11-29 06:45:33 +01:00
parent 600e2db7f7
commit 1ac6bd4cbc
No known key found for this signature in database
GPG Key ID: 3248E46B6BB8C7F7
8 changed files with 81 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
.github/workflows/build.yml vendored
View File

@ -59,13 +59,23 @@ jobs:
run: |
./hack/build_ci_first_pass binaries
env:
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
CACHEDIR_TO: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}-new
-
name: Build ${{ env.CACHEKEY_INTEGRATION_TESTS }}
run: |
./hack/build_ci_first_pass integration-tests
env:
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}
CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}
CACHEDIR_TO: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}-new
-
# FIXME: Temp fix for https://github.com/moby/buildkit/issues/1850
name: Move cache
run: |
rm -rf /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
mv /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}-new /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
rm -rf /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}
mv /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}-new /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }}
test:
runs-on: ubuntu-latest
@ -131,7 +141,7 @@ jobs:
TESTPKGS: ${{ matrix.pkg }}
TESTFLAGS: -v --parallel=6 --timeout=20m
SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }}
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }} /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_INTEGRATION_TESTS }} /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }}
-
name: Upload coverage file
uses: actions/upload-artifact@v2
@ -244,7 +254,14 @@ jobs:
env:
PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,windows/amd64
RUNC_PLATFORMS: ${{ env.PLATFORMS }}
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
CACHEDIR_TO: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}-new
-
# FIXME: Temp fix for https://github.com/moby/buildkit/issues/1850
name: Move cache
run: |
rm -rf /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
mv /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}-new /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
release-base:
runs-on: ubuntu-latest
@ -276,12 +293,12 @@ jobs:
strategy:
fail-fast: false
matrix:
include:
- target-stage: ''
- target-stage: rootless
target-stage:
- ''
- rootless
steps:
-
name: Checkout${{ env.CACHEKEY_BINARIES }}
name: Checkout
uses: actions/checkout@v2
-
name: Cache image${{ matrix.target-stage }}
@ -320,7 +337,14 @@ jobs:
./hack/images "${{ needs.release-base.outputs.tag }}" "$REPO_SLUG_TARGET" "${{ needs.release-base.outputs.push }}"
env:
TARGET: ${{ matrix.target-stage }}
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }} /tmp/.buildkit-cache/image${{ matrix.target-stage }}
CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }} /tmp/.buildkit-cache/image${{ matrix.target-stage }}
CACHEDIR_TO: /tmp/.buildkit-cache/image${{ matrix.target-stage }}-new
-
# FIXME: Temp fix for https://github.com/moby/buildkit/issues/1850
name: Move cache
run: |
rm -rf /tmp/.buildkit-cache/image${{ matrix.target-stage }}
mv /tmp/.buildkit-cache/image${{ matrix.target-stage }}-new /tmp/.buildkit-cache/image${{ matrix.target-stage }}
binaries:
runs-on: ubuntu-latest
@ -359,7 +383,7 @@ jobs:
./hack/release-tar "${{ needs.release-base.outputs.tag }}" release-out
env:
PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,windows/amd64
CACHE_DIR: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }} /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }} /tmp/.buildkit-cache/${{ env.CACHEKEY_CROSS }}
-
name: Move artifacts
run: |
@ -446,11 +470,18 @@ jobs:
run: |
./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" "${{ needs.frontend-base.outputs.tag }}" "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}"
env:
CACHE_DIR: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
CACHEDIR_FROM: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
CACHEDIR_TO: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}-new
-
name: Build ${{ needs.frontend-base.outputs.typ }}/labs
if: needs.frontend-base.outputs.typ == 'master'
run: |
./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" labs "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}"
env:
CACHE_DIR: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
CACHEDIR_FROM: /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
-
# FIXME: Temp fix for https://github.com/moby/buildkit/issues/1850
name: Move cache
run: |
rm -rf /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}
mv /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}-new /tmp/.buildkit-cache/frontend-${{ needs.frontend-base.outputs.typ }}

8
frontend/dockerfile/cmd/dockerfile-frontend/hack/release
View File

@ -68,9 +68,11 @@ fi
importCacheFlags=""
exportCacheFlags=""
if [ "$GITHUB_ACTIONS" = "true" ]; then
if [ -n "$cacheref" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheref"
exportCacheFlags="--cache-to=type=local,dest=$cacheref"
if [ -n "$cacheRefFrom" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheRefFrom"
fi
if [ -n "$cacheRefTo" ]; then
exportCacheFlags="--cache-to=type=local,dest=$cacheRefTo"
fi
fi

8
hack/build_ci_first_pass
View File

@ -35,9 +35,11 @@ if [ "$TRAVIS" = "true" ]; then
exportFlags="$exportFlags:integration-tests"
fi
elif [ "$GITHUB_ACTIONS" = "true" ]; then
if [ -n "$cacheref" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheref"
exportCacheFlags="--cache-to=type=local,dest=$cacheref"
if [ -n "$cacheRefFrom" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheRefFrom"
fi
if [ -n "$cacheRefTo" ]; then
exportCacheFlags="--cache-to=type=local,dest=$cacheRefTo"
fi
fi

8
hack/cross
View File

@ -14,9 +14,11 @@ if [ -n "$MASTERCACHE_PASSWORD" ]; then
exportCacheFlags="--cache-to=type=inline"
outputFlags="--output=type=image,push=true,name=cicache.buildk.it/moby/buildkit/master:binaries-cross-helper"
elif [ "$GITHUB_ACTIONS" = "true" ]; then
if [ -n "$cacheref" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheref"
exportCacheFlags="--cache-to=type=local,dest=$cacheref"
if [ -n "$cacheRefFrom" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheRefFrom"
fi
if [ -n "$cacheRefTo" ]; then
exportCacheFlags="--cache-to=type=local,dest=$cacheRefTo"
fi
fi

9
hack/images
View File

@ -67,8 +67,8 @@ if [[ "$versionTag" == "$TAG" ]]; then
fi
importCacheFlags="--cache-from=type=registry,ref=$tagMain$tagLatest "
if [[ -n "$cacheref" ]] && [[ "$cachetype" = "local" ]]; then
for ref in $cacheref; do
if [[ -n "$cacheRefFrom" ]] && [[ "$cacheType" = "local" ]]; then
for ref in $cacheRefFrom; do
importCacheFlags="$importCacheFlags--cache-from=type=local,src=$ref "
done
fi
@ -77,8 +77,11 @@ if [ -n "$localmode" ]; then
fi
exportCacheFlags=""
if [[ -n "$cacheRefTo" ]] && [[ "$cacheType" = "local" ]]; then
exportCacheFlags="--cache-to=type=local,dest=$cacheRefTo "
fi
if [ "$PUSH" = "push" ]; then
exportCacheFlags="--cache-to=type=inline "
exportCacheFlags="$exportCacheFlags--cache-to=type=inline "
fi
buildxCmd build $targetFlag $importCacheFlags $exportCacheFlags $outputFlag \

4
hack/release-tar
View File

@ -18,8 +18,8 @@ if [ -z "$TAG" ] || [ -z "$OUT" ]; then
fi
importCacheFlags=""
if [[ -n "$cacheref" ]] && [[ "$cachetype" = "local" ]]; then
for ref in $cacheref; do
if [[ -n "$cacheRefFrom" ]] && [[ "$cacheType" = "local" ]]; then
for ref in $cacheRefFrom; do
importCacheFlags="$importCacheFlags--cache-from=type=local,src=$ref "
done
fi

8
hack/test
View File

@ -24,13 +24,13 @@ if [ "$TRAVIS" = "true" ]; then
fi
importCacheFlags=""
if [ -n "$cacheref" ]; then
if [ "$cachetype" = "local" ]; then
for ref in $cacheref; do
if [ -n "$cacheRefFrom" ]; then
if [ "$cacheType" = "local" ]; then
for ref in $cacheRefFrom; do
importCacheFlags="$importCacheFlags--cache-from=type=local,src=$ref "
done
else
importCacheFlags="--cache-from=type=registry,ref=$cacheref:integration-tests "
importCacheFlags="--cache-from=type=registry,ref=$cacheRefFrom:integration-tests "
fi
fi

18
hack/util
View File

@ -7,7 +7,8 @@ export BUILDX_NO_DEFAULT_LOAD=true
: ${GITHUB_ACTIONS=}
: ${TRAVIS_EVENT_TYPE=}
: ${TRAVIS_BRANCH=}
: ${CACHE_DIR=}
: ${CACHEDIR_FROM=}
: ${CACHEDIR_TO=}
if [ "$PREFER_BUILDCTL" = "1" ]; then
echo >&2 "WARNING: PREFER_BUILDCTL is no longer supported. Ignoring."
@ -38,17 +39,20 @@ buildxCmd() {
fi
}
cachetype=""
cacheref=""
cacheType=""
cacheRefFrom=""
cacheRefTo=""
currentref=""
if [ "$GITHUB_ACTIONS" = "true" ]; then
currentref="git://github.com/$GITHUB_REPOSITORY#$GITHUB_REF"
cachetype="local"
cacheref="$CACHE_DIR"
cacheType="local"
cacheRefFrom="$CACHEDIR_FROM"
cacheRefTo="$CACHEDIR_TO"
elif [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then
currentref="git://github.com/moby/buildkit#refs/pull/$TRAVIS_PULL_REQUEST/merge"
cachetype="registry"
cacheref="cicache.buildk.it/moby/buildkit/pr$TRAVIS_BUILD_ID"
cacheType="registry"
cacheRefFrom="cicache.buildk.it/moby/buildkit/pr$TRAVIS_BUILD_ID"
cacheRefTo="$cacheRefFrom"
elif [ -n "$TRAVIS_BRANCH" ]; then
currentref="git://github.com/$TRAVIS_REPO_SLUG#$TRAVIS_BRANCH"
fi