hack: update scripts to be go mod compatible

Signed-off-by: Tibor Vass <tibor@docker.com>
2019-01-26 06:29:13 +00:00 · 2019-01-26 06:29:13 +00:00 · 175506ff7f
parent ad8b35e3d4
commit 175506ff7f
11 changed files with 128 additions and 48 deletions
--- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile
+++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile
@ -4,10 +4,12 @@ FROM --platform=$BUILDPLATFORM tonistiigi/xx:golang@sha256:6f7d999551dd471b58f70

 FROM --platform=$BUILDPLATFORM golang:1.11 AS base
 COPY --from=xgo / /
-WORKDIR /go/src/github.com/moby/buildkit
+WORKDIR /src
+ENV GOFLAGS=-mod=vendor

 FROM base AS version
 ARG CHANNEL
+# TODO: PKG should be inferred from go modules
 RUN --mount=target=. \ 
  PKG=github.com/moby/buildkit/frontend/dockerfile/cmd/dockerfile-frontend VERSION=$(./frontend/dockerfile/cmd/dockerfile-frontend/hack/detect "$CHANNEL") REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi); \
  echo "-X main.Version=${VERSION} -X main.Revision=${REVISION} -X main.Package=${PKG}" | tee /tmp/.ldflags; \
@ -19,6 +21,7 @@ ARG BUILDTAGS=""
 ARG TARGETPLATFORM
 ENV TARGETPLATFORM=$TARGETPLATFORM
 RUN --mount=target=. --mount=type=cache,target=/root/.cache \
+  --mount=target=/go/pkg/mod,type=cache \
  --mount=source=/tmp/.ldflags,target=/tmp/.ldflags,from=version \
  CGO_ENABLED=0 go build -o /dockerfile-frontend -ldflags "-d $(cat /tmp/.ldflags)" -tags "$BUILDTAGS netgo static_build osusergo" ./frontend/dockerfile/cmd/dockerfile-frontend && \
  file /dockerfile-frontend | grep "statically linked"
--- a/hack/dockerfiles/test.Dockerfile
+++ b/hack/dockerfiles/test.Dockerfile
@ -15,11 +15,13 @@ RUN apk add --no-cache g++ linux-headers
 RUN apk add --no-cache git libseccomp-dev make

 FROM gobuild-base AS buildkit-base
-WORKDIR /go/src/github.com/moby/buildkit
+WORKDIR /src
 COPY . .
+# TODO: PKG should be inferred from go modules
 RUN mkdir .tmp; \
  PKG=github.com/moby/buildkit VERSION=$(git describe --match 'v[0-9]*' --dirty='.m' --always) REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi); \
  echo "-X ${PKG}/version.Version=${VERSION} -X ${PKG}/version.Revision=${REVISION} -X ${PKG}/version.Package=${PKG}" | tee .tmp/ldflags
+ENV GOFLAGS=-mod=vendor

 FROM buildkit-base AS buildctl
 ENV CGO_ENABLED=0
@ -68,7 +70,7 @@ RUN git checkout -q "$CONTAINERD10_VERSION" \
 FROM buildkit-base AS buildkitd.oci_only
 ENV CGO_ENABLED=1
 # mitigate https://github.com/moby/moby/pull/35456
-WORKDIR /go/src/github.com/moby/buildkit
+WORKDIR /src
 RUN go build -installsuffix netgo -ldflags "$(cat .tmp/ldflags) -w -extldflags -static" -tags 'no_containerd_worker seccomp netgo cgo static_build' -o /usr/bin/buildkitd.oci_only ./cmd/buildkitd

 FROM buildkit-base AS buildkitd.containerd_only
--- a/hack/dockerfiles/test.buildkit.Dockerfile
+++ b/hack/dockerfiles/test.buildkit.Dockerfile
@ -66,10 +66,12 @@ RUN --mount=from=runc-src,src=/usr/src/runc,target=. --mount=target=/root/.cache
  file /usr/bin/runc | grep "statically linked"

 FROM gobuild-base AS buildkit-base
-WORKDIR /go/src/github.com/moby/buildkit
+WORKDIR /src
+ENV GOFLAGS=-mod=vendor

 # scan the version/revision info
 FROM buildkit-base AS buildkit-version
+# TODO: PKG should be inferred from go modules
 RUN --mount=target=. \
  PKG=github.com/moby/buildkit VERSION=$(git describe --match 'v[0-9]*' --dirty='.m' --always --tags) REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi); \
  echo "-X ${PKG}/version.Version=${VERSION} -X ${PKG}/version.Revision=${REVISION} -X ${PKG}/version.Package=${PKG}" | tee /tmp/.ldflags; \
@ -80,6 +82,7 @@ FROM buildkit-base AS buildctl
 ENV CGO_ENABLED=0
 ARG TARGETPLATFORM
 RUN --mount=target=. --mount=target=/root/.cache,type=cache \
+  --mount=target=/go/pkg/mod,type=cache \
  --mount=source=/tmp/.ldflags,target=/tmp/.ldflags,from=buildkit-version \
  set -x; go build -ldflags "$(cat /tmp/.ldflags)" -o /usr/bin/buildctl ./cmd/buildctl && \
  file /usr/bin/buildctl && file /usr/bin/buildctl | egrep "statically linked|Mach-O|Windows"
@ -89,6 +92,7 @@ FROM buildkit-base AS buildkitd
 ENV CGO_ENABLED=1
 ARG TARGETPLATFORM
 RUN --mount=target=. --mount=target=/root/.cache,type=cache \
+  --mount=target=/go/pkg/mod,type=cache \
  --mount=source=/tmp/.ldflags,target=/tmp/.ldflags,from=buildkit-version \
  go build -ldflags "$(cat /tmp/.ldflags) -w -extldflags -static" -tags 'osusergo seccomp netgo cgo static_build ' -o /usr/bin/buildkitd ./cmd/buildkitd && \
  file /usr/bin/buildkitd | grep "statically linked"
--- a/hack/dockerfiles/vendor.Dockerfile
+++ b/hack/dockerfiles/vendor.Dockerfile
@ -1,13 +1,8 @@
-FROM golang:1.11-alpine AS vndr
+FROM golang:1.11-alpine AS update
 RUN  apk add --no-cache git
-# NOTE: hack scripts override VNDR_VERSION to a specific revision
-ARG VNDR_VERSION=master
-RUN go get -d github.com/LK4D4/vndr \
-  && cd /go/src/github.com/LK4D4/vndr \
-	&& git checkout $VNDR_VERSION \
-	&& go install ./
-WORKDIR /go/src/github.com/moby/buildkit
+WORKDIR /src
 COPY . .
-# Remove vendor first to workaround  https://github.com/LK4D4/vndr/issues/63.
-RUN rm -rf vendor
-RUN vndr --verbose --strict
+RUN go mod tidy && go mod vendor && rm -rf /go/pkg/mod && ln -s /src /out
+
+FROM update AS validate
+RUN ./hack/validate-vendor check
--- a/hack/dockerfiles/vendor.buildkit.Dockerfile
+++ b/hack/dockerfiles/vendor.buildkit.Dockerfile
@ -0,0 +1,18 @@
+# syntax = docker/dockerfile:1.0-experimental
+FROM golang:1.11-alpine AS vendored
+RUN  apk add --no-cache git
+WORKDIR /src
+RUN --mount=target=/src,rw \
+  --mount=target=/go/pkg/mod,type=cache \
+  go mod tidy && go mod vendor && \
+  mkdir /out && cp -r go.mod go.sum vendor /out
+
+FROM scratch AS update
+COPY --from=vendored /out /out
+
+FROM vendored AS validate
+RUN --mount=target=.,rw \
+  git add -A && \
+  rm -rf vendor && \
+  cp -rf /out/* . && \
+  ./hack/validate-vendor check
--- a/hack/test
+++ b/hack/test
@ -62,7 +62,7 @@ case $buildmode in
  ;;
 esac

-cacheVolume=$(docker create -v /root/.cache -v /root/.cache/registry alpine)
+cacheVolume=$(docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod alpine)

 if [ "$TEST_INTEGRATION" == 1 ]; then
  docker run --rm -v /tmp --volumes-from=$cacheVolume -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test ${TESTFLAGS:--v} ${TESTPKGS:-./...}
@ -70,7 +70,7 @@ fi


 if [ "$TEST_GATEWAY" == 1 ]; then
-  docker run --rm $iid go build ./frontend/gateway/client
+  docker run --rm --volumes-from=$cacheVolume $iid go build ./frontend/gateway/client
 fi


--- a/hack/update-generated-files
+++ b/hack/update-generated-files
@ -8,7 +8,7 @@ set -eu -o pipefail -x
 progressFlag=""
 if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi

-gogo_version=$(awk '$1 == "github.com/gogo/protobuf" { print $2 }' vendor.conf)
+gogo_version=$(awk '$1 == "github.com/gogo/protobuf" { print $2 }' go.mod)
 case $buildmode in
 "buildkit")
  output=$(mktemp -d -t buildctl-output.XXXXXXXXXX)
--- a/hack/update-vendor
+++ b/hack/update-vendor
@ -1,12 +1,44 @@
 #!/usr/bin/env bash

+. $(dirname $0)/util
 set -eu -o pipefail -x

-iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
-docker build --build-arg VNDR_VERSION=1fc68ee0c852556a9ed53cbde16247033f104111 --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --force-rm .
-iid=$(cat $iidfile)
-cid=$(docker create $iid noop)
-rm -rf ./vendor
-docker cp $cid:/go/src/github.com/moby/buildkit/vendor .
-docker rm $cid
-rm -f $iidfile
+: ${CONTINUOUS_INTEGRATION=}
+
+progressFlag=""
+if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
+
+case $buildmode in
+"buildkit")
+   output=$(mktemp -d -t buildctl-output.XXXXXXXXXX)
+  buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \
+    --frontend-opt target=update \
+    --frontend-opt filename=./hack/dockerfiles/vendor.buildkit.Dockerfile \
+    --exporter=local --exporter-opt output=$output
+  rm -rf ./vendor
+  cp -R "$output/out/" .
+  rm -rf $output
+  ;;
+*)
+  iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
+  case $buildmode in
+  "docker-buildkit")
+    export DOCKER_BUILDKIT=1
+    docker build --iidfile $iidfile -f ./hack/dockerfiles/vendor.buildkit.Dockerfile --target update --force-rm .
+    ;;
+  *)
+    docker build --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --target update --force-rm .
+    ;;
+  esac
+  iid=$(cat $iidfile)
+  cid=$(docker create $iid noop)
+  rm -rf ./vendor
+
+  docker cp $cid:/out/go.mod .
+  docker cp $cid:/out/go.sum .
+  docker cp $cid:/out/vendor .
+
+  docker rm $cid
+  rm -f $iidfile
+  ;;
+esac
--- a/hack/util
+++ b/hack/util
@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh

 : ${PREFER_BUILDCTL=}
 : ${PREFER_LEGACY=}
@ -8,7 +8,7 @@ newerEqualThan() { # $1=minimum wanted version $2=actual-version
 }

 buildmode="legacy"
-if [ "$PREFER_BUILDCTL" == "1" ]; then
+if [ "$PREFER_BUILDCTL" = "1" ]; then
  buildmode="buildkit";
 else
  serverVersion=$(docker info --format '{{.ServerVersion}}')
--- a/hack/validate-generated-files
+++ b/hack/validate-generated-files
@ -8,10 +8,10 @@ if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"
 case ${1:-} in
 '')
  . $(dirname $0)/util
-  gogo_version=$(awk '$1 == "github.com/gogo/protobuf" { print $2 }' vendor.conf)
+  gogo_version=$(awk '$1 == "github.com/gogo/protobuf" { print $2 }' go.mod)
  case $buildmode in
  "buildkit")
-    buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. --frontend-opt build-arg:GOGO_VERSION=$gogo_version --frontend-opt filename=./hack/dockerfiles/generated-files.buildkit.Dockerfile
+    buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. --frontend-opt build-arg:GOGO_VERSION=$gogo_version --frontend-opt target=validate --frontend-opt filename=./hack/dockerfiles/generated-files.buildkit.Dockerfile
    ;;
  "docker-buildkit")
    export DOCKER_BUILDKIT=1
--- a/hack/validate-vendor
+++ b/hack/validate-vendor
@ -1,22 +1,48 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh

-set -eu -o pipefail -x
+set -eu

-iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
-docker build --build-arg VNDR_VERSION=1fc68ee0c852556a9ed53cbde16247033f104111 --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --force-rm .
-iid=$(cat $iidfile)
-diffs="$(docker run --rm $iid git status --porcelain -- vendor 2>/dev/null)"
-if [ "$diffs" ]; then
-    {
-        set +x
-        echo 'The result of vndr differs'
+: ${CONTINUOUS_INTEGRATION=}
+: ${DOCKER_BUILDKIT=}
+
+progressFlag=""
+if [ "$CONTINUOUS_INTEGRATION" = "true" ]; then progressFlag="--progress=plain"; fi
+
+case ${1:-} in
+'')
+  . $(dirname $0)/util
+  case $buildmode in
+  "buildkit")
+    buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. --frontend-opt filename=./hack/dockerfiles/vendor.buildkit.Dockerfile --frontend-opt target=validate
+    ;;
+  "docker-buildkit")
+    export DOCKER_BUILDKIT=1
+    iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
+    docker build --iidfile $iidfile -f ./hack/dockerfiles/vendor.buildkit.Dockerfile --target validate --force-rm . || exit 1
+    iid=$(cat $iidfile)
+    docker rmi $iid
+    rm -f $iidfile
+    ;;
+  *)
+    docker build -f ./hack/dockerfiles/vendor.Dockerfile --target validate --force-rm .
+    ;;
+  esac
+  ;;
+check)
+  status="$(git status --porcelain -- go.mod go.sum vendor 2>/dev/null)"
+  diffs=$(echo "$status" | grep -v '^[RAD] ' || true)
+  if [ "$diffs" ]; then
+      {
+	set +x
+        echo 'The result of "make vendor" differs'
        echo
        echo "$diffs"
        echo
-        echo 'Please vendor your package with github.com/LK4D4/vndr.'
+        echo 'Please vendor your package with "make vendor"'
        echo
-    } >&2
-    false
-fi
-echo 'Congratulations! All vendoring changes are done the right way.'
-rm -f $iidfile
+      } >&2
+      exit 1
+  fi
+  echo 'Congratulations! All vendoring changes are done the right way.'
+  ;;
+esac