diff --git a/Makefile b/Makefile index daf12af8..072b3c9b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ BINARIES=bin/buildd bin/buildctl -BINARIES_EXTRA=bin/buildd-standalone bin/buildd-containerd bin/buildctl-darwin bin/buildd.exe bin/buildctl.exe +BINARIES_EXTRA=bin/buildd.oci_only bin/buildd.containerd_only bin/buildctl-darwin bin/buildd.exe bin/buildctl.exe DESTDIR=/usr/local binaries: $(BINARIES) diff --git a/README.md b/README.md index 54f606aa..56a19734 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The following command installs `buildd` and `buildctl` to `/usr/local/bin`: $ make && sudo make install ``` -You can also use `make binaries-all` to prepare `buildd-containerd` (containerd worker only) and `buildd-standalone` (OCI worker only). +You can also use `make binaries-all` to prepare `buildd.containerd_only` and `buildd.oci_only`. `examples/buildkit*` directory contains scripts that define how to build different configurations of BuildKit and its dependencies using the `client` package. Running one of these script generates a protobuf definition of a build graph. Note that the script itself does not execute any steps of the build. @@ -50,7 +50,7 @@ You can use `buildctl debug dump-llb` to see what data is in this definition. Ad go run examples/buildkit0/buildkit.go | buildctl debug dump-llb | jq . ``` -To start building use `buildctl build` command. The example script accepts `--target` flag to choose between `containerd` and `standalone` configurations. In standalone mode BuildKit binaries are built together with `runc`. In containerd mode, the `containerd` binary is built as well from the upstream repo. +To start building use `buildctl build` command. The example script accepts `--target` flag to choose between `containerd-worker-only` and `oci-worker-only` configurations. In OCI worker mode BuildKit binaries are built together with `runc`. In containerd worker mode, the `containerd` binary is built as well from the upstream repo. ```bash go run examples/buildkit0/buildkit.go | buildctl build @@ -143,13 +143,12 @@ buildkit can be also used by running the `buildd` daemon inside a Docker contain To run daemon in a container: ``` -docker run -d --privileged -p 1234:1234 tonistiigi/buildkit:standalone --addr tcp://0.0.0.0:1234 +docker run -d --privileged -p 1234:1234 tonistiigi/buildkit --addr tcp://0.0.0.0:1234 --oci-worker=true --containerd-worker=false export BUILDKIT_HOST=tcp://0.0.0.0:1234 buildctl build --help ``` -The `tonistiigi/buildkit:standalone` image can be built locally using the Dockerfile in `./hack/dockerfiles/test.Dockerfile`. - +The `tonistiigi/buildkit` image can be built locally using the Dockerfile in `./hack/dockerfiles/test.Dockerfile`. #### Supported runc version @@ -174,7 +173,7 @@ make test TESTPKGS=./client make test TESTPKGS=./client TESTFLAGS="--run /TestCallDiskUsage -v" # run all integration tests with a specific worker -# supported workers are standalone and containerd +# supported workers are oci and containerd make test TESTPKGS=./client TESTFLAGS="--run //worker=containerd -v" ``` diff --git a/cmd/buildd/main.go b/cmd/buildd/main.go index 117eadc2..99d2b9c3 100644 --- a/cmd/buildd/main.go +++ b/cmd/buildd/main.go @@ -250,7 +250,7 @@ func newWorkerController(c *cli.Context, wiOpt workerInitializerOpt) (*worker.Co } nWorkers := len(wc.GetAll()) if nWorkers == 0 { - return nil, errors.New("no worker found, build the buildkit daemon with tags? (e.g. \"standalone\", \"containerd\")") + return nil, errors.New("no worker found, rebuild the buildkit daemon?") } defaultWorker, err := wc.GetDefault() if err != nil { diff --git a/cmd/buildd/main_containerd.go b/cmd/buildd/main_containerd_worker.go similarity index 98% rename from cmd/buildd/main_containerd.go rename to cmd/buildd/main_containerd_worker.go index 88ab7b06..5a4d1913 100644 --- a/cmd/buildd/main_containerd.go +++ b/cmd/buildd/main_containerd_worker.go @@ -1,4 +1,4 @@ -// +build containerd +// +build linux,!no_containerd_worker package main diff --git a/cmd/buildd/main_standalone.go b/cmd/buildd/main_oci_worker.go similarity index 94% rename from cmd/buildd/main_standalone.go rename to cmd/buildd/main_oci_worker.go index 6c34d344..f5e7b371 100644 --- a/cmd/buildd/main_standalone.go +++ b/cmd/buildd/main_oci_worker.go @@ -1,6 +1,4 @@ -// +build standalone - -// TODO(AkihiroSuda): s/standalone/oci/g +// +build linux,!no_oci_worker package main diff --git a/examples/buildkit0/buildkit.go b/examples/buildkit0/buildkit.go index e2921626..2440f499 100644 --- a/examples/buildkit0/buildkit.go +++ b/examples/buildkit0/buildkit.go @@ -9,14 +9,14 @@ import ( ) type buildOpt struct { - target string - containerd string - runc string + withContainerd bool + containerd string + runc string } func main() { var opt buildOpt - flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)") + flag.BoolVar(&opt.withContainerd, "with-containerd", true, "enable containerd worker") flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version") flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version") flag.Parse() @@ -62,11 +62,11 @@ func buildkit(opt buildOpt) llb.State { Run(llb.Shlex("git clone https://github.com/moby/buildkit.git /go/src/github.com/moby/buildkit")). Dir("/go/src/github.com/moby/buildkit") - builddStandalone := src. - Run(llb.Shlex("go build -o /bin/buildd-standalone -tags standalone ./cmd/buildd")) + builddOCIWorkerOnly := src. + Run(llb.Shlex("go build -o /bin/buildd.oci_only -tags no_containerd_worker ./cmd/buildd")) - builddContainerd := src. - Run(llb.Shlex("go build -o /bin/buildd-containerd -tags containerd ./cmd/buildd")) + buildd := src. + Run(llb.Shlex("go build -o /bin/buildd ./cmd/buildd")) buildctl := src. Run(llb.Shlex("go build -o /bin/buildctl ./cmd/buildctl")) @@ -74,11 +74,11 @@ func buildkit(opt buildOpt) llb.State { r := llb.Image("docker.io/library/alpine:latest") r = copy(buildctl.Root(), "/bin/buildctl", r, "/bin/") r = copy(runc(opt.runc), "/usr/bin/runc", r, "/bin/") - if opt.target == "containerd" { + if opt.withContainerd { r = copy(containerd(opt.containerd), "/go/src/github.com/containerd/containerd/bin/containerd", r, "/bin/") - r = copy(builddContainerd.Root(), "/bin/buildd-containerd", r, "/bin/") + r = copy(buildd.Root(), "/bin/buildd", r, "/bin/") } else { - r = copy(builddStandalone.Root(), "/bin/buildd-standalone", r, "/bin/") + r = copy(builddOCIWorkerOnly.Root(), "/bin/buildd.oci_only", r, "/bin/") } return r } diff --git a/examples/buildkit1/buildkit.go b/examples/buildkit1/buildkit.go index 29948a0a..b925c64f 100644 --- a/examples/buildkit1/buildkit.go +++ b/examples/buildkit1/buildkit.go @@ -9,14 +9,14 @@ import ( ) type buildOpt struct { - target string - containerd string - runc string + withContainerd bool + containerd string + runc string } func main() { var opt buildOpt - flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)") + flag.BoolVar(&opt.withContainerd, "with-containerd", true, "enable containerd worker") flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version") flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version") flag.Parse() @@ -57,11 +57,11 @@ func containerd(version string) llb.State { func buildkit(opt buildOpt) llb.State { src := goBuildBase().With(goFromGit("github.com/moby/buildkit", "master")) - builddStandalone := src. - Run(llb.Shlex("go build -o /bin/buildd-standalone -tags standalone ./cmd/buildd")).Root() + builddOCIWorkerOnly := src. + Run(llb.Shlex("go build -o /bin/buildd.oci_only -tags no_containerd_worker ./cmd/buildd")).Root() - builddContainerd := src. - Run(llb.Shlex("go build -o /bin/buildd-containerd -tags containerd ./cmd/buildd")).Root() + buildd := src. + Run(llb.Shlex("go build -o /bin/buildd ./cmd/buildd")).Root() buildctl := src. Run(llb.Shlex("go build -o /bin/buildctl ./cmd/buildctl")).Root() @@ -71,12 +71,12 @@ func buildkit(opt buildOpt) llb.State { copyFrom(runc(opt.runc), "/usr/bin/runc", "/bin/"), ) - if opt.target == "containerd" { + if opt.withContainerd { return r.With( copyFrom(containerd(opt.containerd), "/go/src/github.com/containerd/containerd/bin/containerd", "/bin/"), - copyFrom(builddContainerd, "/bin/buildd-containerd", "/bin/")) + copyFrom(buildd, "/bin/buildd", "/bin/")) } - return r.With(copyFrom(builddStandalone, "/bin/buildd-standalone", "/bin/")) + return r.With(copyFrom(builddOCIWorkerOnly, "/bin/buildd.oci_only", "/bin/")) } // goFromGit is a helper for cloning a git repo, checking out a tag and copying diff --git a/examples/buildkit2/buildkit.go b/examples/buildkit2/buildkit.go index 1dd3aa2f..74ddaa70 100644 --- a/examples/buildkit2/buildkit.go +++ b/examples/buildkit2/buildkit.go @@ -9,14 +9,14 @@ import ( ) type buildOpt struct { - target string - containerd string - runc string + withContainerd bool + containerd string + runc string } func main() { var opt buildOpt - flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)") + flag.BoolVar(&opt.withContainerd, "with-containerd", true, "enable containerd worker") flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version") flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version") flag.Parse() @@ -66,9 +66,9 @@ func containerd(version string) llb.State { func buildkit(opt buildOpt) llb.State { run := goRepo(goBuildBase(), "github.com/moby/buildkit", "master") - builddStandalone := run(llb.Shlex("go build -o ./bin/buildd-standalone -tags standalone ./cmd/buildd")) + builddOCIWorkerOnly := run(llb.Shlex("go build -o ./bin/buildd.oci_only -tags no_containerd_worker ./cmd/buildd")) - builddContainerd := run(llb.Shlex("go build -o ./bin/buildd-containerd -tags containerd ./cmd/buildd")) + buildd := run(llb.Shlex("go build -o ./bin/buildd ./cmd/buildd")) buildctl := run(llb.Shlex("go build -o ./bin/buildctl ./cmd/buildctl")) @@ -77,12 +77,12 @@ func buildkit(opt buildOpt) llb.State { copyAll(runc(opt.runc), "/bin"), ) - if opt.target == "containerd" { + if opt.withContainerd { return r.With( copyAll(containerd(opt.containerd), "/bin"), - copyAll(builddContainerd, "/bin")) + copyAll(buildd, "/bin")) } - return r.With(copyAll(builddStandalone, "/bin")) + return r.With(copyAll(builddOCIWorkerOnly, "/bin")) } func copyAll(src llb.State, destPath string) llb.StateOption { diff --git a/examples/buildkit3/buildkit.go b/examples/buildkit3/buildkit.go index c1c8bf5b..b012f423 100644 --- a/examples/buildkit3/buildkit.go +++ b/examples/buildkit3/buildkit.go @@ -9,15 +9,15 @@ import ( ) type buildOpt struct { - target string - containerd string - runc string - buildkit string + withContainerd bool + containerd string + runc string + buildkit string } func main() { var opt buildOpt - flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)") + flag.BoolVar(&opt.withContainerd, "with-containerd", true, "enable containerd worker") flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version") flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version") flag.StringVar(&opt.buildkit, "buildkit", "master", "buildkit version") @@ -82,9 +82,9 @@ func buildkit(opt buildOpt) llb.State { } run := goRepo(goBuildBase(), repo, src) - builddStandalone := run(llb.Shlex("go build -o /out/buildd-standalone -tags standalone ./cmd/buildd")) + builddOCIWorkerOnly := run(llb.Shlex("go build -o /out/buildd.oci_only -tags no_containerd_worker ./cmd/buildd")) - builddContainerd := run(llb.Shlex("go build -o /out/buildd-containerd -tags containerd ./cmd/buildd")) + buildd := run(llb.Shlex("go build -o /out/buildd ./cmd/buildd")) buildctl := run(llb.Shlex("go build -o /out/buildctl ./cmd/buildctl")) @@ -93,12 +93,12 @@ func buildkit(opt buildOpt) llb.State { copyAll(runc(opt.runc), "/"), ) - if opt.target == "containerd" { + if opt.withContainerd { return r.With( copyAll(containerd(opt.containerd), "/"), - copyAll(builddContainerd, "/")) + copyAll(buildd, "/")) } - return r.With(copyAll(builddStandalone, "/")) + return r.With(copyAll(builddOCIWorkerOnly, "/")) } func copyAll(src llb.State, destPath string) llb.StateOption { diff --git a/examples/gobuild/main.go b/examples/gobuild/main.go index 6a3b6f53..fe7669a9 100644 --- a/examples/gobuild/main.go +++ b/examples/gobuild/main.go @@ -35,7 +35,7 @@ func run() error { Source: src, MountPath: "/go/src/github.com/moby/buildkit", Pkg: "github.com/moby/buildkit/cmd/buildd", - BuildTags: []string{"standalone"}, + BuildTags: []string{"no_containerd_worker"}, }) if err != nil { return err diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index febea396..57a30fb0 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -609,7 +609,7 @@ RUN ["ls"] cmd := sb.Cmd(args + " --exporter=image --exporter-opt=name=" + target) require.NoError(t, cmd.Run()) - // TODO: expose this test to standalone + // TODO: expose this test to OCI worker var cdAddress string if cd, ok := sb.(interface { diff --git a/hack/dockerfiles/test.Dockerfile b/hack/dockerfiles/test.Dockerfile index 006c3b2e..31f4f1aa 100644 --- a/hack/dockerfiles/test.Dockerfile +++ b/hack/dockerfiles/test.Dockerfile @@ -1,6 +1,6 @@ ARG RUNC_VERSION=74a17296470088de3805e138d3d87c62e613dfc4 ARG CONTAINERD_VERSION=v1.0.0 -# available targets: buildd (standalone+containerd), buildd-standalone, buildd-containerd +# available targets: buildd, buildd.oci_only, buildd.containerd_only ARG BUILDKIT_TARGET=buildd ARG REGISTRY_VERSION=2.6 @@ -22,7 +22,7 @@ RUN go build -ldflags '-d' -o /usr/bin/buildctl ./cmd/buildctl FROM buildkit-base AS buildd ENV CGO_ENABLED=0 -RUN go build -ldflags '-d' -o /usr/bin/buildd -tags "standalone containerd" ./cmd/buildd +RUN go build -ldflags '-d' -o /usr/bin/buildd ./cmd/buildd # test dependencies begin here FROM gobuild-base AS runc @@ -47,20 +47,19 @@ COPY --from=runc /usr/bin/runc /usr/bin/runc COPY --from=containerd /go/src/github.com/containerd/containerd/bin/containerd* /usr/bin/ -FROM buildkit-base AS buildd-standalone +FROM buildkit-base AS buildd.oci_only ENV CGO_ENABLED=0 -RUN go build -ldflags '-d' -o /usr/bin/buildd-standalone -tags standalone ./cmd/buildd +RUN go build -ldflags '-d' -o /usr/bin/buildd.oci_only -tags no_containerd_worker ./cmd/buildd -FROM buildkit-base AS buildd-containerd +FROM buildkit-base AS buildd.containerd_only ENV CGO_ENABLED=0 -RUN go build -ldflags '-d' -o /usr/bin/buildd-containerd -tags containerd ./cmd/buildd +RUN go build -ldflags '-d' -o /usr/bin/buildd.containerd_only -tags no_oci_worker ./cmd/buildd FROM registry:$REGISTRY_VERSION AS registry FROM unit-tests AS integration-tests COPY --from=buildctl /usr/bin/buildctl /usr/bin/ -COPY --from=buildd-containerd /usr/bin/buildd-containerd /usr/bin -COPY --from=buildd-standalone /usr/bin/buildd-standalone /usr/bin +COPY --from=buildd /usr/bin/buildd /usr/bin COPY --from=registry /bin/registry /usr/bin FROM gobuild-base AS cross-windows @@ -78,24 +77,25 @@ FROM alpine AS buildkit-export RUN apk add --no-cache git VOLUME /var/lib/buildkit -# Copy together all binaries for standalone+containerd mode +# Copy together all binaries for oci+containerd mode FROM buildkit-export AS buildkit-buildd COPY --from=runc /usr/bin/runc /usr/bin/ COPY --from=buildd /usr/bin/buildd /usr/bin/ COPY --from=buildctl /usr/bin/buildctl /usr/bin/ +ENTRYPOINT ["buildd"] -# Copy together all binaries needed for standalone mode -FROM buildkit-export AS buildkit-buildd-standalone -COPY --from=buildd-standalone /usr/bin/buildd-standalone /usr/bin/ +# Copy together all binaries needed for oci worker mode +FROM buildkit-export AS buildkit-buildd.oci_only +COPY --from=buildd.oci_only /usr/bin/buildd.oci_only /usr/bin/ COPY --from=buildctl /usr/bin/buildctl /usr/bin/ -ENTRYPOINT ["buildd-standalone"] +ENTRYPOINT ["buildd.oci_only"] -# Copy together all binaries for containerd mode -FROM buildkit-export AS buildkit-buildd-containerd +# Copy together all binaries for containerd worker mode +FROM buildkit-export AS buildkit-buildd.containerd_only COPY --from=runc /usr/bin/runc /usr/bin/ -COPY --from=buildd-containerd /usr/bin/buildd-containerd /usr/bin/ +COPY --from=buildd.containerd_only /usr/bin/buildd.containerd_only /usr/bin/ COPY --from=buildctl /usr/bin/buildctl /usr/bin/ -ENTRYPOINT ["buildd-containerd"] +ENTRYPOINT ["buildd.containerd_only"] FROM alpine AS containerd-runtime COPY --from=runc /usr/bin/runc /usr/bin/ diff --git a/hack/test b/hack/test index 050dbb78..c5968662 100755 --- a/hack/test +++ b/hack/test @@ -5,7 +5,7 @@ set -eu -o pipefail -x # update this to iidfile after 17.06 docker build -t buildkit:test --target integration-tests -f ./hack/dockerfiles/test.Dockerfile --force-rm . -docker run --rm -v /tmp --privileged buildkit:test go test -tags standalone ${TESTFLAGS:--v} ${TESTPKGS:-./...} +docker run --rm -v /tmp --privileged buildkit:test go test -tags no_containerd_worker ${TESTFLAGS:--v} ${TESTPKGS:-./...} docker run --rm buildkit:test go build ./frontend/gateway/client docker run --rm buildkit:test go build ./frontend/dockerfile/cmd/dockerfile-frontend diff --git a/util/testutil/integration/containerd.go b/util/testutil/integration/containerd.go index 5941fc21..799758b2 100644 --- a/util/testutil/integration/containerd.go +++ b/util/testutil/integration/containerd.go @@ -24,7 +24,7 @@ func (c *containerd) New() (sb Sandbox, cl func() error, err error) { if err := lookupBinary("containerd"); err != nil { return nil, nil, err } - if err := lookupBinary("buildd-containerd"); err != nil { + if err := lookupBinary("buildd"); err != nil { return nil, nil, err } if err := requireRoot(); err != nil { @@ -64,7 +64,10 @@ func (c *containerd) New() (sb Sandbox, cl func() error, err error) { return nil, nil, err } - builddSock, stop, err := runBuildd([]string{"buildd-containerd", "--containerd-worker-addr", address}, logs) + builddSock, stop, err := runBuildd([]string{"buildd", + "--oci-worker=false", + "--containerd-worker=true", + "--containerd-worker-addr", address}, logs) if err != nil { return nil, nil, err } diff --git a/util/testutil/integration/standalone.go b/util/testutil/integration/oci.go similarity index 87% rename from util/testutil/integration/standalone.go rename to util/testutil/integration/oci.go index cf49e8b3..83e86298 100644 --- a/util/testutil/integration/standalone.go +++ b/util/testutil/integration/oci.go @@ -15,25 +15,25 @@ import ( ) func init() { - register(&standalone{}) + register(&oci{}) } -type standalone struct { +type oci struct { } -func (s *standalone) Name() string { - return "standalone" +func (s *oci) Name() string { + return "oci" } -func (s *standalone) New() (Sandbox, func() error, error) { - if err := lookupBinary("buildd-standalone"); err != nil { +func (s *oci) New() (Sandbox, func() error, error) { + if err := lookupBinary("buildd"); err != nil { return nil, nil, err } if err := requireRoot(); err != nil { return nil, nil, err } logs := map[string]*bytes.Buffer{} - builddSock, stop, err := runBuildd([]string{"buildd-standalone"}, logs) + builddSock, stop, err := runBuildd([]string{"buildd", "--oci-worker=true", "--containerd-worker=false"}, logs) if err != nil { return nil, nil, err } diff --git a/worker/runc/runc_test.go b/worker/runc/runc_test.go index 16218d25..64fe979e 100644 --- a/worker/runc/runc_test.go +++ b/worker/runc/runc_test.go @@ -1,4 +1,4 @@ -// +build linux,standalone +// +build linux,no_containerd_worker package runc