JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

*: update build tags (`containerd` -> `no_oci_worker`) 

Signed-off-by: Akihiro Suda <suda.akihiro@lab.ntt.co.jp>
docker-18.09
Akihiro Suda 2017-12-15 16:00:13 +09:00
parent 1281fcdf6c
commit 2a33b2e822
16 changed files with 83 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -1,5 +1,5 @@
BINARIES=bin/buildd bin/buildctl
BINARIES_EXTRA=bin/buildd-standalone bin/buildd-containerd bin/buildctl-darwin bin/buildd.exe bin/buildctl.exe
BINARIES_EXTRA=bin/buildd.oci_only bin/buildd.containerd_only bin/buildctl-darwin bin/buildd.exe bin/buildctl.exe
DESTDIR=/usr/local
binaries: $(BINARIES)

11
README.md
View File

@ -40,7 +40,7 @@ The following command installs `buildd` and `buildctl` to `/usr/local/bin`:
$ make && sudo make install
```
You can also use `make binaries-all` to prepare `buildd-containerd` (containerd worker only) and `buildd-standalone` (OCI worker only).
You can also use `make binaries-all` to prepare `buildd.containerd_only` and `buildd.oci_only`.
`examples/buildkit*` directory contains scripts that define how to build different configurations of BuildKit and its dependencies using the `client` package. Running one of these script generates a protobuf definition of a build graph. Note that the script itself does not execute any steps of the build.
@ -50,7 +50,7 @@ You can use `buildctl debug dump-llb` to see what data is in this definition. Ad
go run examples/buildkit0/buildkit.go | buildctl debug dump-llb | jq .
```
To start building use `buildctl build` command. The example script accepts `--target` flag to choose between `containerd` and `standalone` configurations. In standalone mode BuildKit binaries are built together with `runc`. In containerd mode, the `containerd` binary is built as well from the upstream repo.
To start building use `buildctl build` command. The example script accepts `--target` flag to choose between `containerd-worker-only` and `oci-worker-only` configurations. In OCI worker mode BuildKit binaries are built together with `runc`. In containerd worker mode, the `containerd` binary is built as well from the upstream repo.
```bash
go run examples/buildkit0/buildkit.go | buildctl build
@ -143,13 +143,12 @@ buildkit can be also used by running the `buildd` daemon inside a Docker contain
To run daemon in a container:
```
docker run -d --privileged -p 1234:1234 tonistiigi/buildkit:standalone --addr tcp://0.0.0.0:1234
docker run -d --privileged -p 1234:1234 tonistiigi/buildkit --addr tcp://0.0.0.0:1234 --oci-worker=true --containerd-worker=false
export BUILDKIT_HOST=tcp://0.0.0.0:1234
buildctl build --help
```
The `tonistiigi/buildkit:standalone` image can be built locally using the Dockerfile in `./hack/dockerfiles/test.Dockerfile`.
The `tonistiigi/buildkit` image can be built locally using the Dockerfile in `./hack/dockerfiles/test.Dockerfile`.
#### Supported runc version
@ -174,7 +173,7 @@ make test TESTPKGS=./client
make test TESTPKGS=./client TESTFLAGS="--run /TestCallDiskUsage -v"
# run all integration tests with a specific worker
# supported workers are standalone and containerd
# supported workers are oci and containerd
make test TESTPKGS=./client TESTFLAGS="--run //worker=containerd -v"
```

2
cmd/buildd/main.go
View File

@ -250,7 +250,7 @@ func newWorkerController(c *cli.Context, wiOpt workerInitializerOpt) (*worker.Co
}
nWorkers := len(wc.GetAll())
if nWorkers == 0 {
return nil, errors.New("no worker found, build the buildkit daemon with tags? (e.g. \"standalone\", \"containerd\")")
return nil, errors.New("no worker found, rebuild the buildkit daemon?")
}
defaultWorker, err := wc.GetDefault()
if err != nil {

2
cmd/buildd/main_containerd.go → cmd/buildd/main_containerd_worker.go
View File

@ -1,4 +1,4 @@
// +build containerd
// +build linux,!no_containerd_worker
package main

4
cmd/buildd/main_standalone.go → cmd/buildd/main_oci_worker.go
View File

@ -1,6 +1,4 @@
// +build standalone
// TODO(AkihiroSuda): s/standalone/oci/g
// +build linux,!no_oci_worker
package main

22
examples/buildkit0/buildkit.go
View File

@ -9,14 +9,14 @@ import (
)
type buildOpt struct {
target string
containerd string
runc string
withContainerd bool
containerd string
runc string
}
func main() {
var opt buildOpt
flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
flag.BoolVar(&opt.withContainerd, "with-containerd", true, "enable containerd worker")
flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version")
flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
flag.Parse()
@ -62,11 +62,11 @@ func buildkit(opt buildOpt) llb.State {
Run(llb.Shlex("git clone https://github.com/moby/buildkit.git /go/src/github.com/moby/buildkit")).
Dir("/go/src/github.com/moby/buildkit")
builddStandalone := src.
Run(llb.Shlex("go build -o /bin/buildd-standalone -tags standalone ./cmd/buildd"))
builddOCIWorkerOnly := src.
Run(llb.Shlex("go build -o /bin/buildd.oci_only -tags no_containerd_worker ./cmd/buildd"))
builddContainerd := src.
Run(llb.Shlex("go build -o /bin/buildd-containerd -tags containerd ./cmd/buildd"))
buildd := src.
Run(llb.Shlex("go build -o /bin/buildd ./cmd/buildd"))
buildctl := src.
Run(llb.Shlex("go build -o /bin/buildctl ./cmd/buildctl"))
@ -74,11 +74,11 @@ func buildkit(opt buildOpt) llb.State {
r := llb.Image("docker.io/library/alpine:latest")
r = copy(buildctl.Root(), "/bin/buildctl", r, "/bin/")
r = copy(runc(opt.runc), "/usr/bin/runc", r, "/bin/")
if opt.target == "containerd" {
if opt.withContainerd {
r = copy(containerd(opt.containerd), "/go/src/github.com/containerd/containerd/bin/containerd", r, "/bin/")
r = copy(builddContainerd.Root(), "/bin/buildd-containerd", r, "/bin/")
r = copy(buildd.Root(), "/bin/buildd", r, "/bin/")
} else {
r = copy(builddStandalone.Root(), "/bin/buildd-standalone", r, "/bin/")
r = copy(builddOCIWorkerOnly.Root(), "/bin/buildd.oci_only", r, "/bin/")
}
return r
}

22
examples/buildkit1/buildkit.go
View File

@ -9,14 +9,14 @@ import (
)
type buildOpt struct {
target string
containerd string
runc string
withContainerd bool
containerd string
runc string
}
func main() {
var opt buildOpt
flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
flag.BoolVar(&opt.withContainerd, "with-containerd", true, "enable containerd worker")
flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version")
flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
flag.Parse()
@ -57,11 +57,11 @@ func containerd(version string) llb.State {
func buildkit(opt buildOpt) llb.State {
src := goBuildBase().With(goFromGit("github.com/moby/buildkit", "master"))
builddStandalone := src.
Run(llb.Shlex("go build -o /bin/buildd-standalone -tags standalone ./cmd/buildd")).Root()
builddOCIWorkerOnly := src.
Run(llb.Shlex("go build -o /bin/buildd.oci_only -tags no_containerd_worker ./cmd/buildd")).Root()
builddContainerd := src.
Run(llb.Shlex("go build -o /bin/buildd-containerd -tags containerd ./cmd/buildd")).Root()
buildd := src.
Run(llb.Shlex("go build -o /bin/buildd ./cmd/buildd")).Root()
buildctl := src.
Run(llb.Shlex("go build -o /bin/buildctl ./cmd/buildctl")).Root()
@ -71,12 +71,12 @@ func buildkit(opt buildOpt) llb.State {
copyFrom(runc(opt.runc), "/usr/bin/runc", "/bin/"),
)
if opt.target == "containerd" {
if opt.withContainerd {
return r.With(
copyFrom(containerd(opt.containerd), "/go/src/github.com/containerd/containerd/bin/containerd", "/bin/"),
copyFrom(builddContainerd, "/bin/buildd-containerd", "/bin/"))
copyFrom(buildd, "/bin/buildd", "/bin/"))
}
return r.With(copyFrom(builddStandalone, "/bin/buildd-standalone", "/bin/"))
return r.With(copyFrom(builddOCIWorkerOnly, "/bin/buildd.oci_only", "/bin/"))
}
// goFromGit is a helper for cloning a git repo, checking out a tag and copying

18
examples/buildkit2/buildkit.go
View File

@ -9,14 +9,14 @@ import (
)
type buildOpt struct {
target string
containerd string
runc string
withContainerd bool
containerd string
runc string
}
func main() {
var opt buildOpt
flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
flag.BoolVar(&opt.withContainerd, "with-containerd", true, "enable containerd worker")
flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version")
flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
flag.Parse()
@ -66,9 +66,9 @@ func containerd(version string) llb.State {
func buildkit(opt buildOpt) llb.State {
run := goRepo(goBuildBase(), "github.com/moby/buildkit", "master")
builddStandalone := run(llb.Shlex("go build -o ./bin/buildd-standalone -tags standalone ./cmd/buildd"))
builddOCIWorkerOnly := run(llb.Shlex("go build -o ./bin/buildd.oci_only -tags no_containerd_worker ./cmd/buildd"))
builddContainerd := run(llb.Shlex("go build -o ./bin/buildd-containerd -tags containerd ./cmd/buildd"))
buildd := run(llb.Shlex("go build -o ./bin/buildd ./cmd/buildd"))
buildctl := run(llb.Shlex("go build -o ./bin/buildctl ./cmd/buildctl"))
@ -77,12 +77,12 @@ func buildkit(opt buildOpt) llb.State {
copyAll(runc(opt.runc), "/bin"),
)
if opt.target == "containerd" {
if opt.withContainerd {
return r.With(
copyAll(containerd(opt.containerd), "/bin"),
copyAll(builddContainerd, "/bin"))
copyAll(buildd, "/bin"))
}
return r.With(copyAll(builddStandalone, "/bin"))
return r.With(copyAll(builddOCIWorkerOnly, "/bin"))
}
func copyAll(src llb.State, destPath string) llb.StateOption {

20
examples/buildkit3/buildkit.go
View File

@ -9,15 +9,15 @@ import (
)
type buildOpt struct {
target string
containerd string
runc string
buildkit string
withContainerd bool
containerd string
runc string
buildkit string
}
func main() {
var opt buildOpt
flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
flag.BoolVar(&opt.withContainerd, "with-containerd", true, "enable containerd worker")
flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version")
flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
flag.StringVar(&opt.buildkit, "buildkit", "master", "buildkit version")
@ -82,9 +82,9 @@ func buildkit(opt buildOpt) llb.State {
}
run := goRepo(goBuildBase(), repo, src)
builddStandalone := run(llb.Shlex("go build -o /out/buildd-standalone -tags standalone ./cmd/buildd"))
builddOCIWorkerOnly := run(llb.Shlex("go build -o /out/buildd.oci_only -tags no_containerd_worker ./cmd/buildd"))
builddContainerd := run(llb.Shlex("go build -o /out/buildd-containerd -tags containerd ./cmd/buildd"))
buildd := run(llb.Shlex("go build -o /out/buildd ./cmd/buildd"))
buildctl := run(llb.Shlex("go build -o /out/buildctl ./cmd/buildctl"))
@ -93,12 +93,12 @@ func buildkit(opt buildOpt) llb.State {
copyAll(runc(opt.runc), "/"),
)
if opt.target == "containerd" {
if opt.withContainerd {
return r.With(
copyAll(containerd(opt.containerd), "/"),
copyAll(builddContainerd, "/"))
copyAll(buildd, "/"))
}
return r.With(copyAll(builddStandalone, "/"))
return r.With(copyAll(builddOCIWorkerOnly, "/"))
}
func copyAll(src llb.State, destPath string) llb.StateOption {

2
examples/gobuild/main.go
View File

@ -35,7 +35,7 @@ func run() error {
Source: src,
MountPath: "/go/src/github.com/moby/buildkit",
Pkg: "github.com/moby/buildkit/cmd/buildd",
BuildTags: []string{"standalone"},
BuildTags: []string{"no_containerd_worker"},
})
if err != nil {
return err

2
frontend/dockerfile/dockerfile_test.go
View File

@ -609,7 +609,7 @@ RUN ["ls"]
cmd := sb.Cmd(args + " --exporter=image --exporter-opt=name=" + target)
require.NoError(t, cmd.Run())
// TODO: expose this test to standalone
// TODO: expose this test to OCI worker
var cdAddress string
if cd, ok := sb.(interface {

34
hack/dockerfiles/test.Dockerfile
View File

@ -1,6 +1,6 @@
ARG RUNC_VERSION=74a17296470088de3805e138d3d87c62e613dfc4
ARG CONTAINERD_VERSION=v1.0.0
# available targets: buildd (standalone+containerd), buildd-standalone, buildd-containerd
# available targets: buildd, buildd.oci_only, buildd.containerd_only
ARG BUILDKIT_TARGET=buildd
ARG REGISTRY_VERSION=2.6
@ -22,7 +22,7 @@ RUN go build -ldflags '-d' -o /usr/bin/buildctl ./cmd/buildctl
FROM buildkit-base AS buildd
ENV CGO_ENABLED=0
RUN go build -ldflags '-d' -o /usr/bin/buildd -tags "standalone containerd" ./cmd/buildd
RUN go build -ldflags '-d' -o /usr/bin/buildd ./cmd/buildd
# test dependencies begin here
FROM gobuild-base AS runc
@ -47,20 +47,19 @@ COPY --from=runc /usr/bin/runc /usr/bin/runc
COPY --from=containerd /go/src/github.com/containerd/containerd/bin/containerd* /usr/bin/
FROM buildkit-base AS buildd-standalone
FROM buildkit-base AS buildd.oci_only
ENV CGO_ENABLED=0
RUN go build -ldflags '-d' -o /usr/bin/buildd-standalone -tags standalone ./cmd/buildd
RUN go build -ldflags '-d' -o /usr/bin/buildd.oci_only -tags no_containerd_worker ./cmd/buildd
FROM buildkit-base AS buildd-containerd
FROM buildkit-base AS buildd.containerd_only
ENV CGO_ENABLED=0
RUN go build -ldflags '-d' -o /usr/bin/buildd-containerd -tags containerd ./cmd/buildd
RUN go build -ldflags '-d' -o /usr/bin/buildd.containerd_only -tags no_oci_worker ./cmd/buildd
FROM registry:$REGISTRY_VERSION AS registry
FROM unit-tests AS integration-tests
COPY --from=buildctl /usr/bin/buildctl /usr/bin/
COPY --from=buildd-containerd /usr/bin/buildd-containerd /usr/bin
COPY --from=buildd-standalone /usr/bin/buildd-standalone /usr/bin
COPY --from=buildd /usr/bin/buildd /usr/bin
COPY --from=registry /bin/registry /usr/bin
FROM gobuild-base AS cross-windows
@ -78,24 +77,25 @@ FROM alpine AS buildkit-export
RUN apk add --no-cache git
VOLUME /var/lib/buildkit
# Copy together all binaries for standalone+containerd mode
# Copy together all binaries for oci+containerd mode
FROM buildkit-export AS buildkit-buildd
COPY --from=runc /usr/bin/runc /usr/bin/
COPY --from=buildd /usr/bin/buildd /usr/bin/
COPY --from=buildctl /usr/bin/buildctl /usr/bin/
ENTRYPOINT ["buildd"]
# Copy together all binaries needed for standalone mode
FROM buildkit-export AS buildkit-buildd-standalone
COPY --from=buildd-standalone /usr/bin/buildd-standalone /usr/bin/
# Copy together all binaries needed for oci worker mode
FROM buildkit-export AS buildkit-buildd.oci_only
COPY --from=buildd.oci_only /usr/bin/buildd.oci_only /usr/bin/
COPY --from=buildctl /usr/bin/buildctl /usr/bin/
ENTRYPOINT ["buildd-standalone"]
ENTRYPOINT ["buildd.oci_only"]
# Copy together all binaries for containerd mode
FROM buildkit-export AS buildkit-buildd-containerd
# Copy together all binaries for containerd worker mode
FROM buildkit-export AS buildkit-buildd.containerd_only
COPY --from=runc /usr/bin/runc /usr/bin/
COPY --from=buildd-containerd /usr/bin/buildd-containerd /usr/bin/
COPY --from=buildd.containerd_only /usr/bin/buildd.containerd_only /usr/bin/
COPY --from=buildctl /usr/bin/buildctl /usr/bin/
ENTRYPOINT ["buildd-containerd"]
ENTRYPOINT ["buildd.containerd_only"]
FROM alpine AS containerd-runtime
COPY --from=runc /usr/bin/runc /usr/bin/

2
hack/test
View File

@ -5,7 +5,7 @@ set -eu -o pipefail -x
# update this to iidfile after 17.06
docker build -t buildkit:test --target integration-tests -f ./hack/dockerfiles/test.Dockerfile --force-rm .
docker run --rm -v /tmp --privileged buildkit:test go test -tags standalone ${TESTFLAGS:--v} ${TESTPKGS:-./...}
docker run --rm -v /tmp --privileged buildkit:test go test -tags no_containerd_worker ${TESTFLAGS:--v} ${TESTPKGS:-./...}
docker run --rm buildkit:test go build ./frontend/gateway/client
docker run --rm buildkit:test go build ./frontend/dockerfile/cmd/dockerfile-frontend

7
util/testutil/integration/containerd.go
View File

@ -24,7 +24,7 @@ func (c *containerd) New() (sb Sandbox, cl func() error, err error) {
if err := lookupBinary("containerd"); err != nil {
return nil, nil, err
}
if err := lookupBinary("buildd-containerd"); err != nil {
if err := lookupBinary("buildd"); err != nil {
return nil, nil, err
}
if err := requireRoot(); err != nil {
@ -64,7 +64,10 @@ func (c *containerd) New() (sb Sandbox, cl func() error, err error) {
return nil, nil, err
}
builddSock, stop, err := runBuildd([]string{"buildd-containerd", "--containerd-worker-addr", address}, logs)
builddSock, stop, err := runBuildd([]string{"buildd",
"--oci-worker=false",
"--containerd-worker=true",
"--containerd-worker-addr", address}, logs)
if err != nil {
return nil, nil, err
}

14
util/testutil/integration/standalone.go → util/testutil/integration/oci.go
View File

@ -15,25 +15,25 @@ import (
)
func init() {
register(&standalone{})
register(&oci{})
}
type standalone struct {
type oci struct {
}
func (s *standalone) Name() string {
return "standalone"
func (s *oci) Name() string {
return "oci"
}
func (s *standalone) New() (Sandbox, func() error, error) {
if err := lookupBinary("buildd-standalone"); err != nil {
func (s *oci) New() (Sandbox, func() error, error) {
if err := lookupBinary("buildd"); err != nil {
return nil, nil, err
}
if err := requireRoot(); err != nil {
return nil, nil, err
}
logs := map[string]*bytes.Buffer{}
builddSock, stop, err := runBuildd([]string{"buildd-standalone"}, logs)
builddSock, stop, err := runBuildd([]string{"buildd", "--oci-worker=true", "--containerd-worker=false"}, logs)
if err != nil {
return nil, nil, err
}

2
worker/runc/runc_test.go
View File

@ -1,4 +1,4 @@
// +build linux,standalone
// +build linux,no_containerd_worker
package runc