Merge pull request #127 from tonistiigi/dockerfile-update

Dockerfile frontend update
docker-18.09
Tõnis Tiigi 2017-09-25 09:52:52 -07:00 committed by GitHub
commit 9527818ec2
10 changed files with 245 additions and 167 deletions

6
cache/manager.go vendored
View File

@ -162,7 +162,7 @@ func (cm *cacheManager) load(ctx context.Context, id string, opts ...RefOption)
if err := initializeMetadata(rec, opts...); err != nil {
if parent != nil {
parent.Release(ctx)
parent.Release(context.TODO())
}
return nil, err
}
@ -190,7 +190,7 @@ func (cm *cacheManager) New(ctx context.Context, s ImmutableRef, opts ...RefOpti
if _, err := cm.Snapshotter.Prepare(ctx, id, parentID); err != nil {
if parent != nil {
parent.Release(ctx)
parent.Release(context.TODO())
}
return nil, errors.Wrapf(err, "failed to prepare %s", id)
}
@ -207,7 +207,7 @@ func (cm *cacheManager) New(ctx context.Context, s ImmutableRef, opts ...RefOpti
if err := initializeMetadata(rec, opts...); err != nil {
if parent != nil {
parent.Release(ctx)
parent.Release(context.TODO())
}
return nil, err
}

View File

@ -32,7 +32,7 @@ func WithDefaultMetaResolver(ii *ImageInfo) {
}
type ImageMetaResolver interface {
ResolveImageConfig(ctx netcontext.Context, ref string) ([]byte, error)
ResolveImageConfig(ctx netcontext.Context, ref string) (digest.Digest, []byte, error)
}
func NewImageMetaResolver() ImageMetaResolver {
@ -41,7 +41,7 @@ func NewImageMetaResolver() ImageMetaResolver {
Client: http.DefaultClient,
}),
ingester: newInMemoryIngester(),
cache: map[string][]byte{},
cache: map[string]resolveResult{},
locker: locker.NewLocker(),
}
}
@ -57,24 +57,29 @@ type imageMetaResolver struct {
resolver remotes.Resolver
ingester *inMemoryIngester
locker *locker.Locker
cache map[string][]byte
cache map[string]resolveResult
}
func (imr *imageMetaResolver) ResolveImageConfig(ctx netcontext.Context, ref string) ([]byte, error) {
type resolveResult struct {
config []byte
dgst digest.Digest
}
func (imr *imageMetaResolver) ResolveImageConfig(ctx netcontext.Context, ref string) (digest.Digest, []byte, error) {
imr.locker.Lock(ref)
defer imr.locker.Unlock(ref)
if img, ok := imr.cache[ref]; ok {
return img, nil
if res, ok := imr.cache[ref]; ok {
return res.dgst, res.config, nil
}
img, err := imageutil.Config(ctx, ref, imr.resolver, imr.ingester)
dgst, config, err := imageutil.Config(ctx, ref, imr.resolver, imr.ingester)
if err != nil {
return nil, err
return "", nil, err
}
imr.cache[ref] = img
return img, nil
imr.cache[ref] = resolveResult{dgst: dgst, config: config}
return dgst, config, nil
}
type inMemoryIngester struct {

View File

@ -86,7 +86,7 @@ func Image(ref string, opts ...ImageOption) State {
opt(&info)
}
if info.metaResolver != nil {
dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref)
_, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref)
if err != nil {
src.err = err
} else {

View File

@ -16,28 +16,31 @@ import (
)
const (
keyTarget = "target"
keyFilename = "filename"
exporterImageConfig = "containerimage.config"
keyTarget = "target"
keyFilename = "filename"
exporterImageConfig = "containerimage.config"
defaultDockerfileName = "Dockerfile"
localNameDockerfile = "dockerfile"
buildArgPrefix = "build-arg:"
)
type dfFrontend struct{}
func NewDockerfileFrontend() frontend.Frontend {
return &dfFrontend{}
}
type dfFrontend struct{}
func (f *dfFrontend) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBridge, opts map[string]string) (retRef cache.ImmutableRef, exporterAttr map[string]interface{}, retErr error) {
filename := opts[keyFilename]
if filename == "" {
filename = "Dockerfile"
filename = defaultDockerfileName
}
if path.Base(filename) != filename {
return nil, nil, errors.Errorf("invalid filename %s", filename)
}
src := llb.Local("dockerfile", llb.IncludePatterns([]string{filename}))
src := llb.Local(localNameDockerfile, llb.IncludePatterns([]string{filename}))
dt, err := src.Marshal()
if err != nil {
return nil, nil, err
@ -82,7 +85,7 @@ func (f *dfFrontend) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBr
}
lm = nil
if err := ref.Release(ctx); err != nil {
if err := ref.Release(context.TODO()); err != nil {
return nil, nil, err
}
ref = nil
@ -115,8 +118,8 @@ func (f *dfFrontend) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBr
func filterBuildArgs(opt map[string]string) map[string]string {
m := map[string]string{}
for k, v := range opt {
if strings.HasPrefix(k, "build-arg:") {
m[strings.TrimPrefix(k, "build-arg:")] = v
if strings.HasPrefix(k, buildArgPrefix) {
m[strings.TrimPrefix(k, buildArgPrefix)] = v
}
}
return m

View File

@ -20,7 +20,8 @@ import (
)
const (
emptyImageName = "scratch"
emptyImageName = "scratch"
localNameContext = "context"
)
type ConvertOpt struct {
@ -45,47 +46,47 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
}
for i := range metaArgs {
metaArgs[i] = setArgValue(metaArgs[i], opt.BuildArgs)
metaArgs[i] = setBuildArgValue(metaArgs[i], opt.BuildArgs)
}
shlex := NewShellLex(dockerfile.EscapeToken)
var allStages []*dispatchState
stagesByName := map[string]*dispatchState{}
for _, st := range stages {
name, err := shlex.ProcessWord(st.BaseName, combineArgs([]string{}, metaArgs))
if err != nil {
return nil, nil, err
}
st.BaseName = name
state := llb.Scratch()
if st.BaseName != emptyImageName {
state = llb.Image(st.BaseName)
}
ds := &dispatchState{
state: state,
stage: st,
}
if d, ok := stagesByName[st.BaseName]; ok {
ds.base = d
}
allStages = append(allStages, ds)
if st.Name != "" {
stagesByName[strings.ToLower(st.Name)] = ds
}
}
metaResolver := opt.MetaResolver
if metaResolver == nil {
metaResolver = llb.DefaultImageMetaResolver()
}
var allDispatchStates []*dispatchState
dispatchStatesByName := map[string]*dispatchState{}
// set base state for every image
for _, st := range stages {
name, err := shlex.ProcessWord(st.BaseName, toEnvList(metaArgs, nil))
if err != nil {
return nil, nil, err
}
st.BaseName = name
ds := &dispatchState{
stage: st,
}
if d, ok := dispatchStatesByName[st.BaseName]; ok {
ds.base = d
}
allDispatchStates = append(allDispatchStates, ds)
if st.Name != "" {
dispatchStatesByName[strings.ToLower(st.Name)] = ds
}
}
eg, ctx := errgroup.WithContext(ctx)
for i, d := range allStages {
if d.base == nil && d.stage.BaseName != emptyImageName {
for i, d := range allDispatchStates {
// resolve image config for every stage
if d.base == nil {
if d.stage.BaseName == emptyImageName {
d.state = llb.Scratch()
continue
}
func(i int, d *dispatchState) {
eg.Go(func() error {
ref, err := reference.ParseNormalizedNamed(d.stage.BaseName)
@ -94,35 +95,41 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
}
d.stage.BaseName = reference.TagNameOnly(ref).String()
if metaResolver != nil {
dt, err := metaResolver.ResolveImageConfig(ctx, d.stage.BaseName)
if err != nil {
return nil // handle the error while builder is actually running
dgst, dt, err := metaResolver.ResolveImageConfig(ctx, d.stage.BaseName)
if err == nil { // handle the error while builder is actually running
// TODO: detect unreachable stages so config is not pulled for them
var img Image
if err := json.Unmarshal(dt, &img); err != nil {
return err
}
d.image = img
ref, err := reference.WithDigest(ref, dgst)
if err != nil {
return err
}
d.stage.BaseName = ref.String()
_ = ref
}
var img Image
if err := json.Unmarshal(dt, &img); err != nil {
return err
}
allStages[i].image = img
}
d.state = llb.Image(d.stage.BaseName)
return nil
})
}(i, d)
}
}
if err := eg.Wait(); err != nil {
return nil, nil, err
}
for _, d := range allStages {
for _, d := range allDispatchStates {
if d.base != nil {
d.state = d.base.state
d.image = clone(d.base.image)
}
var args []instructions.ArgCommand
// initialize base metadata from image conf
for _, env := range d.image.Config.Env {
parts := strings.SplitN(env, "=", 2)
v := ""
@ -138,92 +145,142 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
return nil, nil, err
}
}
for _, cmd := range d.stage.Commands {
if ex, ok := cmd.(instructions.SupportsSingleWordExpansion); ok {
err := ex.Expand(func(word string) (string, error) {
return shlex.ProcessWord(word, combineArgs(d.image.Config.Env, args))
})
if err != nil {
return nil, nil, err
}
}
switch c := cmd.(type) {
case *instructions.EnvCommand:
err = dispatchEnv(d, c)
case *instructions.RunCommand:
err = dispatchRun(d, c, args)
case *instructions.WorkdirCommand:
err = dispatchWorkdir(d, c)
case *instructions.AddCommand:
err = dispatchCopy(d, c.SourcesAndDest, llb.Local("context"))
case *instructions.LabelCommand:
err = dispatchLabel(d, c)
case *instructions.OnbuildCommand:
err = dispatchOnbuild(d, c)
case *instructions.CmdCommand:
err = dispatchCmd(d, c)
case *instructions.EntrypointCommand:
err = dispatchEntrypoint(d, c)
case *instructions.HealthCheckCommand:
err = dispatchHealthcheck(d, c)
case *instructions.ExposeCommand:
err = dispatchExpose(d, c)
case *instructions.UserCommand:
err = dispatchUser(d, c)
case *instructions.VolumeCommand:
err = dispatchVolume(d, c)
case *instructions.StopSignalCommand:
err = dispatchStopSignal(d, c)
case *instructions.ShellCommand:
err = dispatchShell(d, c)
case *instructions.ArgCommand:
args, err = dispatchArg(d, c, args, metaArgs, opt.BuildArgs)
case *instructions.CopyCommand:
l := llb.Local("context")
if c.From != "" {
index, err := strconv.Atoi(c.From)
if err != nil {
stn, ok := stagesByName[strings.ToLower(c.From)]
if !ok {
return nil, nil, errors.Errorf("stage %s not found", c.From)
}
l = stn.state
} else {
if index >= len(allStages) {
return nil, nil, errors.Errorf("invalid stage index %d", index)
}
l = allStages[index].state
}
}
err = dispatchCopy(d, c.SourcesAndDest, l)
default:
continue
}
if err != nil {
if d.image.Config.User != "" {
if err = dispatchUser(d, &instructions.UserCommand{User: d.image.Config.User}); err != nil {
return nil, nil, err
}
}
opt := dispatchOpt{
allDispatchStates: allDispatchStates,
dispatchStatesByName: dispatchStatesByName,
metaArgs: metaArgs,
buildArgValues: opt.BuildArgs,
shlex: shlex,
}
if err = dispatchOnBuild(d, d.image.Config.OnBuild, opt); err != nil {
return nil, nil, err
}
for _, cmd := range d.stage.Commands {
if err := dispatch(d, cmd, opt); err != nil {
return nil, nil, err
}
}
}
if opt.Target == "" {
return &allStages[len(allStages)-1].state, &allStages[len(allStages)-1].image, nil
return &allDispatchStates[len(allDispatchStates)-1].state, &allDispatchStates[len(allDispatchStates)-1].image, nil
}
state, ok := stagesByName[strings.ToLower(opt.Target)]
state, ok := dispatchStatesByName[strings.ToLower(opt.Target)]
if !ok {
return nil, nil, errors.Errorf("target stage %s could not be found", opt.Target)
}
return &state.state, &state.image, nil
}
type dispatchOpt struct {
allDispatchStates []*dispatchState
dispatchStatesByName map[string]*dispatchState
metaArgs []instructions.ArgCommand
buildArgValues map[string]string
shlex *ShellLex
}
func dispatch(d *dispatchState, cmd instructions.Command, opt dispatchOpt) error {
if ex, ok := cmd.(instructions.SupportsSingleWordExpansion); ok {
err := ex.Expand(func(word string) (string, error) {
return opt.shlex.ProcessWord(word, toEnvList(d.buildArgs, d.image.Config.Env))
})
if err != nil {
return err
}
}
var err error
switch c := cmd.(type) {
case *instructions.EnvCommand:
err = dispatchEnv(d, c)
case *instructions.RunCommand:
err = dispatchRun(d, c)
case *instructions.WorkdirCommand:
err = dispatchWorkdir(d, c)
case *instructions.AddCommand:
err = dispatchCopy(d, c.SourcesAndDest, llb.Local(localNameContext))
case *instructions.LabelCommand:
err = dispatchLabel(d, c)
case *instructions.OnbuildCommand:
err = dispatchOnbuild(d, c)
case *instructions.CmdCommand:
err = dispatchCmd(d, c)
case *instructions.EntrypointCommand:
err = dispatchEntrypoint(d, c)
case *instructions.HealthCheckCommand:
err = dispatchHealthcheck(d, c)
case *instructions.ExposeCommand:
err = dispatchExpose(d, c)
case *instructions.UserCommand:
err = dispatchUser(d, c)
case *instructions.VolumeCommand:
err = dispatchVolume(d, c)
case *instructions.StopSignalCommand:
err = dispatchStopSignal(d, c)
case *instructions.ShellCommand:
err = dispatchShell(d, c)
case *instructions.ArgCommand:
err = dispatchArg(d, c, opt.metaArgs, opt.buildArgValues)
case *instructions.CopyCommand:
l := llb.Local(localNameContext)
if c.From != "" {
index, err := strconv.Atoi(c.From)
if err != nil {
stn, ok := opt.dispatchStatesByName[strings.ToLower(c.From)]
if !ok {
return errors.Errorf("stage %s not found", c.From)
}
l = stn.state
} else {
if index >= len(opt.allDispatchStates) {
return errors.Errorf("invalid stage index %d", index)
}
l = opt.allDispatchStates[index].state
}
}
err = dispatchCopy(d, c.SourcesAndDest, l)
default:
}
return err
}
type dispatchState struct {
state llb.State
image Image
stage instructions.Stage
base *dispatchState
buildArgs []instructions.ArgCommand
}
func dispatchOnBuild(d *dispatchState, triggers []string, opt dispatchOpt) error {
for _, trigger := range triggers {
ast, err := parser.Parse(strings.NewReader(trigger))
if err != nil {
return err
}
if len(ast.AST.Children) != 1 {
return errors.New("onbuild trigger should be a single expression")
}
cmd, err := instructions.ParseCommand(ast.AST.Children[0])
if err != nil {
return err
}
if err := dispatch(d, cmd, opt); err != nil {
return err
}
}
return nil
}
func dispatchEnv(d *dispatchState, c *instructions.EnvCommand) error {
@ -234,7 +291,7 @@ func dispatchEnv(d *dispatchState, c *instructions.EnvCommand) error {
return nil
}
func dispatchRun(d *dispatchState, c *instructions.RunCommand, buildArgs []instructions.ArgCommand) error {
func dispatchRun(d *dispatchState, c *instructions.RunCommand) error {
var args []string = c.CmdLine
if c.PrependShell {
args = append(defaultShell(), strings.Join(args, " "))
@ -242,7 +299,7 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, buildArgs []instr
args = append(d.image.Config.Entrypoint, args...)
}
opt := []llb.RunOption{llb.Args(args)}
for _, arg := range buildArgs {
for _, arg := range d.buildArgs {
opt = append(opt, llb.AddEnv(arg.Key, getArgValue(arg)))
}
d.state = d.state.Run(opt...).Root()
@ -263,7 +320,7 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
// TODO: this should use CopyOp instead. Current implementation is inefficient and doesn't match Dockerfile path suffixes rules
img := llb.Image("tonistiigi/copy@sha256:260a4355be76e0609518ebd7c0e026831c80b8908d4afd3f8e8c942645b1e5cf")
dest := path.Join("/dest", toWorkingDir(d.state, c.Dest()))
dest := path.Join("/dest", pathRelativeToWorkingDir(d.state, c.Dest()))
args := []string{"copy"}
mounts := make([]llb.RunOption, 0, len(c.Sources()))
for i, src := range c.Sources() {
@ -282,13 +339,6 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
return nil
}
func toWorkingDir(s llb.State, p string) string {
if path.IsAbs(p) {
return p
}
return path.Join(s.GetDir(), p)
}
func dispatchMaintainer(d *dispatchState, c instructions.MaintainerCommand) error {
d.image.Author = c.Maintainer
return nil
@ -386,7 +436,7 @@ func dispatchShell(d *dispatchState, c *instructions.ShellCommand) error {
return nil
}
func dispatchArg(d *dispatchState, c *instructions.ArgCommand, args []instructions.ArgCommand, metaArgs []instructions.ArgCommand, buildArgs map[string]string) ([]instructions.ArgCommand, error) {
func dispatchArg(d *dispatchState, c *instructions.ArgCommand, metaArgs []instructions.ArgCommand, buildArgValues map[string]string) error {
if c.Value == nil {
for _, ma := range metaArgs {
if ma.Key == c.Key {
@ -395,8 +445,15 @@ func dispatchArg(d *dispatchState, c *instructions.ArgCommand, args []instructio
}
}
args = append(args, setArgValue(*c, buildArgs))
return args, nil
d.buildArgs = append(d.buildArgs, setBuildArgValue(*c, buildArgValues))
return nil
}
func pathRelativeToWorkingDir(s llb.State, p string) string {
if path.IsAbs(p) {
return p
}
return path.Join(s.GetDir(), p)
}
func splitWildcards(name string) (string, string) {
@ -438,14 +495,14 @@ func equalEnvKeys(from, to string) bool {
return from == to
}
func setArgValue(c instructions.ArgCommand, values map[string]string) instructions.ArgCommand {
func setBuildArgValue(c instructions.ArgCommand, values map[string]string) instructions.ArgCommand {
if v, ok := values[c.Key]; ok {
c.Value = &v
}
return c
}
func combineArgs(env []string, args []instructions.ArgCommand) []string {
func toEnvList(args []instructions.ArgCommand, env []string) []string {
for _, arg := range args {
env = addEnv(env, arg.Key, getArgValue(arg), false)
}

View File

@ -2,6 +2,7 @@ package frontend
import (
"github.com/moby/buildkit/cache"
digest "github.com/opencontainers/go-digest"
"golang.org/x/net/context"
)
@ -11,5 +12,5 @@ type Frontend interface {
type FrontendLLBBridge interface {
Solve(ctx context.Context, vtx [][]byte) (cache.ImmutableRef, error)
ResolveImageConfig(ctx context.Context, ref string) ([]byte, error)
ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error)
}

View File

@ -57,7 +57,7 @@ func (e *execOp) Run(ctx context.Context, inputs []Reference) ([]Reference, erro
defer func() {
for _, o := range outputs {
if o != nil {
go o.Release(ctx)
go o.Release(context.TODO())
}
}
}()

View File

@ -552,7 +552,7 @@ type llbBridge struct {
}
type resolveImageConfig interface {
ResolveImageConfig(ctx context.Context, ref string) ([]byte, error)
ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error)
}
func (s *llbBridge) Solve(ctx context.Context, dt [][]byte) (cache.ImmutableRef, error) {

View File

@ -73,7 +73,7 @@ func (is *imageSource) ID() string {
return source.DockerImageScheme
}
func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string) ([]byte, error) {
func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error) {
return imageutil.Config(ctx, ref, is.resolver, is.ContentStore)
}
@ -108,13 +108,19 @@ func (p *puller) resolve(ctx context.Context) error {
info, err := p.is.ContentStore.Info(ctx, dgst)
if err == nil {
p.ref = p.src.Reference.String()
p.desc = ocispec.Descriptor{
Size: info.Size,
Digest: dgst,
MediaType: ocispec.MediaTypeImageManifest, // TODO: detect schema1/manifest-list
ra, err := p.is.ContentStore.ReaderAt(ctx, dgst)
if err == nil {
mt, err := imageutil.DetectManifestMediaType(ra)
if err == nil {
p.desc = ocispec.Descriptor{
Size: info.Size,
Digest: dgst,
MediaType: mt,
}
resolveProgressDone(nil)
return
}
}
resolveProgressDone(nil)
return
}
}

View File

@ -9,6 +9,7 @@ import (
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -18,10 +19,10 @@ type IngesterProvider interface {
content.Provider
}
func Config(ctx context.Context, str string, resolver remotes.Resolver, ingester IngesterProvider) ([]byte, error) {
func Config(ctx context.Context, str string, resolver remotes.Resolver, ingester IngesterProvider) (digest.Digest, []byte, error) {
ref, err := reference.Parse(str)
if err != nil {
return nil, errors.WithStack(err)
return "", nil, errors.WithStack(err)
}
dgst := ref.Digest()
@ -29,7 +30,7 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, ingester
if dgst != "" {
ra, err := ingester.ReaderAt(ctx, dgst)
if err == nil {
mt, err := detectManifestMediaType(ra)
mt, err := DetectManifestMediaType(ra)
if err == nil {
desc = &ocispec.Descriptor{
Size: ra.Size(),
@ -43,14 +44,14 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, ingester
if desc == nil {
_, desc2, err := resolver.Resolve(ctx, ref.String())
if err != nil {
return nil, err
return "", nil, err
}
desc = &desc2
}
fetcher, err := resolver.Fetcher(ctx, ref.String())
if err != nil {
return nil, err
return "", nil, err
}
handlers := []images.Handler{
@ -58,14 +59,19 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, ingester
childrenConfigHandler(ingester),
}
if err := images.Dispatch(ctx, images.Handlers(handlers...), *desc); err != nil {
return nil, err
return "", nil, err
}
config, err := images.Config(ctx, ingester, *desc, platforms.Format(platforms.Default()))
if err != nil {
return nil, err
return "", nil, err
}
return content.ReadBlob(ctx, ingester, config.Digest)
dt, err := content.ReadBlob(ctx, ingester, config.Digest)
if err != nil {
return "", nil, err
}
return desc.Digest, dt, nil
}
func childrenConfigHandler(provider content.Provider) images.HandlerFunc {
@ -110,7 +116,7 @@ func childrenConfigHandler(provider content.Provider) images.HandlerFunc {
}
// ocispec.MediaTypeImageManifest, // TODO: detect schema1/manifest-list
func detectManifestMediaType(ra content.ReaderAt) (string, error) {
func DetectManifestMediaType(ra content.ReaderAt) (string, error) {
// TODO: schema1
p := make([]byte, ra.Size())