From 49de6752349023ba1c4dc9218513271198eb77fd Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 16 Jun 2020 16:45:17 -0700 Subject: [PATCH] git: support for token authentication Signed-off-by: Tonis Tiigi --- client/llb/source.go | 29 +++++++++- solver/pb/attr.go | 2 + solver/pb/caps.go | 13 ++++- source/git/gitsource.go | 119 +++++++++++++++++++++++++++++++++------- source/gitidentifier.go | 10 ++-- source/identifier.go | 4 ++ 6 files changed, 149 insertions(+), 28 deletions(-) diff --git a/client/llb/source.go b/client/llb/source.go index fd2f44f2..51fdcf92 100644 --- a/client/llb/source.go +++ b/client/llb/source.go @@ -215,7 +215,10 @@ func Git(remote, ref string, opts ...GitOption) State { id += "#" + ref } - gi := &GitInfo{} + gi := &GitInfo{ + AuthHeaderSecret: "GIT_AUTH_HEADER", + AuthTokenSecret: "GIT_AUTH_TOKEN", + } for _, o := range opts { o.SetGitOption(gi) } @@ -228,6 +231,14 @@ func Git(remote, ref string, opts ...GitOption) State { attrs[pb.AttrFullRemoteURL] = url addCap(&gi.Constraints, pb.CapSourceGitFullURL) } + if gi.AuthTokenSecret != "" { + attrs[pb.AttrAuthTokenSecret] = gi.AuthTokenSecret + addCap(&gi.Constraints, pb.CapSourceGitHttpAuth) + } + if gi.AuthHeaderSecret != "" { + attrs[pb.AttrAuthHeaderSecret] = gi.AuthHeaderSecret + addCap(&gi.Constraints, pb.CapSourceGitHttpAuth) + } addCap(&gi.Constraints, pb.CapSourceGit) @@ -246,7 +257,9 @@ func (fn gitOptionFunc) SetGitOption(gi *GitInfo) { type GitInfo struct { constraintsWrapper - KeepGitDir bool + KeepGitDir bool + AuthTokenSecret string + AuthHeaderSecret string } func KeepGitDir() GitOption { @@ -255,6 +268,18 @@ func KeepGitDir() GitOption { }) } +func AuthTokenSecret(v string) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.AuthTokenSecret = v + }) +} + +func AuthHeaderSecret(v string) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.AuthHeaderSecret = v + }) +} + func Scratch() State { return NewState(nil) } diff --git a/solver/pb/attr.go b/solver/pb/attr.go index 97d2971c..f22d5d77 100644 --- a/solver/pb/attr.go +++ b/solver/pb/attr.go @@ -2,6 +2,8 @@ package pb const AttrKeepGitDir = "git.keepgitdir" const AttrFullRemoteURL = "git.fullurl" +const AttrAuthHeaderSecret = "git.authheadersecret" +const AttrAuthTokenSecret = "git.authtokensecret" const AttrLocalSessionID = "local.session" const AttrLocalUniqueID = "local.unique" const AttrIncludePatterns = "local.includepattern" diff --git a/solver/pb/caps.go b/solver/pb/caps.go index 93c77b3e..43a18176 100644 --- a/solver/pb/caps.go +++ b/solver/pb/caps.go @@ -19,9 +19,10 @@ const ( CapSourceLocalExcludePatterns apicaps.CapID = "source.local.excludepatterns" CapSourceLocalSharedKeyHint apicaps.CapID = "source.local.sharedkeyhint" - CapSourceGit apicaps.CapID = "source.git" - CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir" - CapSourceGitFullURL apicaps.CapID = "source.git.fullurl" + CapSourceGit apicaps.CapID = "source.git" + CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir" + CapSourceGitFullURL apicaps.CapID = "source.git.fullurl" + CapSourceGitHttpAuth apicaps.CapID = "source.git.httpauth" CapSourceHTTP apicaps.CapID = "source.http" CapSourceHTTPChecksum apicaps.CapID = "source.http.checksum" @@ -131,6 +132,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapSourceGitHttpAuth, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapSourceHTTP, Enabled: true, diff --git a/source/git/gitsource.go b/source/git/gitsource.go index 5c36a47d..1e005ec5 100644 --- a/source/git/gitsource.go +++ b/source/git/gitsource.go @@ -3,13 +3,16 @@ package git import ( "bytes" "context" + "encoding/base64" "fmt" "io" + "net/url" "os" "os/exec" "path/filepath" "regexp" "strings" + "time" "github.com/docker/docker/pkg/locker" "github.com/moby/buildkit/cache" @@ -17,6 +20,7 @@ import ( "github.com/moby/buildkit/client" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/secrets" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/source" "github.com/moby/buildkit/util/progress/logs" @@ -60,7 +64,7 @@ func (gs *gitSource) ID() string { } // needs to be called with repo lock -func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target string, release func(), retErr error) { +func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []string) (target string, release func(), retErr error) { remoteKey := "git-remote::" + remote sis, err := gs.md.Search(remoteKey) @@ -119,11 +123,11 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target str }() if initializeRepo { - if _, err := gitWithinDir(ctx, dir, "", "init", "--bare"); err != nil { + if _, err := gitWithinDir(ctx, dir, "", auth, "init", "--bare"); err != nil { return "", nil, errors.Wrapf(err, "failed to init repo at %s", dir) } - if _, err := gitWithinDir(ctx, dir, "", "remote", "add", "origin", remote); err != nil { + if _, err := gitWithinDir(ctx, dir, "", auth, "remote", "add", "origin", remote); err != nil { return "", nil, errors.Wrapf(err, "failed add origin repo at %s", dir) } @@ -151,6 +155,8 @@ type gitSourceHandler struct { *gitSource src source.GitIdentifier cacheKey string + sm *session.Manager + auth []string } func (gs *gitSourceHandler) shaToCacheKey(sha string) string { @@ -161,7 +167,7 @@ func (gs *gitSourceHandler) shaToCacheKey(sha string) string { return key } -func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *session.Manager) (source.SourceInstance, error) { +func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager) (source.SourceInstance, error) { gitIdentifier, ok := id.(*source.GitIdentifier) if !ok { return nil, errors.Errorf("invalid git identifier %v", id) @@ -170,9 +176,74 @@ func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *sessi return &gitSourceHandler{ src: *gitIdentifier, gitSource: gs, + sm: sm, }, nil } +type authSecret struct { + token bool + name string +} + +func (gs *gitSourceHandler) authSecretNames() (sec []authSecret, _ error) { + u, err := url.Parse(gs.src.Remote) + if err != nil { + return nil, err + } + if gs.src.AuthHeaderSecret != "" { + sec = append(sec, authSecret{name: gs.src.AuthHeaderSecret + "." + u.Host}) + } + if gs.src.AuthTokenSecret != "" { + sec = append(sec, authSecret{name: gs.src.AuthTokenSecret + "." + u.Host, token: true}) + } + if gs.src.AuthHeaderSecret != "" { + sec = append(sec, authSecret{name: gs.src.AuthHeaderSecret}) + } + if gs.src.AuthTokenSecret != "" { + sec = append(sec, authSecret{name: gs.src.AuthTokenSecret, token: true}) + } + return sec, nil +} + +func (gs *gitSourceHandler) getAuthToken(ctx context.Context) error { + if gs.auth != nil { + return nil + } + sec, err := gs.authSecretNames() + if err != nil { + return err + } + id := session.FromContext(ctx) + if id == "" { + return errors.New("could not access auth tokens without session") + } + + timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + caller, err := gs.sm.Get(timeoutCtx, id) + if err != nil { + return err + } + + for _, s := range sec { + dt, err := secrets.GetSecret(ctx, caller, s.name) + if err != nil { + if errors.Is(err, secrets.ErrNotFound) { + continue + } + return err + } + if s.token { + dt = []byte("basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("x-access-token:%s", dt)))) + } + gs.auth = []string{"-c", "http.extraheader=Authorization: " + string(dt)} + break + } + + return nil +} + func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bool, error) { remote := gs.src.Remote ref := gs.src.Ref @@ -188,7 +259,9 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo return ref, true, nil } - gitDir, unmountGitDir, err := gs.mountRemote(ctx, remote) + gs.getAuthToken(ctx) + + gitDir, unmountGitDir, err := gs.mountRemote(ctx, remote, gs.auth) if err != nil { return "", false, err } @@ -196,14 +269,14 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo // TODO: should we assume that remote tag is immutable? add a timer? - buf, err := gitWithinDir(ctx, gitDir, "", "ls-remote", "origin", ref) + buf, err := gitWithinDir(ctx, gitDir, "", gs.auth, "ls-remote", "origin", ref) if err != nil { return "", false, errors.Wrapf(err, "failed to fetch remote %s", remote) } out := buf.String() idx := strings.Index(out, "\t") if idx == -1 { - return "", false, errors.Errorf("failed to find commit SHA from output: %s", string(out)) + return "", false, errors.Errorf("repository does not contain ref %s, output: %q", ref, string(out)) } sha := string(out[:idx]) @@ -230,6 +303,8 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe } } + gs.getAuthToken(ctx) + snapshotKey := "git-snapshot::" + cacheKey + ":" + gs.src.Subdir gs.locker.Lock(snapshotKey) defer gs.locker.Unlock(snapshotKey) @@ -244,7 +319,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe gs.locker.Lock(gs.src.Remote) defer gs.locker.Unlock(gs.src.Remote) - gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote) + gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, gs.auth) if err != nil { return nil, err } @@ -253,7 +328,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe doFetch := true if isCommitSHA(ref) { // skip fetch if commit already exists - if _, err := gitWithinDir(ctx, gitDir, "", "cat-file", "-e", ref+"^{commit}"); err == nil { + if _, err := gitWithinDir(ctx, gitDir, "", nil, "cat-file", "-e", ref+"^{commit}"); err == nil { doFetch = false } } @@ -277,7 +352,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe // in case the ref is a branch and it now points to a different commit sha // TODO: is there a better way to do this? } - if _, err := gitWithinDir(ctx, gitDir, "", args...); err != nil { + if _, err := gitWithinDir(ctx, gitDir, "", gs.auth, args...); err != nil { return nil, errors.Wrapf(err, "failed to fetch remote %s", gs.src.Remote) } } @@ -313,41 +388,41 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe if err := os.MkdirAll(checkoutDir, 0711); err != nil { return nil, err } - _, err = gitWithinDir(ctx, checkoutDirGit, "", "init") + _, err = gitWithinDir(ctx, checkoutDirGit, "", nil, "init") if err != nil { return nil, err } - _, err = gitWithinDir(ctx, checkoutDirGit, "", "remote", "add", "origin", gitDir) + _, err = gitWithinDir(ctx, checkoutDirGit, "", nil, "remote", "add", "origin", gitDir) if err != nil { return nil, err } pullref := ref if isCommitSHA(ref) { pullref = "refs/buildkit/" + identity.NewID() - _, err = gitWithinDir(ctx, gitDir, "", "update-ref", pullref, ref) + _, err = gitWithinDir(ctx, gitDir, "", gs.auth, "update-ref", pullref, ref) if err != nil { return nil, err } } else { pullref += ":" + pullref } - _, err = gitWithinDir(ctx, checkoutDirGit, "", "fetch", "-u", "--depth=1", "origin", pullref) + _, err = gitWithinDir(ctx, checkoutDirGit, "", gs.auth, "fetch", "-u", "--depth=1", "origin", pullref) if err != nil { return nil, err } - _, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, "checkout", "FETCH_HEAD") + _, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, nil, "checkout", "FETCH_HEAD") if err != nil { return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote) } gitDir = checkoutDirGit } else { - _, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".") + _, err = gitWithinDir(ctx, gitDir, checkoutDir, nil, "checkout", ref, "--", ".") if err != nil { return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote) } } - _, err = gitWithinDir(ctx, gitDir, checkoutDir, "submodule", "update", "--init", "--recursive", "--depth=1") + _, err = gitWithinDir(ctx, gitDir, checkoutDir, gs.auth, "submodule", "update", "--init", "--recursive", "--depth=1") if err != nil { return nil, errors.Wrapf(err, "failed to update submodules for %s", gs.src.Remote) } @@ -396,8 +471,8 @@ func isCommitSHA(str string) bool { return validHex.MatchString(str) } -func gitWithinDir(ctx context.Context, gitDir, workDir string, args ...string) (*bytes.Buffer, error) { - a := []string{"--git-dir", gitDir} +func gitWithinDir(ctx context.Context, gitDir, workDir string, auth []string, args ...string) (*bytes.Buffer, error) { + a := append([]string{"--git-dir", gitDir}, auth...) if workDir != "" { a = append(a, "--work-tree", workDir) } @@ -413,8 +488,14 @@ func git(ctx context.Context, dir string, args ...string) (*bytes.Buffer, error) cmd.Dir = dir // some commands like submodule require this buf := bytes.NewBuffer(nil) errbuf := bytes.NewBuffer(nil) + cmd.Stdin = nil cmd.Stdout = io.MultiWriter(stdout, buf) cmd.Stderr = io.MultiWriter(stderr, errbuf) + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), + "GIT_TERMINAL_PROMPT=0", + // "GIT_TRACE=1", + } // remote git commands spawn helper processes that inherit FDs and don't // handle parent death signal so exec.CommandContext can't be used err := runProcessGroup(ctx, cmd) diff --git a/source/gitidentifier.go b/source/gitidentifier.go index 9f338343..65f3f698 100644 --- a/source/gitidentifier.go +++ b/source/gitidentifier.go @@ -8,10 +8,12 @@ import ( ) type GitIdentifier struct { - Remote string - Ref string - Subdir string - KeepGitDir bool + Remote string + Ref string + Subdir string + KeepGitDir bool + AuthTokenSecret string + AuthHeaderSecret string } func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) { diff --git a/source/identifier.go b/source/identifier.go index 2a861109..fd500a15 100644 --- a/source/identifier.go +++ b/source/identifier.go @@ -103,6 +103,10 @@ func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) { } case pb.AttrFullRemoteURL: id.Remote = v + case pb.AttrAuthHeaderSecret: + id.AuthHeaderSecret = v + case pb.AttrAuthTokenSecret: + id.AuthTokenSecret = v } } }