JohnHammond
/
buildkit
mirror of https://github.com/JohnHammond/buildkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1533 from tonistiigi/git-token-auth 

git: support for token authentication
v0.8
Tõnis Tiigi 2020-06-19 13:24:30 -07:00 committed by GitHub
parent 7c9d706a2a 49de675234
commit b0c5648786
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 149 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
client/llb/source.go
View File

@ -215,7 +215,10 @@ func Git(remote, ref string, opts ...GitOption) State {
id += "#" + ref
}
gi := &GitInfo{}
gi := &GitInfo{
AuthHeaderSecret: "GIT_AUTH_HEADER",
AuthTokenSecret: "GIT_AUTH_TOKEN",
}
for _, o := range opts {
o.SetGitOption(gi)
}
@ -228,6 +231,14 @@ func Git(remote, ref string, opts ...GitOption) State {
attrs[pb.AttrFullRemoteURL] = url
addCap(&gi.Constraints, pb.CapSourceGitFullURL)
}
if gi.AuthTokenSecret != "" {
attrs[pb.AttrAuthTokenSecret] = gi.AuthTokenSecret
addCap(&gi.Constraints, pb.CapSourceGitHttpAuth)
}
if gi.AuthHeaderSecret != "" {
attrs[pb.AttrAuthHeaderSecret] = gi.AuthHeaderSecret
addCap(&gi.Constraints, pb.CapSourceGitHttpAuth)
}
addCap(&gi.Constraints, pb.CapSourceGit)
@ -246,7 +257,9 @@ func (fn gitOptionFunc) SetGitOption(gi *GitInfo) {
type GitInfo struct {
constraintsWrapper
KeepGitDir bool
KeepGitDir bool
AuthTokenSecret string
AuthHeaderSecret string
}
func KeepGitDir() GitOption {
@ -255,6 +268,18 @@ func KeepGitDir() GitOption {
})
}
func AuthTokenSecret(v string) GitOption {
return gitOptionFunc(func(gi *GitInfo) {
gi.AuthTokenSecret = v
})
}
func AuthHeaderSecret(v string) GitOption {
return gitOptionFunc(func(gi *GitInfo) {
gi.AuthHeaderSecret = v
})
}
func Scratch() State {
return NewState(nil)
}

2
solver/pb/attr.go
View File

@ -2,6 +2,8 @@ package pb
const AttrKeepGitDir = "git.keepgitdir"
const AttrFullRemoteURL = "git.fullurl"
const AttrAuthHeaderSecret = "git.authheadersecret"
const AttrAuthTokenSecret = "git.authtokensecret"
const AttrLocalSessionID = "local.session"
const AttrLocalUniqueID = "local.unique"
const AttrIncludePatterns = "local.includepattern"

13
solver/pb/caps.go
View File

@ -19,9 +19,10 @@ const (
CapSourceLocalExcludePatterns apicaps.CapID = "source.local.excludepatterns"
CapSourceLocalSharedKeyHint apicaps.CapID = "source.local.sharedkeyhint"
CapSourceGit apicaps.CapID = "source.git"
CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir"
CapSourceGitFullURL apicaps.CapID = "source.git.fullurl"
CapSourceGit apicaps.CapID = "source.git"
CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir"
CapSourceGitFullURL apicaps.CapID = "source.git.fullurl"
CapSourceGitHttpAuth apicaps.CapID = "source.git.httpauth"
CapSourceHTTP apicaps.CapID = "source.http"
CapSourceHTTPChecksum apicaps.CapID = "source.http.checksum"
@ -131,6 +132,12 @@ func init() {
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapSourceGitHttpAuth,
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapSourceHTTP,
Enabled: true,

119
source/git/gitsource.go
View File

@ -3,13 +3,16 @@ package git
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"io"
"net/url"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/docker/docker/pkg/locker"
"github.com/moby/buildkit/cache"
@ -17,6 +20,7 @@ import (
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/secrets"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/util/progress/logs"
@ -60,7 +64,7 @@ func (gs *gitSource) ID() string {
}
// needs to be called with repo lock
func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target string, release func(), retErr error) {
func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []string) (target string, release func(), retErr error) {
remoteKey := "git-remote::" + remote
sis, err := gs.md.Search(remoteKey)
@ -119,11 +123,11 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target str
}()
if initializeRepo {
if _, err := gitWithinDir(ctx, dir, "", "init", "--bare"); err != nil {
if _, err := gitWithinDir(ctx, dir, "", auth, "init", "--bare"); err != nil {
return "", nil, errors.Wrapf(err, "failed to init repo at %s", dir)
}
if _, err := gitWithinDir(ctx, dir, "", "remote", "add", "origin", remote); err != nil {
if _, err := gitWithinDir(ctx, dir, "", auth, "remote", "add", "origin", remote); err != nil {
return "", nil, errors.Wrapf(err, "failed add origin repo at %s", dir)
}
@ -151,6 +155,8 @@ type gitSourceHandler struct {
*gitSource
src source.GitIdentifier
cacheKey string
sm *session.Manager
auth []string
}
func (gs *gitSourceHandler) shaToCacheKey(sha string) string {
@ -161,7 +167,7 @@ func (gs *gitSourceHandler) shaToCacheKey(sha string) string {
return key
}
func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *session.Manager) (source.SourceInstance, error) {
func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager) (source.SourceInstance, error) {
gitIdentifier, ok := id.(*source.GitIdentifier)
if !ok {
return nil, errors.Errorf("invalid git identifier %v", id)
@ -170,9 +176,74 @@ func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *sessi
return &gitSourceHandler{
src: *gitIdentifier,
gitSource: gs,
sm: sm,
}, nil
}
type authSecret struct {
token bool
name string
}
func (gs *gitSourceHandler) authSecretNames() (sec []authSecret, _ error) {
u, err := url.Parse(gs.src.Remote)
if err != nil {
return nil, err
}
if gs.src.AuthHeaderSecret != "" {
sec = append(sec, authSecret{name: gs.src.AuthHeaderSecret + "." + u.Host})
}
if gs.src.AuthTokenSecret != "" {
sec = append(sec, authSecret{name: gs.src.AuthTokenSecret + "." + u.Host, token: true})
}
if gs.src.AuthHeaderSecret != "" {
sec = append(sec, authSecret{name: gs.src.AuthHeaderSecret})
}
if gs.src.AuthTokenSecret != "" {
sec = append(sec, authSecret{name: gs.src.AuthTokenSecret, token: true})
}
return sec, nil
}
func (gs *gitSourceHandler) getAuthToken(ctx context.Context) error {
if gs.auth != nil {
return nil
}
sec, err := gs.authSecretNames()
if err != nil {
return err
}
id := session.FromContext(ctx)
if id == "" {
return errors.New("could not access auth tokens without session")
}
timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
caller, err := gs.sm.Get(timeoutCtx, id)
if err != nil {
return err
}
for _, s := range sec {
dt, err := secrets.GetSecret(ctx, caller, s.name)
if err != nil {
if errors.Is(err, secrets.ErrNotFound) {
continue
}
return err
}
if s.token {
dt = []byte("basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("x-access-token:%s", dt))))
}
gs.auth = []string{"-c", "http.extraheader=Authorization: " + string(dt)}
break
}
return nil
}
func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bool, error) {
remote := gs.src.Remote
ref := gs.src.Ref
@ -188,7 +259,9 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo
return ref, true, nil
}
gitDir, unmountGitDir, err := gs.mountRemote(ctx, remote)
gs.getAuthToken(ctx)
gitDir, unmountGitDir, err := gs.mountRemote(ctx, remote, gs.auth)
if err != nil {
return "", false, err
}
@ -196,14 +269,14 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo
// TODO: should we assume that remote tag is immutable? add a timer?
buf, err := gitWithinDir(ctx, gitDir, "", "ls-remote", "origin", ref)
buf, err := gitWithinDir(ctx, gitDir, "", gs.auth, "ls-remote", "origin", ref)
if err != nil {
return "", false, errors.Wrapf(err, "failed to fetch remote %s", remote)
}
out := buf.String()
idx := strings.Index(out, "\t")
if idx == -1 {
return "", false, errors.Errorf("failed to find commit SHA from output: %s", string(out))
return "", false, errors.Errorf("repository does not contain ref %s, output: %q", ref, string(out))
}
sha := string(out[:idx])
@ -230,6 +303,8 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
}
}
gs.getAuthToken(ctx)
snapshotKey := "git-snapshot::" + cacheKey + ":" + gs.src.Subdir
gs.locker.Lock(snapshotKey)
defer gs.locker.Unlock(snapshotKey)
@ -244,7 +319,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
gs.locker.Lock(gs.src.Remote)
defer gs.locker.Unlock(gs.src.Remote)
gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote)
gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, gs.auth)
if err != nil {
return nil, err
}
@ -253,7 +328,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
doFetch := true
if isCommitSHA(ref) {
// skip fetch if commit already exists
if _, err := gitWithinDir(ctx, gitDir, "", "cat-file", "-e", ref+"^{commit}"); err == nil {
if _, err := gitWithinDir(ctx, gitDir, "", nil, "cat-file", "-e", ref+"^{commit}"); err == nil {
doFetch = false
}
}
@ -277,7 +352,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
// in case the ref is a branch and it now points to a different commit sha
// TODO: is there a better way to do this?
}
if _, err := gitWithinDir(ctx, gitDir, "", args...); err != nil {
if _, err := gitWithinDir(ctx, gitDir, "", gs.auth, args...); err != nil {
return nil, errors.Wrapf(err, "failed to fetch remote %s", gs.src.Remote)
}
}
@ -313,41 +388,41 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
if err := os.MkdirAll(checkoutDir, 0711); err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", "init")
_, err = gitWithinDir(ctx, checkoutDirGit, "", nil, "init")
if err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", "remote", "add", "origin", gitDir)
_, err = gitWithinDir(ctx, checkoutDirGit, "", nil, "remote", "add", "origin", gitDir)
if err != nil {
return nil, err
}
pullref := ref
if isCommitSHA(ref) {
pullref = "refs/buildkit/" + identity.NewID()
_, err = gitWithinDir(ctx, gitDir, "", "update-ref", pullref, ref)
_, err = gitWithinDir(ctx, gitDir, "", gs.auth, "update-ref", pullref, ref)
if err != nil {
return nil, err
}
} else {
pullref += ":" + pullref
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", "fetch", "-u", "--depth=1", "origin", pullref)
_, err = gitWithinDir(ctx, checkoutDirGit, "", gs.auth, "fetch", "-u", "--depth=1", "origin", pullref)
if err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, "checkout", "FETCH_HEAD")
_, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, nil, "checkout", "FETCH_HEAD")
if err != nil {
return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
}
gitDir = checkoutDirGit
} else {
_, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".")
_, err = gitWithinDir(ctx, gitDir, checkoutDir, nil, "checkout", ref, "--", ".")
if err != nil {
return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
}
}
_, err = gitWithinDir(ctx, gitDir, checkoutDir, "submodule", "update", "--init", "--recursive", "--depth=1")
_, err = gitWithinDir(ctx, gitDir, checkoutDir, gs.auth, "submodule", "update", "--init", "--recursive", "--depth=1")
if err != nil {
return nil, errors.Wrapf(err, "failed to update submodules for %s", gs.src.Remote)
}
@ -396,8 +471,8 @@ func isCommitSHA(str string) bool {
return validHex.MatchString(str)
}
func gitWithinDir(ctx context.Context, gitDir, workDir string, args ...string) (*bytes.Buffer, error) {
a := []string{"--git-dir", gitDir}
func gitWithinDir(ctx context.Context, gitDir, workDir string, auth []string, args ...string) (*bytes.Buffer, error) {
a := append([]string{"--git-dir", gitDir}, auth...)
if workDir != "" {
a = append(a, "--work-tree", workDir)
}
@ -413,8 +488,14 @@ func git(ctx context.Context, dir string, args ...string) (*bytes.Buffer, error)
cmd.Dir = dir // some commands like submodule require this
buf := bytes.NewBuffer(nil)
errbuf := bytes.NewBuffer(nil)
cmd.Stdin = nil
cmd.Stdout = io.MultiWriter(stdout, buf)
cmd.Stderr = io.MultiWriter(stderr, errbuf)
cmd.Env = []string{
"PATH=" + os.Getenv("PATH"),
"GIT_TERMINAL_PROMPT=0",
// "GIT_TRACE=1",
}
// remote git commands spawn helper processes that inherit FDs and don't
// handle parent death signal so exec.CommandContext can't be used
err := runProcessGroup(ctx, cmd)

10
source/gitidentifier.go
View File

@ -8,10 +8,12 @@ import (
)
type GitIdentifier struct {
Remote string
Ref string
Subdir string
KeepGitDir bool
Remote string
Ref string
Subdir string
KeepGitDir bool
AuthTokenSecret string
AuthHeaderSecret string
}
func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) {

4
source/identifier.go
View File

@ -103,6 +103,10 @@ func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) {
}
case pb.AttrFullRemoteURL:
id.Remote = v
case pb.AttrAuthHeaderSecret:
id.AuthHeaderSecret = v
case pb.AttrAuthTokenSecret:
id.AuthTokenSecret = v
}
}
}