git: support subdir component

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

solver/pb/caps.go

@@ -25,6 +25,7 @@ const (
 	CapSourceGitHTTPAuth      apicaps.CapID = "source.git.httpauth"
 	CapSourceGitKnownSSHHosts apicaps.CapID = "source.git.knownsshhosts"
 	CapSourceGitMountSSHSock  apicaps.CapID = "source.git.mountsshsock"
+	CapSourceGitSubdir        apicaps.CapID = "source.git.subdir"
 
 	CapSourceHTTP         apicaps.CapID = "source.http"
 	CapSourceHTTPChecksum apicaps.CapID = "source.http.checksum"
@@ -152,6 +153,12 @@ func init() {
 		Status:  apicaps.CapStatusExperimental,
 	})
 
+	Caps.Init(apicaps.Cap{
+		ID:      CapSourceGitSubdir,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapSourceHTTP,
 		Enabled: true,

source/git/gitsource.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"os/exec"
 	"os/user"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -170,6 +171,9 @@ func (gs *gitSourceHandler) shaToCacheKey(sha string) string {
 	if gs.src.KeepGitDir {
 		key += ".git"
 	}
+	if gs.src.Subdir != "" {
+		key += ":" + gs.src.Subdir
+	}
 	return key
 }
 
@@ -480,7 +484,12 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
 		}
 	}()
 
-	if gs.src.KeepGitDir {
+	subdir := path.Clean(gs.src.Subdir)
+	if subdir == "/" {
+		subdir = "."
+	}
+
+	if gs.src.KeepGitDir && subdir == "." {
 		checkoutDirGit := filepath.Join(checkoutDir, ".git")
 		if err := os.MkdirAll(checkoutDir, 0711); err != nil {
 			return nil, err
@@ -513,10 +522,44 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
 		}
 		gitDir = checkoutDirGit
 	} else {
-		_, err = gitWithinDir(ctx, gitDir, checkoutDir, sock, knownHosts, nil, "checkout", ref, "--", ".")
+		cd := checkoutDir
+		if subdir != "." {
+			cd, err = ioutil.TempDir(cd, "checkout")
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to create temporary checkout dir")
+			}
+		}
+		_, err = gitWithinDir(ctx, gitDir, cd, sock, knownHosts, nil, "checkout", ref, "--", ".")
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to checkout remote %s", redactCredentials(gs.src.Remote))
 		}
+		if subdir != "." {
+			d, err := os.Open(filepath.Join(cd, subdir))
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to open subdir %v", subdir)
+			}
+			defer func() {
+				if d != nil {
+					d.Close()
+				}
+			}()
+			names, err := d.Readdirnames(0)
+			if err != nil {
+				return nil, err
+			}
+			for _, n := range names {
+				if err := os.Rename(filepath.Join(cd, subdir, n), filepath.Join(checkoutDir, n)); err != nil {
+					return nil, err
+				}
+			}
+			if err := d.Close(); err != nil {
+				return nil, err
+			}
+			d = nil // reset defer
+			if err := os.RemoveAll(cd); err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	_, err = gitWithinDir(ctx, gitDir, checkoutDir, sock, knownHosts, gs.auth, "submodule", "update", "--init", "--recursive", "--depth=1")

source/git/gitsource_test.go

@@ -348,6 +348,82 @@ func TestCredentialRedaction(t *testing.T) {
 	require.False(t, strings.Contains(err.Error(), "keepthissecret"))
 }
 
+func TestSubdir(t *testing.T) {
+	testSubdir(t, false)
+}
+func TestSubdirKeepGitDir(t *testing.T) {
+	testSubdir(t, true)
+}
+
+func testSubdir(t *testing.T, keepGitDir bool) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Depends on unimplemented containerd bind-mount support on Windows")
+	}
+
+	t.Parallel()
+	ctx := context.TODO()
+
+	tmpdir, err := ioutil.TempDir("", "buildkit-state")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpdir)
+
+	gs := setupGitSource(t, tmpdir)
+
+	repodir, err := ioutil.TempDir("", "buildkit-gitsource")
+	require.NoError(t, err)
+	defer os.RemoveAll(repodir)
+
+	err = runShell(repodir,
+		"git init",
+		"git config --local user.email test",
+		"git config --local user.name test",
+		"echo foo > abc",
+		"mkdir sub",
+		"echo abc > sub/bar",
+		"git add abc sub",
+		"git commit -m initial",
+	)
+	require.NoError(t, err)
+
+	id := &source.GitIdentifier{Remote: repodir, KeepGitDir: keepGitDir, Subdir: "sub"}
+
+	g, err := gs.Resolve(ctx, id, nil, nil)
+	require.NoError(t, err)
+
+	key1, _, done, err := g.CacheKey(ctx, nil, 0)
+	require.NoError(t, err)
+	require.True(t, done)
+
+	expLen := 44
+	if keepGitDir {
+		expLen += 4
+	}
+
+	require.Equal(t, expLen, len(key1))
+
+	ref1, err := g.Snapshot(ctx, nil)
+	require.NoError(t, err)
+	defer ref1.Release(context.TODO())
+
+	mount, err := ref1.Mount(ctx, false, nil)
+	require.NoError(t, err)
+
+	lm := snapshot.LocalMounter(mount)
+	dir, err := lm.Mount()
+	require.NoError(t, err)
+	defer lm.Unmount()
+
+	fis, err := ioutil.ReadDir(dir)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(fis))
+
+	dt, err := ioutil.ReadFile(filepath.Join(dir, "bar"))
+	require.NoError(t, err)
+
+	require.Equal(t, "abc\n", string(dt))
+}
+
 func setupGitSource(t *testing.T, tmpdir string) source.Source {
 	snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
 	assert.NoError(t, err)
@@ -437,6 +513,7 @@ func runShell(dir string, cmds ...string) error {
 			cmd = exec.Command("sh", "-c", args)
 		}
 		cmd.Dir = dir
+		cmd.Stderr = os.Stderr
 		if err := cmd.Run(); err != nil {
 			return errors.Wrapf(err, "error running %v", args)
 		}

source/gitidentifier.go

@@ -2,10 +2,10 @@ package source
 
 import (
 	"net/url"
+	"path"
 	"strings"
 
 	"github.com/moby/buildkit/util/sshutil"
-	"github.com/pkg/errors"
 )
 
 type GitIdentifier struct {
@@ -46,8 +46,8 @@ func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) {
 		u.Fragment = ""
 		repo.Remote = u.String()
 	}
-	if repo.Subdir != "" {
-		return nil, errors.Errorf("subdir not supported yet")
+	if sd := path.Clean(repo.Subdir); sd == "/" || sd == "." {
+		repo.Subdir = ""
 	}
 	return &repo, nil
 }