2017-07-07 21:35:10 +00:00
package git
import (
"bytes"
2018-01-16 22:30:10 +00:00
"context"
2020-06-16 23:45:17 +00:00
"encoding/base64"
2017-07-25 19:11:52 +00:00
"fmt"
2017-07-07 21:35:10 +00:00
"io"
2020-11-10 20:42:33 +00:00
"io/ioutil"
2020-06-16 23:45:17 +00:00
"net/url"
2017-07-08 23:25:07 +00:00
"os"
2017-07-07 21:35:10 +00:00
"os/exec"
2020-11-10 20:42:33 +00:00
"os/user"
2021-05-20 04:10:58 +00:00
"path"
2017-07-08 23:25:07 +00:00
"path/filepath"
2017-07-07 21:35:10 +00:00
"regexp"
2020-11-10 20:42:33 +00:00
"strconv"
2017-07-07 21:35:10 +00:00
"strings"
"github.com/moby/buildkit/cache"
2018-07-26 19:07:52 +00:00
"github.com/moby/buildkit/client"
2017-07-08 23:25:07 +00:00
"github.com/moby/buildkit/identity"
2019-02-23 12:56:04 +00:00
"github.com/moby/buildkit/session"
2020-06-16 23:45:17 +00:00
"github.com/moby/buildkit/session/secrets"
2020-11-10 20:42:33 +00:00
"github.com/moby/buildkit/session/sshforward"
2017-07-07 21:35:10 +00:00
"github.com/moby/buildkit/snapshot"
2020-05-28 20:46:33 +00:00
"github.com/moby/buildkit/solver"
2017-07-07 21:35:10 +00:00
"github.com/moby/buildkit/source"
2021-09-17 23:39:19 +00:00
srctypes "github.com/moby/buildkit/source/types"
2021-07-09 00:09:35 +00:00
"github.com/moby/buildkit/util/bklog"
2017-07-07 21:35:10 +00:00
"github.com/moby/buildkit/util/progress/logs"
2021-09-14 15:39:03 +00:00
"github.com/moby/buildkit/util/urlutil"
2020-10-30 08:25:05 +00:00
"github.com/moby/locker"
2017-07-07 21:35:10 +00:00
"github.com/pkg/errors"
2020-11-10 20:42:33 +00:00
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
2017-07-07 21:35:10 +00:00
)
var validHex = regexp . MustCompile ( ` ^[a-f0-9] { 40}$ ` )
2021-03-13 03:52:27 +00:00
var defaultBranch = regexp . MustCompile ( ` refs/heads/(\S+) ` )
2017-07-07 21:35:10 +00:00
type Opt struct {
CacheAccessor cache . Accessor
}
type gitSource struct {
cache cache . Accessor
locker * locker . Locker
}
2018-11-28 09:43:43 +00:00
// Supported returns nil if the system supports Git source
func Supported ( ) error {
if err := exec . Command ( "git" , "version" ) . Run ( ) ; err != nil {
return errors . Wrap ( err , "failed to find git binary" )
}
return nil
}
2017-07-07 21:35:10 +00:00
func NewSource ( opt Opt ) ( source . Source , error ) {
gs := & gitSource {
cache : opt . CacheAccessor ,
2018-06-08 18:00:37 +00:00
locker : locker . New ( ) ,
2017-07-07 21:35:10 +00:00
}
return gs , nil
}
func ( gs * gitSource ) ID ( ) string {
2021-09-17 23:39:19 +00:00
return srctypes . GitScheme
2017-07-07 21:35:10 +00:00
}
// needs to be called with repo lock
2020-10-27 06:13:39 +00:00
func ( gs * gitSource ) mountRemote ( ctx context . Context , remote string , auth [ ] string , g session . Group ) ( target string , release func ( ) , retErr error ) {
2021-07-09 00:09:35 +00:00
sis , err := searchGitRemote ( ctx , gs . cache , remote )
2017-07-07 21:35:10 +00:00
if err != nil {
2021-09-14 15:39:03 +00:00
return "" , nil , errors . Wrapf ( err , "failed to search metadata for %s" , urlutil . RedactCredentials ( remote ) )
2017-07-07 21:35:10 +00:00
}
var remoteRef cache . MutableRef
for _ , si := range sis {
remoteRef , err = gs . cache . GetMutable ( ctx , si . ID ( ) )
if err != nil {
2020-04-19 05:17:47 +00:00
if errors . Is ( err , cache . ErrLocked ) {
2017-07-07 21:35:10 +00:00
// should never really happen as no other function should access this metadata, but lets be graceful
2021-09-14 15:39:03 +00:00
bklog . G ( ctx ) . Warnf ( "mutable ref for %s %s was locked: %v" , urlutil . RedactCredentials ( remote ) , si . ID ( ) , err )
2017-07-07 21:35:10 +00:00
continue
}
2021-09-14 15:39:03 +00:00
return "" , nil , errors . Wrapf ( err , "failed to get mutable ref for %s" , urlutil . RedactCredentials ( remote ) )
2017-07-07 21:35:10 +00:00
}
break
}
initializeRepo := false
if remoteRef == nil {
2021-09-14 15:39:03 +00:00
remoteRef , err = gs . cache . New ( ctx , nil , g , cache . CachePolicyRetain , cache . WithDescription ( fmt . Sprintf ( "shared git repo for %s" , urlutil . RedactCredentials ( remote ) ) ) )
2017-07-07 21:35:10 +00:00
if err != nil {
2021-09-14 15:39:03 +00:00
return "" , nil , errors . Wrapf ( err , "failed to create new mutable for %s" , urlutil . RedactCredentials ( remote ) )
2017-07-07 21:35:10 +00:00
}
initializeRepo = true
}
releaseRemoteRef := func ( ) {
2017-07-14 18:59:31 +00:00
remoteRef . Release ( context . TODO ( ) )
2017-07-07 21:35:10 +00:00
}
defer func ( ) {
if retErr != nil && remoteRef != nil {
releaseRemoteRef ( )
}
} ( )
2020-10-27 06:13:39 +00:00
mount , err := remoteRef . Mount ( ctx , false , g )
2017-07-07 21:35:10 +00:00
if err != nil {
return "" , nil , err
}
lm := snapshot . LocalMounter ( mount )
dir , err := lm . Mount ( )
if err != nil {
return "" , nil , err
}
defer func ( ) {
if retErr != nil {
lm . Unmount ( )
}
} ( )
if initializeRepo {
2020-11-10 20:42:33 +00:00
if _ , err := gitWithinDir ( ctx , dir , "" , "" , "" , auth , "init" , "--bare" ) ; err != nil {
2017-07-07 21:35:10 +00:00
return "" , nil , errors . Wrapf ( err , "failed to init repo at %s" , dir )
}
2020-11-10 20:42:33 +00:00
if _ , err := gitWithinDir ( ctx , dir , "" , "" , "" , auth , "remote" , "add" , "origin" , remote ) ; err != nil {
2017-07-07 21:35:10 +00:00
return "" , nil , errors . Wrapf ( err , "failed add origin repo at %s" , dir )
}
2021-07-09 00:09:35 +00:00
// save new remote metadata
md := cacheRefMetadata { remoteRef }
if err := md . setGitRemote ( remote ) ; err != nil {
2017-07-07 21:35:10 +00:00
return "" , nil , err
}
}
return dir , func ( ) {
lm . Unmount ( )
releaseRemoteRef ( )
} , nil
}
type gitSourceHandler struct {
* gitSource
2017-07-08 23:25:07 +00:00
src source . GitIdentifier
2017-07-07 21:35:10 +00:00
cacheKey string
2020-06-16 23:45:17 +00:00
sm * session . Manager
auth [ ] string
2017-07-07 21:35:10 +00:00
}
2019-08-14 19:12:25 +00:00
func ( gs * gitSourceHandler ) shaToCacheKey ( sha string ) string {
key := sha
if gs . src . KeepGitDir {
key += ".git"
}
2021-05-20 04:10:58 +00:00
if gs . src . Subdir != "" {
key += ":" + gs . src . Subdir
}
2019-08-14 19:12:25 +00:00
return key
}
2020-05-28 20:46:33 +00:00
func ( gs * gitSource ) Resolve ( ctx context . Context , id source . Identifier , sm * session . Manager , _ solver . Vertex ) ( source . SourceInstance , error ) {
2017-07-07 21:35:10 +00:00
gitIdentifier , ok := id . ( * source . GitIdentifier )
if ! ok {
return nil , errors . Errorf ( "invalid git identifier %v" , id )
}
return & gitSourceHandler {
2017-07-08 23:25:07 +00:00
src : * gitIdentifier ,
2017-07-07 21:35:10 +00:00
gitSource : gs ,
2020-06-16 23:45:17 +00:00
sm : sm ,
2017-07-07 21:35:10 +00:00
} , nil
}
2020-06-16 23:45:17 +00:00
type authSecret struct {
token bool
name string
}
func ( gs * gitSourceHandler ) authSecretNames ( ) ( sec [ ] authSecret , _ error ) {
u , err := url . Parse ( gs . src . Remote )
if err != nil {
return nil , err
}
if gs . src . AuthHeaderSecret != "" {
sec = append ( sec , authSecret { name : gs . src . AuthHeaderSecret + "." + u . Host } )
}
if gs . src . AuthTokenSecret != "" {
sec = append ( sec , authSecret { name : gs . src . AuthTokenSecret + "." + u . Host , token : true } )
}
if gs . src . AuthHeaderSecret != "" {
sec = append ( sec , authSecret { name : gs . src . AuthHeaderSecret } )
}
if gs . src . AuthTokenSecret != "" {
sec = append ( sec , authSecret { name : gs . src . AuthTokenSecret , token : true } )
}
return sec , nil
}
2020-06-30 01:06:02 +00:00
func ( gs * gitSourceHandler ) getAuthToken ( ctx context . Context , g session . Group ) error {
2020-06-16 23:45:17 +00:00
if gs . auth != nil {
return nil
}
sec , err := gs . authSecretNames ( )
if err != nil {
return err
}
2020-06-30 01:06:02 +00:00
return gs . sm . Any ( ctx , g , func ( ctx context . Context , _ string , caller session . Caller ) error {
for _ , s := range sec {
dt , err := secrets . GetSecret ( ctx , caller , s . name )
if err != nil {
if errors . Is ( err , secrets . ErrNotFound ) {
continue
}
return err
2020-06-16 23:45:17 +00:00
}
2020-06-30 01:06:02 +00:00
if s . token {
dt = [ ] byte ( "basic " + base64 . StdEncoding . EncodeToString ( [ ] byte ( fmt . Sprintf ( "x-access-token:%s" , dt ) ) ) )
}
2021-02-19 08:22:02 +00:00
gs . auth = [ ] string { "-c" , "http." + tokenScope ( gs . src . Remote ) + ".extraheader=Authorization: " + string ( dt ) }
2020-06-30 01:06:02 +00:00
break
2020-06-16 23:45:17 +00:00
}
2020-06-30 01:06:02 +00:00
return nil
} )
2020-06-16 23:45:17 +00:00
}
2020-11-13 01:29:49 +00:00
func ( gs * gitSourceHandler ) mountSSHAuthSock ( ctx context . Context , sshID string , g session . Group ) ( string , func ( ) error , error ) {
2020-11-10 20:42:33 +00:00
var caller session . Caller
err := gs . sm . Any ( ctx , g , func ( ctx context . Context , _ string , c session . Caller ) error {
if err := sshforward . CheckSSHID ( ctx , c , sshID ) ; err != nil {
if st , ok := status . FromError ( err ) ; ok && st . Code ( ) == codes . Unimplemented {
return errors . Errorf ( "no SSH key %q forwarded from the client" , sshID )
}
return err
}
caller = c
return nil
} )
if err != nil {
return "" , nil , err
}
usr , err := user . Current ( )
if err != nil {
return "" , nil , err
}
2020-11-13 01:29:49 +00:00
// best effort, default to root
2020-11-10 20:42:33 +00:00
uid , _ := strconv . Atoi ( usr . Uid )
2020-11-13 01:29:49 +00:00
gid , _ := strconv . Atoi ( usr . Gid )
2020-11-10 20:42:33 +00:00
sock , cleanup , err := sshforward . MountSSHSocket ( ctx , caller , sshforward . SocketOpt {
ID : sshID ,
UID : uid ,
GID : gid ,
2020-11-11 06:28:30 +00:00
Mode : 0700 ,
2020-11-10 20:42:33 +00:00
} )
if err != nil {
return "" , nil , err
}
return sock , cleanup , nil
}
func ( gs * gitSourceHandler ) mountKnownHosts ( ctx context . Context ) ( string , func ( ) error , error ) {
if gs . src . KnownSSHHosts == "" {
return "" , nil , errors . Errorf ( "no configured known hosts forwarded from the client" )
}
knownHosts , err := ioutil . TempFile ( "" , "" )
if err != nil {
return "" , nil , err
}
cleanup := func ( ) error {
return os . Remove ( knownHosts . Name ( ) )
}
_ , err = knownHosts . Write ( [ ] byte ( gs . src . KnownSSHHosts ) )
if err != nil {
cleanup ( )
return "" , nil , err
}
err = knownHosts . Close ( )
if err != nil {
cleanup ( )
return "" , nil , err
}
return knownHosts . Name ( ) , cleanup , nil
}
2021-09-17 23:39:19 +00:00
func ( gs * gitSourceHandler ) CacheKey ( ctx context . Context , g session . Group , index int ) ( string , string , solver . CacheOpts , bool , error ) {
2017-07-07 21:35:10 +00:00
remote := gs . src . Remote
gs . locker . Lock ( remote )
defer gs . locker . Unlock ( remote )
2021-07-06 23:34:43 +00:00
if ref := gs . src . Ref ; ref != "" && isCommitSHA ( ref ) {
2021-09-17 23:39:19 +00:00
cacheKey := gs . shaToCacheKey ( ref )
gs . cacheKey = cacheKey
return cacheKey , ref , nil , true , nil
2021-07-06 23:34:43 +00:00
}
2020-06-30 01:06:02 +00:00
gs . getAuthToken ( ctx , g )
2020-06-16 23:45:17 +00:00
2020-10-27 06:13:39 +00:00
gitDir , unmountGitDir , err := gs . mountRemote ( ctx , remote , gs . auth , g )
2017-07-07 21:35:10 +00:00
if err != nil {
2021-09-17 23:39:19 +00:00
return "" , "" , nil , false , err
2017-07-07 21:35:10 +00:00
}
defer unmountGitDir ( )
2020-11-10 20:42:33 +00:00
var sock string
2020-11-13 01:29:49 +00:00
if gs . src . MountSSHSock != "" {
2020-11-10 20:42:33 +00:00
var unmountSock func ( ) error
2020-11-13 01:29:49 +00:00
sock , unmountSock , err = gs . mountSSHAuthSock ( ctx , gs . src . MountSSHSock , g )
2020-11-10 20:42:33 +00:00
if err != nil {
2021-09-17 23:39:19 +00:00
return "" , "" , nil , false , err
2020-11-10 20:42:33 +00:00
}
defer unmountSock ( )
}
var knownHosts string
if gs . src . KnownSSHHosts != "" {
var unmountKnownHosts func ( ) error
knownHosts , unmountKnownHosts , err = gs . mountKnownHosts ( ctx )
if err != nil {
2021-09-17 23:39:19 +00:00
return "" , "" , nil , false , err
2020-11-10 20:42:33 +00:00
}
defer unmountKnownHosts ( )
}
2021-03-13 03:52:27 +00:00
ref := gs . src . Ref
if ref == "" {
ref , err = getDefaultBranch ( ctx , gitDir , "" , sock , knownHosts , gs . auth , gs . src . Remote )
if err != nil {
2021-09-17 23:39:19 +00:00
return "" , "" , nil , false , err
2021-03-13 03:52:27 +00:00
}
}
2017-07-07 21:35:10 +00:00
// TODO: should we assume that remote tag is immutable? add a timer?
2020-11-10 20:42:33 +00:00
buf , err := gitWithinDir ( ctx , gitDir , "" , sock , knownHosts , gs . auth , "ls-remote" , "origin" , ref )
2017-07-07 21:35:10 +00:00
if err != nil {
2021-09-17 23:39:19 +00:00
return "" , "" , nil , false , errors . Wrapf ( err , "failed to fetch remote %s" , urlutil . RedactCredentials ( remote ) )
2017-07-07 21:35:10 +00:00
}
out := buf . String ( )
idx := strings . Index ( out , "\t" )
if idx == - 1 {
2021-09-17 23:39:19 +00:00
return "" , "" , nil , false , errors . Errorf ( "repository does not contain ref %s, output: %q" , ref , string ( out ) )
2017-07-07 21:35:10 +00:00
}
sha := string ( out [ : idx ] )
if ! isCommitSHA ( sha ) {
2021-09-17 23:39:19 +00:00
return "" , "" , nil , false , errors . Errorf ( "invalid commit sha %q" , sha )
2017-07-07 21:35:10 +00:00
}
2021-09-17 23:39:19 +00:00
cacheKey := gs . shaToCacheKey ( sha )
gs . cacheKey = cacheKey
return cacheKey , sha , nil , true , nil
2017-07-07 21:35:10 +00:00
}
2020-06-30 01:06:02 +00:00
func ( gs * gitSourceHandler ) Snapshot ( ctx context . Context , g session . Group ) ( out cache . ImmutableRef , retErr error ) {
2017-07-07 21:35:10 +00:00
cacheKey := gs . cacheKey
if cacheKey == "" {
var err error
2021-09-17 23:39:19 +00:00
cacheKey , _ , _ , _ , err = gs . CacheKey ( ctx , g , 0 )
2017-07-07 21:35:10 +00:00
if err != nil {
return nil , err
}
}
2020-06-30 01:06:02 +00:00
gs . getAuthToken ( ctx , g )
2020-06-16 23:45:17 +00:00
2021-07-09 00:09:35 +00:00
snapshotKey := cacheKey + ":" + gs . src . Subdir
2017-07-07 21:35:10 +00:00
gs . locker . Lock ( snapshotKey )
defer gs . locker . Unlock ( snapshotKey )
2021-07-09 00:09:35 +00:00
sis , err := searchGitSnapshot ( ctx , gs . cache , snapshotKey )
2017-07-07 21:35:10 +00:00
if err != nil {
return nil , errors . Wrapf ( err , "failed to search metadata for %s" , snapshotKey )
}
if len ( sis ) > 0 {
2022-02-03 18:54:24 +00:00
return gs . cache . Get ( ctx , sis [ 0 ] . ID ( ) , nil )
2017-07-07 21:35:10 +00:00
}
2017-07-21 17:58:24 +00:00
gs . locker . Lock ( gs . src . Remote )
defer gs . locker . Unlock ( gs . src . Remote )
2020-10-27 06:13:39 +00:00
gitDir , unmountGitDir , err := gs . mountRemote ( ctx , gs . src . Remote , gs . auth , g )
2017-07-07 21:35:10 +00:00
if err != nil {
return nil , err
}
defer unmountGitDir ( )
2020-11-10 20:42:33 +00:00
var sock string
2020-11-13 01:29:49 +00:00
if gs . src . MountSSHSock != "" {
2020-11-10 20:42:33 +00:00
var unmountSock func ( ) error
2020-11-13 01:29:49 +00:00
sock , unmountSock , err = gs . mountSSHAuthSock ( ctx , gs . src . MountSSHSock , g )
2020-11-10 20:42:33 +00:00
if err != nil {
return nil , err
}
defer unmountSock ( )
}
var knownHosts string
if gs . src . KnownSSHHosts != "" {
var unmountKnownHosts func ( ) error
knownHosts , unmountKnownHosts , err = gs . mountKnownHosts ( ctx )
if err != nil {
return nil , err
}
defer unmountKnownHosts ( )
}
2021-03-13 03:52:27 +00:00
ref := gs . src . Ref
if ref == "" {
ref , err = getDefaultBranch ( ctx , gitDir , "" , sock , knownHosts , gs . auth , gs . src . Remote )
if err != nil {
return nil , err
}
}
2017-07-07 21:35:10 +00:00
doFetch := true
if isCommitSHA ( ref ) {
// skip fetch if commit already exists
2020-11-10 20:42:33 +00:00
if _ , err := gitWithinDir ( ctx , gitDir , "" , sock , knownHosts , nil , "cat-file" , "-e" , ref + "^{commit}" ) ; err == nil {
2017-07-07 21:35:10 +00:00
doFetch = false
}
}
if doFetch {
2019-01-22 22:04:54 +00:00
// make sure no old lock files have leaked
os . RemoveAll ( filepath . Join ( gitDir , "shallow.lock" ) )
2017-12-09 18:09:23 +00:00
args := [ ] string { "fetch" }
2017-07-07 21:35:10 +00:00
if ! isCommitSHA ( ref ) { // TODO: find a branch from ls-remote?
args = append ( args , "--depth=1" , "--no-tags" )
2017-07-08 23:25:07 +00:00
} else {
if _ , err := os . Lstat ( filepath . Join ( gitDir , "shallow" ) ) ; err == nil {
args = append ( args , "--unshallow" )
}
2017-07-07 21:35:10 +00:00
}
args = append ( args , "origin" )
if ! isCommitSHA ( ref ) {
2020-03-28 00:37:01 +00:00
args = append ( args , "--force" , ref + ":tags/" + ref )
2020-03-28 01:34:58 +00:00
// local refs are needed so they would be advertised on next fetches. Force is used
// in case the ref is a branch and it now points to a different commit sha
2017-07-07 21:35:10 +00:00
// TODO: is there a better way to do this?
}
2020-11-10 20:42:33 +00:00
if _ , err := gitWithinDir ( ctx , gitDir , "" , sock , knownHosts , gs . auth , args ... ) ; err != nil {
2021-09-14 15:39:03 +00:00
return nil , errors . Wrapf ( err , "failed to fetch remote %s" , urlutil . RedactCredentials ( gs . src . Remote ) )
2017-07-07 21:35:10 +00:00
}
2021-10-06 06:59:33 +00:00
_ , err = gitWithinDir ( ctx , gitDir , "" , sock , knownHosts , nil , "reflog" , "expire" , "--all" , "--expire=now" )
if err != nil {
return nil , errors . Wrapf ( err , "failed to expire reflog for remote %s" , urlutil . RedactCredentials ( gs . src . Remote ) )
}
2017-07-07 21:35:10 +00:00
}
2020-10-27 06:13:39 +00:00
checkoutRef , err := gs . cache . New ( ctx , nil , g , cache . WithRecordType ( client . UsageRecordTypeGitCheckout ) , cache . WithDescription ( fmt . Sprintf ( "git snapshot for %s#%s" , gs . src . Remote , ref ) ) )
2017-07-07 21:35:10 +00:00
if err != nil {
2021-09-14 15:39:03 +00:00
return nil , errors . Wrapf ( err , "failed to create new mutable for %s" , urlutil . RedactCredentials ( gs . src . Remote ) )
2017-07-07 21:35:10 +00:00
}
defer func ( ) {
if retErr != nil && checkoutRef != nil {
2017-07-14 18:59:31 +00:00
checkoutRef . Release ( context . TODO ( ) )
2017-07-07 21:35:10 +00:00
}
} ( )
2020-10-27 06:13:39 +00:00
mount , err := checkoutRef . Mount ( ctx , false , g )
2017-07-07 21:35:10 +00:00
if err != nil {
return nil , err
}
lm := snapshot . LocalMounter ( mount )
checkoutDir , err := lm . Mount ( )
if err != nil {
return nil , err
}
2017-07-08 23:25:07 +00:00
defer func ( ) {
if retErr != nil && lm != nil {
lm . Unmount ( )
}
} ( )
2017-07-07 21:35:10 +00:00
2021-05-20 04:10:58 +00:00
subdir := path . Clean ( gs . src . Subdir )
if subdir == "/" {
subdir = "."
}
if gs . src . KeepGitDir && subdir == "." {
2019-08-14 19:12:25 +00:00
checkoutDirGit := filepath . Join ( checkoutDir , ".git" )
if err := os . MkdirAll ( checkoutDir , 0711 ) ; err != nil {
return nil , err
}
2020-11-10 20:42:33 +00:00
_ , err = gitWithinDir ( ctx , checkoutDirGit , "" , sock , knownHosts , nil , "init" )
2017-07-08 23:25:07 +00:00
if err != nil {
return nil , err
}
2020-11-10 20:42:33 +00:00
_ , err = gitWithinDir ( ctx , checkoutDirGit , "" , sock , knownHosts , nil , "remote" , "add" , "origin" , gitDir )
2017-07-08 23:25:07 +00:00
if err != nil {
return nil , err
}
2022-01-19 19:09:17 +00:00
gitCatFileBuf , err := gitWithinDir ( ctx , gitDir , "" , sock , knownHosts , gs . auth , "cat-file" , "-t" , ref )
if err != nil {
return nil , err
}
isAnnotatedTag := strings . TrimSpace ( gitCatFileBuf . String ( ) ) == "tag"
2017-07-08 23:25:07 +00:00
pullref := ref
2022-01-19 19:09:17 +00:00
if isAnnotatedTag {
pullref += ":refs/tags/" + pullref
} else if isCommitSHA ( ref ) {
2017-07-08 23:25:07 +00:00
pullref = "refs/buildkit/" + identity . NewID ( )
2020-11-10 20:42:33 +00:00
_ , err = gitWithinDir ( ctx , gitDir , "" , sock , knownHosts , gs . auth , "update-ref" , pullref , ref )
2017-07-08 23:25:07 +00:00
if err != nil {
return nil , err
}
2019-08-14 19:12:25 +00:00
} else {
pullref += ":" + pullref
2017-07-08 23:25:07 +00:00
}
2020-11-10 20:42:33 +00:00
_ , err = gitWithinDir ( ctx , checkoutDirGit , "" , sock , knownHosts , gs . auth , "fetch" , "-u" , "--depth=1" , "origin" , pullref )
2017-07-08 23:25:07 +00:00
if err != nil {
return nil , err
}
2020-11-10 20:42:33 +00:00
_ , err = gitWithinDir ( ctx , checkoutDirGit , checkoutDir , sock , knownHosts , nil , "checkout" , "FETCH_HEAD" )
2017-07-08 23:25:07 +00:00
if err != nil {
2021-09-14 15:39:03 +00:00
return nil , errors . Wrapf ( err , "failed to checkout remote %s" , urlutil . RedactCredentials ( gs . src . Remote ) )
2017-07-08 23:25:07 +00:00
}
2021-10-06 06:59:33 +00:00
_ , err = gitWithinDir ( ctx , checkoutDirGit , "" , sock , knownHosts , nil , "remote" , "set-url" , "origin" , urlutil . RedactCredentials ( gs . src . Remote ) )
if err != nil {
return nil , errors . Wrapf ( err , "failed to set remote origin to %s" , urlutil . RedactCredentials ( gs . src . Remote ) )
}
_ , err = gitWithinDir ( ctx , checkoutDirGit , "" , sock , knownHosts , nil , "reflog" , "expire" , "--all" , "--expire=now" )
if err != nil {
return nil , errors . Wrapf ( err , "failed to expire reflog for remote %s" , urlutil . RedactCredentials ( gs . src . Remote ) )
}
if err := os . Remove ( filepath . Join ( checkoutDirGit , "FETCH_HEAD" ) ) ; err != nil && ! errors . Is ( err , os . ErrNotExist ) {
return nil , errors . Wrapf ( err , "failed to remove FETCH_HEAD for remote %s" , urlutil . RedactCredentials ( gs . src . Remote ) )
}
2019-08-14 19:12:25 +00:00
gitDir = checkoutDirGit
2017-07-08 23:25:07 +00:00
} else {
2021-05-20 04:10:58 +00:00
cd := checkoutDir
if subdir != "." {
cd , err = ioutil . TempDir ( cd , "checkout" )
if err != nil {
return nil , errors . Wrapf ( err , "failed to create temporary checkout dir" )
}
}
_ , err = gitWithinDir ( ctx , gitDir , cd , sock , knownHosts , nil , "checkout" , ref , "--" , "." )
2017-07-08 23:25:07 +00:00
if err != nil {
2021-09-14 15:39:03 +00:00
return nil , errors . Wrapf ( err , "failed to checkout remote %s" , urlutil . RedactCredentials ( gs . src . Remote ) )
2017-07-08 23:25:07 +00:00
}
2021-05-20 04:10:58 +00:00
if subdir != "." {
d , err := os . Open ( filepath . Join ( cd , subdir ) )
if err != nil {
return nil , errors . Wrapf ( err , "failed to open subdir %v" , subdir )
}
defer func ( ) {
if d != nil {
d . Close ( )
}
} ( )
names , err := d . Readdirnames ( 0 )
if err != nil {
return nil , err
}
for _ , n := range names {
if err := os . Rename ( filepath . Join ( cd , subdir , n ) , filepath . Join ( checkoutDir , n ) ) ; err != nil {
return nil , err
}
}
if err := d . Close ( ) ; err != nil {
return nil , err
}
d = nil // reset defer
if err := os . RemoveAll ( cd ) ; err != nil {
return nil , err
}
}
2017-07-07 21:35:10 +00:00
}
2017-12-09 18:09:23 +00:00
2020-11-10 20:42:33 +00:00
_ , err = gitWithinDir ( ctx , gitDir , checkoutDir , sock , knownHosts , gs . auth , "submodule" , "update" , "--init" , "--recursive" , "--depth=1" )
2017-12-09 18:09:23 +00:00
if err != nil {
2021-09-14 15:39:03 +00:00
return nil , errors . Wrapf ( err , "failed to update submodules for %s" , urlutil . RedactCredentials ( gs . src . Remote ) )
2017-12-09 18:09:23 +00:00
}
2019-03-20 21:54:20 +00:00
if idmap := mount . IdentityMapping ( ) ; idmap != nil {
u := idmap . RootPair ( )
err := filepath . Walk ( gitDir , func ( p string , f os . FileInfo , err error ) error {
return os . Lchown ( p , u . UID , u . GID )
} )
if err != nil {
return nil , errors . Wrap ( err , "failed to remap git checkout" )
}
}
2017-07-08 23:25:07 +00:00
lm . Unmount ( )
lm = nil
2017-07-07 21:35:10 +00:00
2017-07-14 18:59:31 +00:00
snap , err := checkoutRef . Commit ( ctx )
2017-07-07 21:35:10 +00:00
if err != nil {
return nil , err
}
checkoutRef = nil
defer func ( ) {
if retErr != nil {
snap . Release ( context . TODO ( ) )
}
} ( )
2021-07-09 00:09:35 +00:00
md := cacheRefMetadata { snap }
if err := md . setGitSnapshot ( snapshotKey ) ; err != nil {
2017-07-07 21:35:10 +00:00
return nil , err
}
return snap , nil
}
func isCommitSHA ( str string ) bool {
return validHex . MatchString ( str )
}
2020-11-10 20:42:33 +00:00
func gitWithinDir ( ctx context . Context , gitDir , workDir , sshAuthSock , knownHosts string , auth [ ] string , args ... string ) ( * bytes . Buffer , error ) {
2020-06-16 23:45:17 +00:00
a := append ( [ ] string { "--git-dir" , gitDir } , auth ... )
2017-07-07 21:35:10 +00:00
if workDir != "" {
a = append ( a , "--work-tree" , workDir )
}
2020-11-10 20:42:33 +00:00
return git ( ctx , workDir , sshAuthSock , knownHosts , append ( a , args ... ) ... )
2017-07-07 21:35:10 +00:00
}
2020-11-13 03:17:00 +00:00
func getGitSSHCommand ( knownHosts string ) string {
gitSSHCommand := "ssh -F /dev/null"
if knownHosts != "" {
gitSSHCommand += " -o UserKnownHostsFile=" + knownHosts
} else {
gitSSHCommand += " -o StrictHostKeyChecking=no"
}
return gitSSHCommand
}
2022-01-25 03:18:20 +00:00
func git ( ctx context . Context , dir , sshAuthSock , knownHosts string , args ... string ) ( _ * bytes . Buffer , err error ) {
2017-12-16 07:17:06 +00:00
for {
2022-01-25 03:18:20 +00:00
stdout , stderr , flush := logs . NewLogStreams ( ctx , false )
2017-12-16 07:17:06 +00:00
defer stdout . Close ( )
defer stderr . Close ( )
2022-01-25 03:18:20 +00:00
defer func ( ) {
if err != nil {
flush ( )
}
} ( )
2018-04-11 00:22:27 +00:00
cmd := exec . Command ( "git" , args ... )
2017-12-16 07:17:06 +00:00
cmd . Dir = dir // some commands like submodule require this
buf := bytes . NewBuffer ( nil )
errbuf := bytes . NewBuffer ( nil )
2020-06-16 23:45:17 +00:00
cmd . Stdin = nil
2017-12-16 07:17:06 +00:00
cmd . Stdout = io . MultiWriter ( stdout , buf )
cmd . Stderr = io . MultiWriter ( stderr , errbuf )
2020-06-16 23:45:17 +00:00
cmd . Env = [ ] string {
"PATH=" + os . Getenv ( "PATH" ) ,
"GIT_TERMINAL_PROMPT=0" ,
2020-11-13 03:17:00 +00:00
"GIT_SSH_COMMAND=" + getGitSSHCommand ( knownHosts ) ,
2020-06-16 23:45:17 +00:00
// "GIT_TRACE=1",
}
2020-11-10 20:42:33 +00:00
if sshAuthSock != "" {
cmd . Env = append ( cmd . Env , "SSH_AUTH_SOCK=" + sshAuthSock )
}
2018-04-11 00:22:27 +00:00
// remote git commands spawn helper processes that inherit FDs and don't
// handle parent death signal so exec.CommandContext can't be used
err := runProcessGroup ( ctx , cmd )
2017-12-16 07:17:06 +00:00
if err != nil {
if strings . Contains ( errbuf . String ( ) , "--depth" ) || strings . Contains ( errbuf . String ( ) , "shallow" ) {
if newArgs := argsNoDepth ( args ) ; len ( args ) > len ( newArgs ) {
args = newArgs
continue
}
}
}
return buf , err
}
}
func argsNoDepth ( args [ ] string ) [ ] string {
out := make ( [ ] string , 0 , len ( args ) )
for _ , a := range args {
if a != "--depth=1" {
out = append ( out , a )
}
}
return out
2017-07-07 21:35:10 +00:00
}
2021-02-19 08:22:02 +00:00
func tokenScope ( remote string ) string {
// generally we can only use the token for fetching main remote but in case of github.com we do best effort
// to try reuse same token for all github.com remotes. This is the same behavior actions/checkout uses
for _ , pfx := range [ ] string { "https://github.com/" , "https://www.github.com/" } {
if strings . HasPrefix ( remote , pfx ) {
return pfx
}
}
return remote
}
2021-03-13 03:52:27 +00:00
// getDefaultBranch gets the default branch of a repository using ls-remote
func getDefaultBranch ( ctx context . Context , gitDir , workDir , sshAuthSock , knownHosts string , auth [ ] string , remoteURL string ) ( string , error ) {
buf , err := gitWithinDir ( ctx , gitDir , workDir , sshAuthSock , knownHosts , auth , "ls-remote" , "--symref" , remoteURL , "HEAD" )
if err != nil {
2021-09-14 15:39:03 +00:00
return "" , errors . Wrapf ( err , "error fetching default branch for repository %s" , urlutil . RedactCredentials ( remoteURL ) )
2021-03-13 03:52:27 +00:00
}
ss := defaultBranch . FindAllStringSubmatch ( buf . String ( ) , - 1 )
if len ( ss ) == 0 || len ( ss [ 0 ] ) != 2 {
2021-09-14 15:39:03 +00:00
return "" , errors . Errorf ( "could not find default branch for repository: %s" , urlutil . RedactCredentials ( remoteURL ) )
2021-03-13 03:52:27 +00:00
}
return ss [ 0 ] [ 1 ] , nil
}
2021-07-09 00:09:35 +00:00
const keyGitRemote = "git-remote"
const gitRemoteIndex = keyGitRemote + "::"
const keyGitSnapshot = "git-snapshot"
const gitSnapshotIndex = keyGitSnapshot + "::"
func search ( ctx context . Context , store cache . MetadataStore , key string , idx string ) ( [ ] cacheRefMetadata , error ) {
var results [ ] cacheRefMetadata
mds , err := store . Search ( ctx , idx + key )
if err != nil {
return nil , err
}
for _ , md := range mds {
results = append ( results , cacheRefMetadata { md } )
}
return results , nil
}
func searchGitRemote ( ctx context . Context , store cache . MetadataStore , remote string ) ( [ ] cacheRefMetadata , error ) {
return search ( ctx , store , remote , gitRemoteIndex )
}
func searchGitSnapshot ( ctx context . Context , store cache . MetadataStore , key string ) ( [ ] cacheRefMetadata , error ) {
return search ( ctx , store , key , gitSnapshotIndex )
}
type cacheRefMetadata struct {
cache . RefMetadata
}
func ( md cacheRefMetadata ) setGitSnapshot ( key string ) error {
return md . SetString ( keyGitSnapshot , key , gitSnapshotIndex + key )
}
func ( md cacheRefMetadata ) setGitRemote ( key string ) error {
return md . SetString ( keyGitRemote , key , gitRemoteIndex + key )
}