fix: provide only available capabilities to insecure environment

The problem this change is trying to fix are the environments where some
capabilities are already dropped, so they can't be granted to the
job with `--security=insecure`.

I know that probably fixed set of capabilities was implemented to
provide a stable build environment, but at the same time this breaks
environments with reduced capabilities.

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>

client/build_test.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -1518,29 +1519,36 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox
 
 	product := "buildkit_test"
 
-	var command []string
+	command := []string{"sh", "-c", `cat /proc/self/status | grep CapEff | cut -f 2`}
 	mode := llb.SecurityModeSandbox
 	var allowedEntitlements []entitlements.Entitlement
+	var assertCaps func(caps uint64)
 	secMode := sb.Value("secmode")
 	if secMode == securitySandbox {
-		/*
+		assertCaps = func(caps uint64) {
-			$ capsh --decode=00000000a80425fb
+			/*
-			0x00000000a80425fb=cap_chown,cap_dac_override,cap_fowner,cap_fsetid,cap_kill,cap_setgid,cap_setuid,cap_setpcap,
+				$ capsh --decode=00000000a80425fb
-			cap_net_bind_service,cap_net_raw,cap_sys_chroot,cap_mknod,cap_audit_write,cap_setfcap
+				0x00000000a80425fb=cap_chown,cap_dac_override,cap_fowner,cap_fsetid,cap_kill,cap_setgid,cap_setuid,cap_setpcap,
-		*/
+				cap_net_bind_service,cap_net_raw,cap_sys_chroot,cap_mknod,cap_audit_write,cap_setfcap
-		command = []string{"sh", "-c", `cat /proc/self/status | grep CapEff | grep "00000000a80425fb"`}
+			*/
+			require.EqualValues(t, 0xa80425fb, caps)
+		}
 		allowedEntitlements = []entitlements.Entitlement{}
 	} else {
 		skipDockerd(t, sb)
-		/*
+		assertCaps = func(caps uint64) {
-			$ capsh --decode=0000003fffffffff
+			/*
-			0x0000003fffffffff=cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_kill,cap_setgid,
+				$ capsh --decode=0000003fffffffff
-			cap_setuid,cap_setpcap,cap_linux_immutable,cap_net_bind_service,cap_net_broadcast,cap_net_admin,cap_net_raw,
+				0x0000003fffffffff=cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_kill,cap_setgid,
-			cap_ipc_lock,cap_ipc_owner,cap_sys_module,cap_sys_rawio,cap_sys_chroot,cap_sys_ptrace,cap_sys_pacct,cap_sys_admin,
+				cap_setuid,cap_setpcap,cap_linux_immutable,cap_net_bind_service,cap_net_broadcast,cap_net_admin,cap_net_raw,
-			cap_sys_boot,cap_sys_nice,cap_sys_resource,cap_sys_time,cap_sys_tty_config,cap_mknod,cap_lease,cap_audit_write,
+				cap_ipc_lock,cap_ipc_owner,cap_sys_module,cap_sys_rawio,cap_sys_chroot,cap_sys_ptrace,cap_sys_pacct,cap_sys_admin,
-			cap_audit_control,cap_setfcap,cap_mac_override,cap_mac_admin,cap_syslog,cap_wake_alarm,cap_block_suspend,cap_audit_read
+				cap_sys_boot,cap_sys_nice,cap_sys_resource,cap_sys_time,cap_sys_tty_config,cap_mknod,cap_lease,cap_audit_write,
-		*/
+				cap_audit_control,cap_setfcap,cap_mac_override,cap_mac_admin,cap_syslog,cap_wake_alarm,cap_block_suspend,cap_audit_read
-		command = []string{"sh", "-c", `cat /proc/self/status | grep CapEff | grep "0000003fffffffff"`}
+			*/
+
+			// require that _at least_ minimum capabilities are granted
+			require.EqualValues(t, 0x3fffffffff, caps&0x3fffffffff)
+		}
 		mode = llb.SecurityModeInsecure
 		allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}
 	}
@@ -1594,6 +1602,11 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox
 
 		require.NoError(t, err)
 
+		capsValue, err := strconv.ParseUint(strings.TrimSpace(stdout.String()), 16, 64)
+		require.NoError(t, err)
+
+		assertCaps(capsValue)
+
 		return &client.Result{}, nil
 	}
 

client/client_test.go

@@ -17,6 +17,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"testing"
@@ -676,29 +677,36 @@ func testPushByDigest(t *testing.T, sb integration.Sandbox) {
 }
 
 func testSecurityMode(t *testing.T, sb integration.Sandbox) {
-	var command string
+	command := `sh -c 'cat /proc/self/status | grep CapEff | cut -f 2 > /out'`
 	mode := llb.SecurityModeSandbox
 	var allowedEntitlements []entitlements.Entitlement
+	var assertCaps func(caps uint64)
 	secMode := sb.Value("secmode")
 	if secMode == securitySandbox {
-		/*
+		assertCaps = func(caps uint64) {
-			$ capsh --decode=00000000a80425fb
+			/*
-			0x00000000a80425fb=cap_chown,cap_dac_override,cap_fowner,cap_fsetid,cap_kill,cap_setgid,cap_setuid,cap_setpcap,
+				$ capsh --decode=00000000a80425fb
-			cap_net_bind_service,cap_net_raw,cap_sys_chroot,cap_mknod,cap_audit_write,cap_setfcap
+				0x00000000a80425fb=cap_chown,cap_dac_override,cap_fowner,cap_fsetid,cap_kill,cap_setgid,cap_setuid,cap_setpcap,
-		*/
+				cap_net_bind_service,cap_net_raw,cap_sys_chroot,cap_mknod,cap_audit_write,cap_setfcap
-		command = `sh -c 'cat /proc/self/status | grep CapEff | grep "00000000a80425fb"'`
+			*/
+			require.EqualValues(t, 0xa80425fb, caps)
+		}
 		allowedEntitlements = []entitlements.Entitlement{}
 	} else {
 		skipDockerd(t, sb)
-		/*
+		assertCaps = func(caps uint64) {
-			$ capsh --decode=0000003fffffffff
+			/*
-			0x0000003fffffffff=cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_kill,cap_setgid,
+				$ capsh --decode=0000003fffffffff
-			cap_setuid,cap_setpcap,cap_linux_immutable,cap_net_bind_service,cap_net_broadcast,cap_net_admin,cap_net_raw,
+				0x0000003fffffffff=cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_kill,cap_setgid,
-			cap_ipc_lock,cap_ipc_owner,cap_sys_module,cap_sys_rawio,cap_sys_chroot,cap_sys_ptrace,cap_sys_pacct,cap_sys_admin,
+				cap_setuid,cap_setpcap,cap_linux_immutable,cap_net_bind_service,cap_net_broadcast,cap_net_admin,cap_net_raw,
-			cap_sys_boot,cap_sys_nice,cap_sys_resource,cap_sys_time,cap_sys_tty_config,cap_mknod,cap_lease,cap_audit_write,
+				cap_ipc_lock,cap_ipc_owner,cap_sys_module,cap_sys_rawio,cap_sys_chroot,cap_sys_ptrace,cap_sys_pacct,cap_sys_admin,
-			cap_audit_control,cap_setfcap,cap_mac_override,cap_mac_admin,cap_syslog,cap_wake_alarm,cap_block_suspend,cap_audit_read
+				cap_sys_boot,cap_sys_nice,cap_sys_resource,cap_sys_time,cap_sys_tty_config,cap_mknod,cap_lease,cap_audit_write,
-		*/
+				cap_audit_control,cap_setfcap,cap_mac_override,cap_mac_admin,cap_syslog,cap_wake_alarm,cap_block_suspend,cap_audit_read
-		command = `sh -c 'cat /proc/self/status | grep CapEff | grep "0000003fffffffff"'`
+			*/
+
+			// require that _at least_ minimum capabilities are granted
+			require.EqualValues(t, 0x3fffffffff, caps&0x3fffffffff)
+		}
 		mode = llb.SecurityModeInsecure
 		allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}
 	}
@@ -714,11 +722,31 @@ func testSecurityMode(t *testing.T, sb integration.Sandbox) {
 	def, err := st.Marshal(sb.Context())
 	require.NoError(t, err)
 
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
 	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
 		AllowedEntitlements: allowedEntitlements,
 	}, nil)
 
 	require.NoError(t, err)
+
+	contents, err := ioutil.ReadFile(filepath.Join(destDir, "out"))
+	require.NoError(t, err)
+
+	caps, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 16, 64)
+	require.NoError(t, err)
+
+	t.Logf("Caps: %x", caps)
+
+	assertCaps(caps)
 }
 
 func testSecurityModeSysfs(t *testing.T, sb integration.Sandbox) {

frontend/dockerfile/dockerfile_runsecurity_test.go

@@ -1,3 +1,4 @@
+//go:build dfrunsecurity
 // +build dfrunsecurity
 
 package dockerfile
@@ -93,7 +94,7 @@ func testRunSecurityInsecure(t *testing.T, sb integration.Sandbox) {
 
 	dockerfile := []byte(`
 FROM busybox
-RUN --security=insecure [ "$(cat /proc/self/status | grep CapBnd)" == "CapBnd:	0000003fffffffff" ]
+RUN --security=insecure [ "$(printf '%x' $(( $(cat /proc/self/status | grep CapBnd | cut -f 2 | sed s#^#0x#) & 0x3fffffffff)))" == "3fffffffff" ]
 RUN [ "$(cat /proc/self/status | grep CapBnd)" == "CapBnd:	00000000a80425fb" ]
 `)
 

util/entitlements/security/security_linux.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"sync"
 
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/oci"
+	"github.com/containerd/containerd/pkg/cap"
 	"github.com/containerd/containerd/pkg/userns"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
@@ -17,46 +19,11 @@ import (
 // WithInsecureSpec sets spec with All capability.
 func WithInsecureSpec() oci.SpecOpts {
 	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
-		addCaps := []string{
+		addCaps, err := getAllCaps()
-			"CAP_FSETID",
+		if err != nil {
-			"CAP_KILL",
+			return err
-			"CAP_FOWNER",
-			"CAP_MKNOD",
-			"CAP_CHOWN",
-			"CAP_DAC_OVERRIDE",
-			"CAP_NET_RAW",
-			"CAP_SETGID",
-			"CAP_SETUID",
-			"CAP_SETPCAP",
-			"CAP_SETFCAP",
-			"CAP_NET_BIND_SERVICE",
-			"CAP_SYS_CHROOT",
-			"CAP_AUDIT_WRITE",
-			"CAP_MAC_ADMIN",
-			"CAP_MAC_OVERRIDE",
-			"CAP_DAC_READ_SEARCH",
-			"CAP_SYS_PTRACE",
-			"CAP_SYS_MODULE",
-			"CAP_SYSLOG",
-			"CAP_SYS_RAWIO",
-			"CAP_SYS_ADMIN",
-			"CAP_LINUX_IMMUTABLE",
-			"CAP_SYS_BOOT",
-			"CAP_SYS_NICE",
-			"CAP_SYS_PACCT",
-			"CAP_SYS_TTY_CONFIG",
-			"CAP_SYS_TIME",
-			"CAP_WAKE_ALARM",
-			"CAP_AUDIT_READ",
-			"CAP_AUDIT_CONTROL",
-			"CAP_SYS_RESOURCE",
-			"CAP_BLOCK_SUSPEND",
-			"CAP_IPC_LOCK",
-			"CAP_IPC_OWNER",
-			"CAP_LEASE",
-			"CAP_NET_ADMIN",
-			"CAP_NET_BROADCAST",
 		}
+
 		s.Process.Capabilities.Bounding = append(s.Process.Capabilities.Bounding, addCaps...)
 		s.Process.Capabilities.Ambient = append(s.Process.Capabilities.Ambient, addCaps...)
 		s.Process.Capabilities.Effective = append(s.Process.Capabilities.Effective, addCaps...)
@@ -160,3 +127,76 @@ func getFreeLoopID() (int, error) {
 	}
 	return 0, errors.Errorf("error getting free loop device: %v", uerr)
 }
+
+var (
+	currentCaps     []string
+	currentCapsErr  error
+	currentCapsOnce sync.Once
+)
+
+func getCurrentCaps() ([]string, error) {
+	currentCapsOnce.Do(func() {
+		currentCaps, currentCapsErr = cap.Current()
+	})
+
+	return currentCaps, currentCapsErr
+}
+
+func getAllCaps() ([]string, error) {
+	availableCaps, err := getCurrentCaps()
+	if err != nil {
+		return nil, fmt.Errorf("error getting current capabilities: %s", err)
+	}
+
+	// see if any of the base linux35Caps are not available to be granted
+	// they are either not supported by the kernel or dropped at the process level
+	for _, cap := range availableCaps {
+		if _, exists := linux35Caps[cap]; !exists {
+			logrus.Warnf("capability %s could not be granted for insecure mode", cap)
+		}
+	}
+
+	return availableCaps, nil
+}
+
+// linux35Caps provides a list of capabilities available on Linux 3.5 kernel
+var linux35Caps = map[string]struct{}{
+	"CAP_FSETID":           {},
+	"CAP_KILL":             {},
+	"CAP_FOWNER":           {},
+	"CAP_MKNOD":            {},
+	"CAP_CHOWN":            {},
+	"CAP_DAC_OVERRIDE":     {},
+	"CAP_NET_RAW":          {},
+	"CAP_SETGID":           {},
+	"CAP_SETUID":           {},
+	"CAP_SETPCAP":          {},
+	"CAP_SETFCAP":          {},
+	"CAP_NET_BIND_SERVICE": {},
+	"CAP_SYS_CHROOT":       {},
+	"CAP_AUDIT_WRITE":      {},
+	"CAP_MAC_ADMIN":        {},
+	"CAP_MAC_OVERRIDE":     {},
+	"CAP_DAC_READ_SEARCH":  {},
+	"CAP_SYS_PTRACE":       {},
+	"CAP_SYS_MODULE":       {},
+	"CAP_SYSLOG":           {},
+	"CAP_SYS_RAWIO":        {},
+	"CAP_SYS_ADMIN":        {},
+	"CAP_LINUX_IMMUTABLE":  {},
+	"CAP_SYS_BOOT":         {},
+	"CAP_SYS_NICE":         {},
+	"CAP_SYS_PACCT":        {},
+	"CAP_SYS_TTY_CONFIG":   {},
+	"CAP_SYS_TIME":         {},
+	"CAP_WAKE_ALARM":       {},
+	"CAP_AUDIT_READ":       {},
+	"CAP_AUDIT_CONTROL":    {},
+	"CAP_SYS_RESOURCE":     {},
+	"CAP_BLOCK_SUSPEND":    {},
+	"CAP_IPC_LOCK":         {},
+	"CAP_IPC_OWNER":        {},
+	"CAP_LEASE":            {},
+	"CAP_NET_ADMIN":        {},
+	"CAP_NET_BROADCAST":    {},
+}