From a5d1cfc1e4ff4488c8636abaed9c66e65f3c4831 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Tue, 5 Oct 2021 18:51:23 +0300 Subject: [PATCH] fix: provide only available capabilities to insecure environment The problem this change is trying to fix are the environments where some capabilities are already dropped, so they can't be granted to the job with `--security=insecure`. I know that probably fixed set of capabilities was implemented to provide a stable build environment, but at the same time this breaks environments with reduced capabilities. Signed-off-by: Andrey Smirnov --- client/build_test.go | 45 ++++--- client/client_test.go | 60 ++++++--- .../dockerfile/dockerfile_runsecurity_test.go | 3 +- util/entitlements/security/security_linux.go | 118 ++++++++++++------ 4 files changed, 154 insertions(+), 72 deletions(-) diff --git a/client/build_test.go b/client/build_test.go index d9a0f0b5..72f7ef5d 100644 --- a/client/build_test.go +++ b/client/build_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -1518,29 +1519,36 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox product := "buildkit_test" - var command []string + command := []string{"sh", "-c", `cat /proc/self/status | grep CapEff | cut -f 2`} mode := llb.SecurityModeSandbox var allowedEntitlements []entitlements.Entitlement + var assertCaps func(caps uint64) secMode := sb.Value("secmode") if secMode == securitySandbox { - /* - $ capsh --decode=00000000a80425fb - 0x00000000a80425fb=cap_chown,cap_dac_override,cap_fowner,cap_fsetid,cap_kill,cap_setgid,cap_setuid,cap_setpcap, - cap_net_bind_service,cap_net_raw,cap_sys_chroot,cap_mknod,cap_audit_write,cap_setfcap - */ - command = []string{"sh", "-c", `cat /proc/self/status | grep CapEff | grep "00000000a80425fb"`} + assertCaps = func(caps uint64) { + /* + $ capsh --decode=00000000a80425fb + 0x00000000a80425fb=cap_chown,cap_dac_override,cap_fowner,cap_fsetid,cap_kill,cap_setgid,cap_setuid,cap_setpcap, + cap_net_bind_service,cap_net_raw,cap_sys_chroot,cap_mknod,cap_audit_write,cap_setfcap + */ + require.EqualValues(t, 0xa80425fb, caps) + } allowedEntitlements = []entitlements.Entitlement{} } else { skipDockerd(t, sb) - /* - $ capsh --decode=0000003fffffffff - 0x0000003fffffffff=cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_kill,cap_setgid, - cap_setuid,cap_setpcap,cap_linux_immutable,cap_net_bind_service,cap_net_broadcast,cap_net_admin,cap_net_raw, - cap_ipc_lock,cap_ipc_owner,cap_sys_module,cap_sys_rawio,cap_sys_chroot,cap_sys_ptrace,cap_sys_pacct,cap_sys_admin, - cap_sys_boot,cap_sys_nice,cap_sys_resource,cap_sys_time,cap_sys_tty_config,cap_mknod,cap_lease,cap_audit_write, - cap_audit_control,cap_setfcap,cap_mac_override,cap_mac_admin,cap_syslog,cap_wake_alarm,cap_block_suspend,cap_audit_read - */ - command = []string{"sh", "-c", `cat /proc/self/status | grep CapEff | grep "0000003fffffffff"`} + assertCaps = func(caps uint64) { + /* + $ capsh --decode=0000003fffffffff + 0x0000003fffffffff=cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_kill,cap_setgid, + cap_setuid,cap_setpcap,cap_linux_immutable,cap_net_bind_service,cap_net_broadcast,cap_net_admin,cap_net_raw, + cap_ipc_lock,cap_ipc_owner,cap_sys_module,cap_sys_rawio,cap_sys_chroot,cap_sys_ptrace,cap_sys_pacct,cap_sys_admin, + cap_sys_boot,cap_sys_nice,cap_sys_resource,cap_sys_time,cap_sys_tty_config,cap_mknod,cap_lease,cap_audit_write, + cap_audit_control,cap_setfcap,cap_mac_override,cap_mac_admin,cap_syslog,cap_wake_alarm,cap_block_suspend,cap_audit_read + */ + + // require that _at least_ minimum capabilities are granted + require.EqualValues(t, 0x3fffffffff, caps&0x3fffffffff) + } mode = llb.SecurityModeInsecure allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure} } @@ -1594,6 +1602,11 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox require.NoError(t, err) + capsValue, err := strconv.ParseUint(strings.TrimSpace(stdout.String()), 16, 64) + require.NoError(t, err) + + assertCaps(capsValue) + return &client.Result{}, nil } diff --git a/client/client_test.go b/client/client_test.go index e2e39378..62e25161 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -17,6 +17,7 @@ import ( "os" "path/filepath" "runtime" + "strconv" "strings" "syscall" "testing" @@ -676,29 +677,36 @@ func testPushByDigest(t *testing.T, sb integration.Sandbox) { } func testSecurityMode(t *testing.T, sb integration.Sandbox) { - var command string + command := `sh -c 'cat /proc/self/status | grep CapEff | cut -f 2 > /out'` mode := llb.SecurityModeSandbox var allowedEntitlements []entitlements.Entitlement + var assertCaps func(caps uint64) secMode := sb.Value("secmode") if secMode == securitySandbox { - /* - $ capsh --decode=00000000a80425fb - 0x00000000a80425fb=cap_chown,cap_dac_override,cap_fowner,cap_fsetid,cap_kill,cap_setgid,cap_setuid,cap_setpcap, - cap_net_bind_service,cap_net_raw,cap_sys_chroot,cap_mknod,cap_audit_write,cap_setfcap - */ - command = `sh -c 'cat /proc/self/status | grep CapEff | grep "00000000a80425fb"'` + assertCaps = func(caps uint64) { + /* + $ capsh --decode=00000000a80425fb + 0x00000000a80425fb=cap_chown,cap_dac_override,cap_fowner,cap_fsetid,cap_kill,cap_setgid,cap_setuid,cap_setpcap, + cap_net_bind_service,cap_net_raw,cap_sys_chroot,cap_mknod,cap_audit_write,cap_setfcap + */ + require.EqualValues(t, 0xa80425fb, caps) + } allowedEntitlements = []entitlements.Entitlement{} } else { skipDockerd(t, sb) - /* - $ capsh --decode=0000003fffffffff - 0x0000003fffffffff=cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_kill,cap_setgid, - cap_setuid,cap_setpcap,cap_linux_immutable,cap_net_bind_service,cap_net_broadcast,cap_net_admin,cap_net_raw, - cap_ipc_lock,cap_ipc_owner,cap_sys_module,cap_sys_rawio,cap_sys_chroot,cap_sys_ptrace,cap_sys_pacct,cap_sys_admin, - cap_sys_boot,cap_sys_nice,cap_sys_resource,cap_sys_time,cap_sys_tty_config,cap_mknod,cap_lease,cap_audit_write, - cap_audit_control,cap_setfcap,cap_mac_override,cap_mac_admin,cap_syslog,cap_wake_alarm,cap_block_suspend,cap_audit_read - */ - command = `sh -c 'cat /proc/self/status | grep CapEff | grep "0000003fffffffff"'` + assertCaps = func(caps uint64) { + /* + $ capsh --decode=0000003fffffffff + 0x0000003fffffffff=cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_kill,cap_setgid, + cap_setuid,cap_setpcap,cap_linux_immutable,cap_net_bind_service,cap_net_broadcast,cap_net_admin,cap_net_raw, + cap_ipc_lock,cap_ipc_owner,cap_sys_module,cap_sys_rawio,cap_sys_chroot,cap_sys_ptrace,cap_sys_pacct,cap_sys_admin, + cap_sys_boot,cap_sys_nice,cap_sys_resource,cap_sys_time,cap_sys_tty_config,cap_mknod,cap_lease,cap_audit_write, + cap_audit_control,cap_setfcap,cap_mac_override,cap_mac_admin,cap_syslog,cap_wake_alarm,cap_block_suspend,cap_audit_read + */ + + // require that _at least_ minimum capabilities are granted + require.EqualValues(t, 0x3fffffffff, caps&0x3fffffffff) + } mode = llb.SecurityModeInsecure allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure} } @@ -714,11 +722,31 @@ func testSecurityMode(t *testing.T, sb integration.Sandbox) { def, err := st.Marshal(sb.Context()) require.NoError(t, err) + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + }, + }, AllowedEntitlements: allowedEntitlements, }, nil) require.NoError(t, err) + + contents, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + + caps, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 16, 64) + require.NoError(t, err) + + t.Logf("Caps: %x", caps) + + assertCaps(caps) } func testSecurityModeSysfs(t *testing.T, sb integration.Sandbox) { diff --git a/frontend/dockerfile/dockerfile_runsecurity_test.go b/frontend/dockerfile/dockerfile_runsecurity_test.go index 2cf10687..358db673 100644 --- a/frontend/dockerfile/dockerfile_runsecurity_test.go +++ b/frontend/dockerfile/dockerfile_runsecurity_test.go @@ -1,3 +1,4 @@ +//go:build dfrunsecurity // +build dfrunsecurity package dockerfile @@ -93,7 +94,7 @@ func testRunSecurityInsecure(t *testing.T, sb integration.Sandbox) { dockerfile := []byte(` FROM busybox -RUN --security=insecure [ "$(cat /proc/self/status | grep CapBnd)" == "CapBnd: 0000003fffffffff" ] +RUN --security=insecure [ "$(printf '%x' $(( $(cat /proc/self/status | grep CapBnd | cut -f 2 | sed s#^#0x#) & 0x3fffffffff)))" == "3fffffffff" ] RUN [ "$(cat /proc/self/status | grep CapBnd)" == "CapBnd: 00000000a80425fb" ] `) diff --git a/util/entitlements/security/security_linux.go b/util/entitlements/security/security_linux.go index 0f285fb2..ccff21f5 100644 --- a/util/entitlements/security/security_linux.go +++ b/util/entitlements/security/security_linux.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "os" + "sync" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/pkg/cap" "github.com/containerd/containerd/pkg/userns" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -17,46 +19,11 @@ import ( // WithInsecureSpec sets spec with All capability. func WithInsecureSpec() oci.SpecOpts { return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { - addCaps := []string{ - "CAP_FSETID", - "CAP_KILL", - "CAP_FOWNER", - "CAP_MKNOD", - "CAP_CHOWN", - "CAP_DAC_OVERRIDE", - "CAP_NET_RAW", - "CAP_SETGID", - "CAP_SETUID", - "CAP_SETPCAP", - "CAP_SETFCAP", - "CAP_NET_BIND_SERVICE", - "CAP_SYS_CHROOT", - "CAP_AUDIT_WRITE", - "CAP_MAC_ADMIN", - "CAP_MAC_OVERRIDE", - "CAP_DAC_READ_SEARCH", - "CAP_SYS_PTRACE", - "CAP_SYS_MODULE", - "CAP_SYSLOG", - "CAP_SYS_RAWIO", - "CAP_SYS_ADMIN", - "CAP_LINUX_IMMUTABLE", - "CAP_SYS_BOOT", - "CAP_SYS_NICE", - "CAP_SYS_PACCT", - "CAP_SYS_TTY_CONFIG", - "CAP_SYS_TIME", - "CAP_WAKE_ALARM", - "CAP_AUDIT_READ", - "CAP_AUDIT_CONTROL", - "CAP_SYS_RESOURCE", - "CAP_BLOCK_SUSPEND", - "CAP_IPC_LOCK", - "CAP_IPC_OWNER", - "CAP_LEASE", - "CAP_NET_ADMIN", - "CAP_NET_BROADCAST", + addCaps, err := getAllCaps() + if err != nil { + return err } + s.Process.Capabilities.Bounding = append(s.Process.Capabilities.Bounding, addCaps...) s.Process.Capabilities.Ambient = append(s.Process.Capabilities.Ambient, addCaps...) s.Process.Capabilities.Effective = append(s.Process.Capabilities.Effective, addCaps...) @@ -160,3 +127,76 @@ func getFreeLoopID() (int, error) { } return 0, errors.Errorf("error getting free loop device: %v", uerr) } + +var ( + currentCaps []string + currentCapsErr error + currentCapsOnce sync.Once +) + +func getCurrentCaps() ([]string, error) { + currentCapsOnce.Do(func() { + currentCaps, currentCapsErr = cap.Current() + }) + + return currentCaps, currentCapsErr +} + +func getAllCaps() ([]string, error) { + availableCaps, err := getCurrentCaps() + if err != nil { + return nil, fmt.Errorf("error getting current capabilities: %s", err) + } + + // see if any of the base linux35Caps are not available to be granted + // they are either not supported by the kernel or dropped at the process level + for _, cap := range availableCaps { + if _, exists := linux35Caps[cap]; !exists { + logrus.Warnf("capability %s could not be granted for insecure mode", cap) + } + } + + return availableCaps, nil +} + +// linux35Caps provides a list of capabilities available on Linux 3.5 kernel +var linux35Caps = map[string]struct{}{ + "CAP_FSETID": {}, + "CAP_KILL": {}, + "CAP_FOWNER": {}, + "CAP_MKNOD": {}, + "CAP_CHOWN": {}, + "CAP_DAC_OVERRIDE": {}, + "CAP_NET_RAW": {}, + "CAP_SETGID": {}, + "CAP_SETUID": {}, + "CAP_SETPCAP": {}, + "CAP_SETFCAP": {}, + "CAP_NET_BIND_SERVICE": {}, + "CAP_SYS_CHROOT": {}, + "CAP_AUDIT_WRITE": {}, + "CAP_MAC_ADMIN": {}, + "CAP_MAC_OVERRIDE": {}, + "CAP_DAC_READ_SEARCH": {}, + "CAP_SYS_PTRACE": {}, + "CAP_SYS_MODULE": {}, + "CAP_SYSLOG": {}, + "CAP_SYS_RAWIO": {}, + "CAP_SYS_ADMIN": {}, + "CAP_LINUX_IMMUTABLE": {}, + "CAP_SYS_BOOT": {}, + "CAP_SYS_NICE": {}, + "CAP_SYS_PACCT": {}, + "CAP_SYS_TTY_CONFIG": {}, + "CAP_SYS_TIME": {}, + "CAP_WAKE_ALARM": {}, + "CAP_AUDIT_READ": {}, + "CAP_AUDIT_CONTROL": {}, + "CAP_SYS_RESOURCE": {}, + "CAP_BLOCK_SUSPEND": {}, + "CAP_IPC_LOCK": {}, + "CAP_IPC_OWNER": {}, + "CAP_LEASE": {}, + "CAP_NET_ADMIN": {}, + "CAP_NET_BROADCAST": {}, +}