diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go
index 32e9e353..9835f308 100644
--- a/cmd/buildkitd/main.go
+++ b/cmd/buildkitd/main.go
@@ -31,6 +31,7 @@ import (
 	"github.com/moby/buildkit/version"
 	"github.com/moby/buildkit/worker"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/opencontainers/runc/libcontainer/system"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -74,14 +75,19 @@ func main() {
 	defaultRoot := appdefaults.Root
 	defaultAddress := appdefaults.Address
 	rootlessUsage := "set all the default options to be compatible with rootless containers"
-	if runningAsUnprivilegedUser() {
+	if system.RunningInUserNS() {
 		app.Flags = append(app.Flags, cli.BoolTFlag{
 			Name:  "rootless",
 			Usage: rootlessUsage + " (default: true)",
 		})
-		defaultRoot = appdefaults.UserRoot()
-		defaultAddress = appdefaults.UserAddress()
-		appdefaults.EnsureUserAddressDir()
+		// if buildkitd is being executed as the mapped-root (not only EUID==0 but also $USER==root)
+		// in a user namespace, we need to enable the rootless mode but
+		// we don't want to honor $HOME for setting up default paths.
+		if u := os.Getenv("USER"); u != "" && u != "root" {
+			defaultRoot = appdefaults.UserRoot()
+			defaultAddress = appdefaults.UserAddress()
+			appdefaults.EnsureUserAddressDir()
+		}
 	} else {
 		app.Flags = append(app.Flags, cli.BoolFlag{
 			Name:  "rootless",
diff --git a/cmd/buildkitd/main_linux.go b/cmd/buildkitd/main_linux.go
deleted file mode 100644
index 27823fd6..00000000
--- a/cmd/buildkitd/main_linux.go
+++ /dev/null
@@ -1,9 +0,0 @@
-// +build linux
-
-package main
-
-import "github.com/opencontainers/runc/libcontainer/system"
-
-func runningAsUnprivilegedUser() bool {
-	return system.GetParentNSeuid() != 0 || system.RunningInUserNS()
-}
diff --git a/cmd/buildkitd/main_nolinux.go b/cmd/buildkitd/main_nolinux.go
deleted file mode 100644
index 883baa16..00000000
--- a/cmd/buildkitd/main_nolinux.go
+++ /dev/null
@@ -1,7 +0,0 @@
-// +build !linux
-
-package main
-
-func runningAsUnprivilegedUser() bool {
-	return false
-}
diff --git a/cmd/buildkitd/main_oci_worker.go b/cmd/buildkitd/main_oci_worker.go
index 3a1ca86f..976aae2f 100644
--- a/cmd/buildkitd/main_oci_worker.go
+++ b/cmd/buildkitd/main_oci_worker.go
@@ -13,6 +13,7 @@ import (
 	"github.com/moby/buildkit/worker/base"
 	"github.com/moby/buildkit/worker/runc"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/opencontainers/runc/libcontainer/system"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -41,7 +42,7 @@ func init() {
 	}
 	n := "oci-worker-rootless"
 	u := "enable rootless mode"
-	if runningAsUnprivilegedUser() {
+	if system.RunningInUserNS() {
 		flags = append(flags, cli.BoolTFlag{
 			Name:  n,
 			Usage: u,