Add a DOKS check for custom node labels and taints

In DOKS labels and taints applied to nodes will be lost when the cluster
is upgraded or a node is otherwise replaced. This can cause problems for
workloads if labels or taints are used for scheduling.

Add a warning if any node in a cluster has custom labels or taints.

checks.md

@@ -510,3 +510,24 @@ spec:
       - name: nginx
         image: nginx:1.7.9
 ```
+
+###### Node Labels and Taints
+
+Name:         `node-labels-and-taints`
+
+Group:        `doks`
+
+Description: When a DOKS cluster is upgraded, all worker nodes are replaced, and
+replacement nodes do not retain any custom labels or taints that were set on the
+pre-upgrade nodes. This check reports any nodes that have had labels or taints
+set by the user, which would be lost on upgrade or other node replacement.
+
+How to fix:
+
+```bash
+kubectl label node <node-name> <label-key>-
+kubectl taint node <node-name> <taint-key>-
+```
+
+Note the trailing `-` on the label or taint key; this causes `kubectl` to delete
+the label or taint.

checks/diagnostic.go

@@ -75,4 +75,6 @@ const (
 	ValidatingWebhookConfiguration Kind = "validating webhook configuration"
 	// MutatingWebhookConfiguration identifies Kubernetes objects of kind `mutating webhook configuration`
 	MutatingWebhookConfiguration Kind = "mutating webhook configuration"
+	// Node identifies a Kubernetes node object.
+	Node Kind = "node"
 )

checks/doks/node_labels_taints.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2019 DigitalOcean
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package doks
+
+import (
+	"strings"
+
+	"github.com/digitalocean/clusterlint/checks"
+	"github.com/digitalocean/clusterlint/kube"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func init() {
+	checks.Register(&nodeLabelsTaintsCheck{})
+}
+
+type nodeLabelsTaintsCheck struct{}
+
+// Name returns the name of the check.
+func (*nodeLabelsTaintsCheck) Name() string {
+	return "node-labels-and-taints"
+}
+
+// Groups returns groups for this check.
+func (*nodeLabelsTaintsCheck) Groups() []string {
+	return []string{"doks"}
+}
+
+// Description returns a description of the check.
+func (*nodeLabelsTaintsCheck) Description() string {
+	return "Checks that nodes do not have custom labels or taints configured."
+}
+
+// Run runs the check.
+func (c *nodeLabelsTaintsCheck) Run(objects *kube.Objects) ([]checks.Diagnostic, error) {
+	var diagnostics []checks.Diagnostic
+
+	for _, node := range objects.Nodes.Items {
+		for labelKey := range node.Labels {
+			if !isKubernetesLabel(labelKey) && !isDOKSLabel(labelKey) {
+				d := checks.Diagnostic{
+					Check:    c.Name(),
+					Severity: checks.Warning,
+					Message:  "Custom node labels will be lost if node is replaced or upgraded.",
+					Kind:     checks.Node,
+					Object:   &node.ObjectMeta,
+				}
+				diagnostics = append(diagnostics, d)
+				// Produce only one label diagnostic per node.
+				break
+			}
+		}
+		for _, taint := range node.Spec.Taints {
+			if !isDOKSTaint(taint) {
+				d := checks.Diagnostic{
+					Check:    c.Name(),
+					Severity: checks.Warning,
+					Message:  "Custom node taints will be lost if node is replaced or upgraded.",
+					Kind:     checks.Node,
+					Object:   &node.ObjectMeta,
+				}
+				diagnostics = append(diagnostics, d)
+				// Produce only one taint diagnostic per node.
+				break
+			}
+		}
+	}
+
+	return diagnostics, nil
+}
+
+func isKubernetesLabel(key string) bool {
+	// Built-in Kubernetes labels are in various subdomains of
+	// kubernetes.io. Assume all such labels are built in.
+	return strings.Contains(key, corev1.ResourceDefaultNamespacePrefix)
+}
+
+func isDOKSLabel(key string) bool {
+	// DOKS labels use the doks.digitalocean.com namespace. Assume all such
+	// labels are set by DOKS.
+	if strings.HasPrefix(key, "doks.digitalocean.com/") {
+		return true
+	}
+
+	// CCM also sets a region label.
+	if key == "region" {
+		return true
+	}
+
+	return false
+}
+
+func isDOKSTaint(taint corev1.Taint) bool {
+	// Currently DOKS never sets taints.
+	return false
+}

checks/doks/node_labels_taints_test.go

@@ -0,0 +1,165 @@
+/*
+Copyright 2019 DigitalOcean
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package doks
+
+import (
+	"testing"
+
+	"github.com/digitalocean/clusterlint/checks"
+	"github.com/digitalocean/clusterlint/kube"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestNodeLabels(t *testing.T) {
+	tests := []struct {
+		name                string
+		nodeLabels          map[string]string
+		expectedDiagnostics []checks.Diagnostic
+	}{
+		{
+			name:                "no labels",
+			nodeLabels:          nil,
+			expectedDiagnostics: nil,
+		},
+		{
+			name: "only doks labels",
+			nodeLabels: map[string]string{
+				"doks.digitalocean.com/foo": "bar",
+				"doks.digitalocean.com/baz": "xyzzy",
+			},
+			expectedDiagnostics: nil,
+		},
+		{
+			name: "only built-in labels",
+			nodeLabels: map[string]string{
+				"kubernetes.io/hostname":                   "a-hostname",
+				"beta.kubernetes.io/os":                    "linux",
+				"failure-domain.beta.kubernetes.io/region": "tor1",
+			},
+			expectedDiagnostics: nil,
+		},
+		{
+			name: "only region label",
+			nodeLabels: map[string]string{
+				"region": "tor1",
+			},
+			expectedDiagnostics: nil,
+		},
+		{
+			name: "custom labels",
+			nodeLabels: map[string]string{
+				"doks.digitalocean.com/foo":                "bar",
+				"doks.digitalocean.com/baz":                "xyzzy",
+				"kubernetes.io/hostname":                   "a-hostname",
+				"example.com/custom-label":                 "bad",
+				"example.com/another-label":                "real-bad",
+				"beta.kubernetes.io/os":                    "linux",
+				"failure-domain.beta.kubernetes.io/region": "tor1",
+				"region": "tor1",
+			},
+			expectedDiagnostics: []checks.Diagnostic{{
+				Check:    "node-labels-and-taints",
+				Severity: checks.Warning,
+				Message:  "Custom node labels will be lost if node is replaced or upgraded.",
+				Kind:     checks.Node,
+				Object: &metav1.ObjectMeta{
+					Labels: map[string]string{
+						"doks.digitalocean.com/foo":                "bar",
+						"doks.digitalocean.com/baz":                "xyzzy",
+						"kubernetes.io/hostname":                   "a-hostname",
+						"example.com/custom-label":                 "bad",
+						"example.com/another-label":                "real-bad",
+						"beta.kubernetes.io/os":                    "linux",
+						"failure-domain.beta.kubernetes.io/region": "tor1",
+						"region": "tor1",
+					},
+				},
+			}},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			objects := &kube.Objects{
+				Nodes: &corev1.NodeList{
+					Items: []corev1.Node{{
+						ObjectMeta: metav1.ObjectMeta{
+							Labels: test.nodeLabels,
+						},
+					}},
+				},
+			}
+
+			check := &nodeLabelsTaintsCheck{}
+
+			ds, err := check.Run(objects)
+			assert.NoError(t, err)
+			assert.ElementsMatch(t, test.expectedDiagnostics, ds)
+		})
+	}
+}
+
+func TestNodeTaints(t *testing.T) {
+	tests := []struct {
+		name                string
+		taints              []corev1.Taint
+		expectedDiagnostics []checks.Diagnostic
+	}{
+		{
+			name:                "no taints",
+			taints:              nil,
+			expectedDiagnostics: nil,
+		},
+		{
+			name: "custom taints",
+			taints: []corev1.Taint{{
+				Key:    "example.com/my-taint",
+				Value:  "foo",
+				Effect: corev1.TaintEffectNoSchedule,
+			}},
+			expectedDiagnostics: []checks.Diagnostic{{
+				Check:    "node-labels-and-taints",
+				Severity: checks.Warning,
+				Message:  "Custom node taints will be lost if node is replaced or upgraded.",
+				Kind:     checks.Node,
+				Object:   &metav1.ObjectMeta{},
+			}},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			objects := &kube.Objects{
+				Nodes: &corev1.NodeList{
+					Items: []corev1.Node{{
+						Spec: corev1.NodeSpec{
+							Taints: test.taints,
+						},
+					}},
+				},
+			}
+
+			check := &nodeLabelsTaintsCheck{}
+
+			ds, err := check.Run(objects)
+			assert.NoError(t, err)
+			assert.ElementsMatch(t, test.expectedDiagnostics, ds)
+		})
+	}
+}