diff --git a/solver/edge.go b/solver/edge.go
index 97cbe0c1..beee0a8d 100644
--- a/solver/edge.go
+++ b/solver/edge.go
@@ -29,7 +29,7 @@ func newEdge(ed Edge, op activeOp, index *edgeIndex) *edge {
 		edge:         ed,
 		op:           op,
 		depRequests:  map[pipe.Receiver]*dep{},
-		keyMap:       map[string]*CacheKey{},
+		keyMap:       map[string]struct{}{},
 		cacheRecords: map[string]*CacheRecord{},
 		index:        index,
 	}
@@ -51,7 +51,7 @@ type edge struct {
 	execReq         pipe.Receiver
 	err             error
 	cacheRecords    map[string]*CacheRecord
-	keyMap          map[string]*CacheKey
+	keyMap          map[string]struct{}
 
 	noCacheMatchPossible      bool
 	allDepsCompletedCacheFast bool
@@ -527,6 +527,10 @@ func (e *edge) recalcCurrentState() {
 		}
 	}
 
+	for key := range newKeys {
+		e.keyMap[key] = struct{}{}
+	}
+
 	for _, r := range newKeys {
 		// TODO: add all deps automatically
 		mergedKey := r.clone()
@@ -615,7 +619,33 @@ func (e *edge) recalcCurrentState() {
 		e.allDepsCompleted = e.cacheMapDone && allDepsCompleted
 
 		if e.allDepsStateCacheSlow && len(e.cacheRecords) > 0 && e.state == edgeStatusCacheFast {
-			e.state = edgeStatusCacheSlow
+			openKeys := map[string]struct{}{}
+			for _, dep := range e.deps {
+				isSlowIncomplete := e.slowCacheFunc(dep) != nil && (dep.state == edgeStatusCacheSlow || (dep.state == edgeStatusComplete && !dep.slowCacheComplete))
+				if !isSlowIncomplete {
+					openDepKeys := map[string]struct{}{}
+					for key := range dep.keyMap {
+						if _, ok := e.keyMap[key]; !ok {
+							openDepKeys[key] = struct{}{}
+						}
+					}
+					if len(openKeys) != 0 {
+						for k := range openKeys {
+							if _, ok := openDepKeys[k]; !ok {
+								delete(openKeys, k)
+							}
+						}
+					} else {
+						openKeys = openDepKeys
+					}
+					if len(openKeys) == 0 {
+						e.state = edgeStatusCacheSlow
+						if debugScheduler {
+							logrus.Debugf("upgrade to cache-slow because no open keys")
+						}
+					}
+				}
+			}
 		}
 	}
 }