diff --git a/cache/manager.go b/cache/manager.go
index 05d8ca12..5bb1b818 100644
--- a/cache/manager.go
+++ b/cache/manager.go
@@ -236,12 +236,10 @@ func (cm *cacheManager) DiskUsage(ctx context.Context) ([]*client.UsageInfo, err
 			break
 		}
 		for id := range rescan {
-			if parentID := m[id].parent; parentID != "" {
-				p := m[parentID]
-				p.refs--
-				if p.refs == 0 && p.parent != "" {
-					rescan[parentID] = struct{}{}
-				}
+			v := m[id]
+			if v.refs == 0 && v.parent != "" {
+				m[v.parent].refs--
+				rescan[v.parent] = struct{}{}
 			}
 			delete(rescan, id)
 		}
@@ -274,7 +272,7 @@ func (cm *cacheManager) DiskUsage(ctx context.Context) ([]*client.UsageInfo, err
 						return err
 					}
 					d.Size = s
-					return ref.Release(ctx)
+					return ref.Release(context.TODO())
 				})
 			}(d)
 		}
diff --git a/cache/refs.go b/cache/refs.go
index 48294e27..41af54fc 100644
--- a/cache/refs.go
+++ b/cache/refs.go
@@ -129,11 +129,6 @@ func (sr *immutableRef) Release(ctx context.Context) error {
 }
 
 func (sr *immutableRef) release(ctx context.Context) error {
-	if sr.parent != nil {
-		if err := sr.parent.(*immutableRef).release(ctx); err != nil {
-			return err
-		}
-	}
 	if sr.viewMount != nil {
 		if err := sr.cm.Snapshotter.Remove(ctx, sr.view); err != nil {
 			return err
diff --git a/solver/solver.go b/solver/solver.go
index b62d4754..752422ec 100644
--- a/solver/solver.go
+++ b/solver/solver.go
@@ -117,6 +117,12 @@ func (s *Solver) getRefs(ctx context.Context, j *job, g *opVertex) (retRef []cac
 		}
 	}
 
+	defer func() {
+		for _, r := range inputs {
+			go r.Release(context.TODO())
+		}
+	}()
+
 	// release anything else
 	for _, r := range refs {
 		for _, r := range r {