fix: Revert "fix(controller): rollout stuck in Progressing. fixes #3988" (#4201)

Revert "fix(controller):  rollout stuck in `Progressing`. fixes #3988 (#4072)"

This reverts commit f4289b7.

Author: zachaller

rollout/bluegreen_test.go

@@ -1406,7 +1406,6 @@ func TestBlueGreenReadyToScaleDownOldReplica(t *testing.T) {
 	f.objects = append(f.objects, r2)
 	f.serviceLister = append(f.serviceLister, s)
 
-	f.expectPatchReplicaSetAction(rs2)
 	updatedRSIndex := f.expectUpdateReplicaSetAction(rs2)
 	patchIndex := f.expectPatchRolloutAction(r2)
 	f.run(getKey(r2, t))

rollout/replicaset.go

@@ -107,16 +107,6 @@ func (c *rolloutContext) removeScaleDownDeadlines() error {
 			toRemove = append(toRemove, c.stableRS)
 		}
 	}
-	for _, rs := range c.otherRSs {
-		remainScaleDownDeadlines, err := replicasetutil.GetTimeRemainingBeforeScaleDownDeadline(rs)
-		if err != nil {
-			c.log.Warnf("%v", err)
-			continue
-		}
-		if replicasetutil.HasScaleDownDeadline(rs) && remainScaleDownDeadlines == nil {
-			toRemove = append(toRemove, rs)
-		}
-	}
 	for _, rs := range toRemove {
 		err := c.removeScaleDownDelay(rs)
 		if err != nil {