Skip to content

Commit 84d33bd

Browse files
kddejongkddejong
authored
fix: delete and recreate stack in ROLLBACK_COMPLETE state (#191)
* fix: delete and recreate stack in ROLLBACK_COMPLETE state (#107) When a stack is in ROLLBACK_COMPLETE state it cannot be updated. This change detects that state, deletes the stack, and recreates it via a new change set. Closes #107 * docs: add DeleteStack permission and ROLLBACK_COMPLETE recovery docs
1 parent 334ecac commit 84d33bd

File tree

4 files changed

+415
-4
lines changed

4 files changed

+415
-4
lines changed

‎README.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ This action requires the following minimum set of permissions:
243243
"cloudformation:CreateChangeSet",
244244
"cloudformation:DescribeChangeSet",
245245
"cloudformation:DeleteChangeSet",
246+
"cloudformation:DeleteStack",
246247
"cloudformation:ExecuteChangeSet",
247248
"cloudformation:DescribeEvents"
248249
],
@@ -281,12 +282,14 @@ The action makes the following AWS CloudFormation API calls depending on the ope
281282
- `DescribeEvents` - Retrieve detailed error information for validation failures
282283
- `DeleteChangeSet` - Clean up failed change sets (unless `no-delete-failed-changeset` is set)
283284

285+
**ROLLBACK_COMPLETE Recovery:**
286+
287+
- `DeleteStack` - Automatically delete stacks stuck in `ROLLBACK_COMPLETE` state before recreating
288+
284289
**Event Streaming (during stack operations):**
285290

286291
- `DescribeEvents` - Monitor real-time CloudFormation events during deployment
287292

288-
> The policy above prevents the stack from being deleted - add `cloudformation:DeleteStack` if deletion is required for your use case
289-
290293
## Example
291294

292295
You want to run your microservices with [Amazon Elastic Kubernetes Services](https://aws.amazon.com/eks/) and leverage the best-practices to run the cluster? Using this GitHub Action you can customize and deploy the [modular and scalable Amazon EKS architecture](https://aws.amazon.com/quickstart/architecture/amazon-eks/) provided in an AWS Quick Start to your AWS Account. The following workflow enables you to create and update a Kubernetes cluster using a manual workflow trigger.

‎__tests__/deploy.test.ts‎

Lines changed: 309 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import {
22
CloudFormationClient,
3+
CloudFormationServiceException,
34
DescribeStacksCommand,
45
DescribeChangeSetCommand,
56
DescribeEventsCommand,
67
CreateChangeSetCommand,
8+
DeleteStackCommand,
79
ExecuteChangeSetCommand,
810
StackStatus,
911
ChangeSetStatus,
@@ -13,6 +15,7 @@ import { mockClient } from 'aws-sdk-client-mock'
1315
import {
1416
waitUntilStackOperationComplete,
1517
updateStack,
18+
deployStack,
1619
executeExistingChangeSet
1720
} from '../src/deploy'
1821
import * as core from '@actions/core'
@@ -564,4 +567,310 @@ describe('Deploy error scenarios', () => {
564567
expect(result.stackId).toBe('test-stack-id')
565568
})
566569
})
570+
571+
describe('deployStack', () => {
572+
it('deletes and recreates stack in ROLLBACK_COMPLETE state', async () => {
573+
// First call: getStack returns ROLLBACK_COMPLETE
574+
// Second call: waitUntilStackDeleteComplete sees DELETE_COMPLETE
575+
// Third call: waitUntilStackOperationComplete after execute
576+
mockCfnClient
577+
.on(DescribeStacksCommand)
578+
.resolvesOnce({
579+
Stacks: [
580+
{
581+
StackName: 'TestStack',
582+
StackId: 'test-stack-id',
583+
StackStatus: StackStatus.ROLLBACK_COMPLETE,
584+
CreationTime: new Date()
585+
}
586+
]
587+
})
588+
.resolvesOnce({
589+
Stacks: [
590+
{
591+
StackName: 'TestStack',
592+
StackId: 'test-stack-id',
593+
StackStatus: StackStatus.DELETE_COMPLETE,
594+
CreationTime: new Date()
595+
}
596+
]
597+
})
598+
.resolves({
599+
Stacks: [
600+
{
601+
StackName: 'TestStack',
602+
StackId: 'new-stack-id',
603+
StackStatus: StackStatus.CREATE_COMPLETE,
604+
CreationTime: new Date()
605+
}
606+
]
607+
})
608+
609+
mockCfnClient.on(DeleteStackCommand).resolves({})
610+
mockCfnClient
611+
.on(CreateChangeSetCommand)
612+
.resolves({ Id: 'test-cs-id', StackId: 'new-stack-id' })
613+
mockCfnClient.on(DescribeChangeSetCommand).resolves({
614+
Status: ChangeSetStatus.CREATE_COMPLETE,
615+
Changes: [{ Type: 'Resource' }]
616+
})
617+
mockCfnClient.on(ExecuteChangeSetCommand).resolves({})
618+
619+
const result = await deployStack(
620+
cfn,
621+
{
622+
StackName: 'TestStack',
623+
TemplateBody: '{}',
624+
Capabilities: []
625+
},
626+
'test-cs',
627+
false,
628+
false,
629+
false,
630+
60
631+
)
632+
633+
expect(mockCfnClient.commandCalls(DeleteStackCommand)).toHaveLength(1)
634+
expect(core.info).toHaveBeenCalledWith(
635+
expect.stringContaining('ROLLBACK_COMPLETE')
636+
)
637+
expect(result.stackId).toBe('new-stack-id')
638+
})
639+
640+
it('does not delete stack when not in ROLLBACK_COMPLETE state', async () => {
641+
mockCfnClient.on(DescribeStacksCommand).resolves({
642+
Stacks: [
643+
{
644+
StackName: 'TestStack',
645+
StackId: 'test-stack-id',
646+
StackStatus: StackStatus.CREATE_COMPLETE,
647+
CreationTime: new Date()
648+
}
649+
]
650+
})
651+
652+
mockCfnClient
653+
.on(CreateChangeSetCommand)
654+
.resolves({ Id: 'test-cs-id', StackId: 'test-stack-id' })
655+
mockCfnClient.on(DescribeChangeSetCommand).resolves({
656+
Status: ChangeSetStatus.CREATE_COMPLETE,
657+
Changes: [{ Type: 'Resource' }]
658+
})
659+
mockCfnClient.on(ExecuteChangeSetCommand).resolves({})
660+
661+
await deployStack(
662+
cfn,
663+
{
664+
StackName: 'TestStack',
665+
TemplateBody: '{}',
666+
Capabilities: []
667+
},
668+
'test-cs',
669+
false,
670+
false,
671+
false,
672+
60
673+
)
674+
675+
expect(mockCfnClient.commandCalls(DeleteStackCommand)).toHaveLength(0)
676+
})
677+
678+
it('handles delete completing when stack disappears', async () => {
679+
mockCfnClient
680+
.on(DescribeStacksCommand)
681+
.resolvesOnce({
682+
Stacks: [
683+
{
684+
StackName: 'TestStack',
685+
StackId: 'test-stack-id',
686+
StackStatus: StackStatus.ROLLBACK_COMPLETE,
687+
CreationTime: new Date()
688+
}
689+
]
690+
})
691+
.resolvesOnce({ Stacks: [] })
692+
.resolves({
693+
Stacks: [
694+
{
695+
StackName: 'TestStack',
696+
StackId: 'new-stack-id',
697+
StackStatus: StackStatus.CREATE_COMPLETE,
698+
CreationTime: new Date()
699+
}
700+
]
701+
})
702+
703+
mockCfnClient.on(DeleteStackCommand).resolves({})
704+
mockCfnClient
705+
.on(CreateChangeSetCommand)
706+
.resolves({ Id: 'test-cs-id', StackId: 'new-stack-id' })
707+
mockCfnClient.on(DescribeChangeSetCommand).resolves({
708+
Status: ChangeSetStatus.CREATE_COMPLETE,
709+
Changes: [{ Type: 'Resource' }]
710+
})
711+
mockCfnClient.on(ExecuteChangeSetCommand).resolves({})
712+
713+
const result = await deployStack(
714+
cfn,
715+
{ StackName: 'TestStack', TemplateBody: '{}', Capabilities: [] },
716+
'test-cs',
717+
false,
718+
false,
719+
false,
720+
60
721+
)
722+
723+
expect(mockCfnClient.commandCalls(DeleteStackCommand)).toHaveLength(1)
724+
expect(result.stackId).toBe('new-stack-id')
725+
})
726+
727+
it('handles delete completing via ValidationError', async () => {
728+
const validationError = new CloudFormationServiceException({
729+
name: 'ValidationError',
730+
$fault: 'client',
731+
$metadata: { httpStatusCode: 400 },
732+
message: 'Stack does not exist'
733+
})
734+
validationError.name = 'ValidationError'
735+
736+
mockCfnClient
737+
.on(DescribeStacksCommand)
738+
.resolvesOnce({
739+
Stacks: [
740+
{
741+
StackName: 'TestStack',
742+
StackId: 'test-stack-id',
743+
StackStatus: StackStatus.ROLLBACK_COMPLETE,
744+
CreationTime: new Date()
745+
}
746+
]
747+
})
748+
.rejectsOnce(validationError)
749+
.resolves({
750+
Stacks: [
751+
{
752+
StackName: 'TestStack',
753+
StackId: 'new-stack-id',
754+
StackStatus: StackStatus.CREATE_COMPLETE,
755+
CreationTime: new Date()
756+
}
757+
]
758+
})
759+
760+
mockCfnClient.on(DeleteStackCommand).resolves({})
761+
mockCfnClient
762+
.on(CreateChangeSetCommand)
763+
.resolves({ Id: 'test-cs-id', StackId: 'new-stack-id' })
764+
mockCfnClient.on(DescribeChangeSetCommand).resolves({
765+
Status: ChangeSetStatus.CREATE_COMPLETE,
766+
Changes: [{ Type: 'Resource' }]
767+
})
768+
mockCfnClient.on(ExecuteChangeSetCommand).resolves({})
769+
770+
const result = await deployStack(
771+
cfn,
772+
{ StackName: 'TestStack', TemplateBody: '{}', Capabilities: [] },
773+
'test-cs',
774+
false,
775+
false,
776+
false,
777+
60
778+
)
779+
780+
expect(result.stackId).toBe('new-stack-id')
781+
})
782+
783+
it('throws error when stack deletion fails', async () => {
784+
mockCfnClient
785+
.on(DescribeStacksCommand)
786+
.resolvesOnce({
787+
Stacks: [
788+
{
789+
StackName: 'TestStack',
790+
StackId: 'test-stack-id',
791+
StackStatus: StackStatus.ROLLBACK_COMPLETE,
792+
CreationTime: new Date()
793+
}
794+
]
795+
})
796+
.resolves({
797+
Stacks: [
798+
{
799+
StackName: 'TestStack',
800+
StackId: 'test-stack-id',
801+
StackStatus: StackStatus.DELETE_FAILED,
802+
CreationTime: new Date()
803+
}
804+
]
805+
})
806+
807+
mockCfnClient.on(DeleteStackCommand).resolves({})
808+
809+
await expect(
810+
deployStack(
811+
cfn,
812+
{ StackName: 'TestStack', TemplateBody: '{}', Capabilities: [] },
813+
'test-cs',
814+
false,
815+
false,
816+
false,
817+
60
818+
)
819+
).rejects.toThrow('Stack deletion failed for TestStack')
820+
})
821+
822+
it('times out waiting for stack deletion', async () => {
823+
mockCfnClient
824+
.on(DescribeStacksCommand)
825+
.resolvesOnce({
826+
Stacks: [
827+
{
828+
StackName: 'TestStack',
829+
StackId: 'test-stack-id',
830+
StackStatus: StackStatus.ROLLBACK_COMPLETE,
831+
CreationTime: new Date()
832+
}
833+
]
834+
})
835+
.resolves({
836+
Stacks: [
837+
{
838+
StackName: 'TestStack',
839+
StackId: 'test-stack-id',
840+
StackStatus: StackStatus.DELETE_IN_PROGRESS,
841+
CreationTime: new Date()
842+
}
843+
]
844+
})
845+
846+
mockCfnClient.on(DeleteStackCommand).resolves({})
847+
848+
const realDateNow = Date.now
849+
let callCount = 0
850+
Date.now = jest.fn(() => {
851+
callCount++
852+
// First call is the startTime capture, subsequent calls exceed maxWaitTime
853+
return callCount <= 1 ? 0 : 2 * 1000 + 1
854+
})
855+
856+
const realSetTimeout = global.setTimeout
857+
global.setTimeout = ((fn: () => void) =>
858+
realSetTimeout(fn, 0)) as unknown as typeof setTimeout
859+
860+
await expect(
861+
deployStack(
862+
cfn,
863+
{ StackName: 'TestStack', TemplateBody: '{}', Capabilities: [] },
864+
'test-cs',
865+
false,
866+
false,
867+
false,
868+
2
869+
)
870+
).rejects.toThrow('Timeout waiting for stack deletion after 2 seconds')
871+
872+
Date.now = realDateNow
873+
global.setTimeout = realSetTimeout
874+
})
875+
})
567876
})

0 commit comments

Comments
 (0)