From 3e491abe162d1f15113bced9829788b7fc57884b Mon Sep 17 00:00:00 2001
From: Ole Langbehn <ole.langbehn@inoio.de>
Date: Thu, 26 Sep 2024 16:10:34 +0200
Subject: [PATCH] HOLI-10040 HOLI-10041 retry terraform destroy in CI for more
reliable destroy
---
.gitignore | 1 +
terraform/environments/scripts/destroy-env.sh | 45 ++++++++++++++-----
2 files changed, 35 insertions(+), 11 deletions(-)
diff --git a/.gitignore b/.gitignore
index 133ae09..3e3d850 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
.envrc.local
coverage
+terraform*.log
diff --git a/terraform/environments/scripts/destroy-env.sh b/terraform/environments/scripts/destroy-env.sh
index 89d2951..d813769 100755
--- a/terraform/environments/scripts/destroy-env.sh
+++ b/terraform/environments/scripts/destroy-env.sh
@@ -1,18 +1,41 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
# exit when any command fails
set -ex
-# enable full debug output in terraform, which is only written to logfiles
-export TF_LOG=TRACE
+# enable debug output in terraform
+export TF_LOG=DEBUG
-cd terraform/environments
+# retry logic for destroy: sometimes, a full workspace destroy does not work. This can be due to e.g.:
+# * implicit dependencies between terraform resources not declared with depends_on,
+# * unclean shutdown of resources, e.g. service does not close db connections, db still sees clients connected,
+# * GCP stuff not allowing our resources to be deleted.
+# Most of the time, retrying a destroy fixes these causes.
+retry() {
+ for i in {1..3}; do
+ set +e
+ "$@"
+ retval=$?
+ set -e
+ if [ "$retval" -ne "0" ]; then
+ if [ "$i" -lt "3" ]; then
+ echo "command '$*' failed in try $i, retrying after 60 seconds"
+ sleep 60 # let things settle a bit
+ else
+ echo "command '$*' failed in try $i, giving up"
+ exit $retval
+ fi
+ else
+ break # success
+ fi
+ done
+}
-WORKSPACE=$1
-TAG=$2
+cd "$(dirname "$0")"/..
-TF_LOG_PATH=terraform-init.log terraform init
-TF_LOG_PATH=terraform-version.log terraform version
-TF_LOG_PATH=terraform-workspace.log terraform workspace new "$WORKSPACE" || terraform workspace select "$WORKSPACE"
-TF_LOG_PATH=terraform-plan.log terraform plan -var="image_tag=$TAG" -out plan -no-color | tee tfplan.plain
-TF_LOG_PATH=terraform-apply.log terraform apply -auto-approve -parallelism=50 plan
\ No newline at end of file
+TF_LOG_PATH=terraform-init.log terraform init
+TF_LOG_PATH=terraform-version.log terraform version
+TF_LOG_PATH=terraform-workspace.log terraform workspace select -or-create=true "$1"
+TF_LOG_PATH=terraform-destroy.log retry terraform destroy -auto-approve -var="image_tag=dummy"
+TF_LOG_PATH=terraform-ws-default.log terraform workspace select default
+TF_LOG_PATH=terraform-ws-delete.log terraform workspace delete "$1"
--
GitLab