diff --git a/.gitignore b/.gitignore index 133ae09859ba003be1a2a45e01c67bb65307f54a..3e3d850ca7100199bb3f35f92b827f8958d67fda 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .envrc.local coverage +terraform*.log diff --git a/terraform/environments/scripts/destroy-env.sh b/terraform/environments/scripts/destroy-env.sh index 89d2951406f8c790079204b1a0fe0d1308130939..d8137694745e9a6bcded13142bf6a5e666b55e6c 100755 --- a/terraform/environments/scripts/destroy-env.sh +++ b/terraform/environments/scripts/destroy-env.sh @@ -1,18 +1,41 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash # exit when any command fails set -ex -# enable full debug output in terraform, which is only written to logfiles -export TF_LOG=TRACE +# enable debug output in terraform +export TF_LOG=DEBUG -cd terraform/environments +# retry logic for destroy: sometimes, a full workspace destroy does not work. This can be due to e.g.: +# * implicit dependencies between terraform resources not declared with depends_on, +# * unclean shutdown of resources, e.g. service does not close db connections, db still sees clients connected, +# * GCP stuff not allowing our resources to be deleted. +# Most of the time, retrying a destroy fixes these causes. +retry() { + for i in {1..3}; do + set +e + "$@" + retval=$? + set -e + if [ "$retval" -ne "0" ]; then + if [ "$i" -lt "3" ]; then + echo "command '$*' failed in try $i, retrying after 60 seconds" + sleep 60 # let things settle a bit + else + echo "command '$*' failed in try $i, giving up" + exit $retval + fi + else + break # success + fi + done +} -WORKSPACE=$1 -TAG=$2 +cd "$(dirname "$0")"/.. -TF_LOG_PATH=terraform-init.log terraform init -TF_LOG_PATH=terraform-version.log terraform version -TF_LOG_PATH=terraform-workspace.log terraform workspace new "$WORKSPACE" || terraform workspace select "$WORKSPACE" -TF_LOG_PATH=terraform-plan.log terraform plan -var="image_tag=$TAG" -out plan -no-color | tee tfplan.plain -TF_LOG_PATH=terraform-apply.log terraform apply -auto-approve -parallelism=50 plan \ No newline at end of file +TF_LOG_PATH=terraform-init.log terraform init +TF_LOG_PATH=terraform-version.log terraform version +TF_LOG_PATH=terraform-workspace.log terraform workspace select -or-create=true "$1" +TF_LOG_PATH=terraform-destroy.log retry terraform destroy -auto-approve -var="image_tag=dummy" +TF_LOG_PATH=terraform-ws-default.log terraform workspace select default +TF_LOG_PATH=terraform-ws-delete.log terraform workspace delete "$1"