diff --git a/terraform/environments/init.tf b/terraform/environments/init.tf index e3681012a9ef76f7c992ac63a21c049b8f9f581c..e2c1545b3f7918ba3a67709132cadeeb3707675b 100644 --- a/terraform/environments/init.tf +++ b/terraform/environments/init.tf @@ -21,6 +21,14 @@ data "terraform_remote_state" "holi_infra_state" { } } +data "terraform_remote_state" "holi_infra_monitoring_state" { + backend = "gcs" + config = { + bucket = "holi-shared-terraform-state" + prefix = "infra-monitoring" + } +} + data "terraform_remote_state" "okuna_common_state" { backend = "gcs" config = { diff --git a/terraform/environments/pubsub.tf b/terraform/environments/pubsub.tf index 85b19743ae47365f3b19d1741e183a9820e45499..fd306d5959cd29c55f03a7210ddc6d78be3a7303 100644 --- a/terraform/environments/pubsub.tf +++ b/terraform/environments/pubsub.tf @@ -18,3 +18,23 @@ resource "google_pubsub_subscription" "holi-search-integration-okuna-sub" { } } } + +resource "google_monitoring_alert_policy" "pubsub_unacked_messages" { + project = data.terraform_remote_state.holi_infra_state.outputs.shared_project_id + display_name = "PubSub Unacked Messages Alert" + combiner = "OR" + conditions { + display_name = "Unacked messages condition" + condition_prometheus_query_language { + query = "rate(pubsub_googleapis_com:subscription_oldest_unacked_message_age{subscription_id='${google_pubsub_subscription.holi-search-integration-okuna-sub.name}'}[5m]) > 0" + duration = "300s" + } + } + + notification_channels = [local.environment_name == "production" ? data.terraform_remote_state.holi_infra_monitoring_state.outputs.monitoring_notification_channel_rocket_chat_matching_data_production_id : data.terraform_remote_state.holi_infra_monitoring_state.outputs.monitoring_notification_channel_rocket_chat_matching_data_staging_id] + + documentation { + content = "Alert triggered when Pub/Sub subscription(s) (subscription_id='${google_pubsub_subscription.holi-search-integration-okuna-sub.name}') have unacknowledged messages for more than 5 minutes" + mime_type = "text/markdown" + } +}