Merge pull request #7204 from GeorgianaElena/fiddle-with-pagers

Add a new pvc-outage alertmanager receiver

Author: GeorgianaElena

helm-charts/support/enc-support.secret.values.yaml

@@ -7,10 +7,9 @@ prometheus:
         - name: ENC[AES256_GCM,data:07mFH5D36H9XR9yGkLJM,iv:jQgUKCV/lWeT1qGICj1/961tgSjSDOio1dk+ybWboyc=,tag:/DLa17ekhqBbjNLzbdb4hg==,type:str]
           pagerduty_configs:
             - routing_key: ENC[AES256_GCM,data:wecu/x0cTsUODGxoDbIh35sPvZI5NyMxtKeM3Hrvyno=,iv:IdZQ6BPbzXGO+evAlYB50G8fxgLajIPWu0utRovYxpk=,tag:V451xfOWo4bDyOhFfa31mw==,type:str]
-        - name: ENC[AES256_GCM,data:z5nrLx65s+HYdStl2GnXJrP0npKn0ZP8,iv:H7AKr/Hk/aWoSAeZWGls+zsRCM2xjCvQBYcnnG9Xzsg=,tag:O2DckmmvSoxnTocxIviC/g==,type:str]
+        - name: ENC[AES256_GCM,data:iEMiTUTCYxdVWag3O7Zn8lfALbi0sxYU/jk=,iv:3fpZdP4wMyZLycvh75a6zXNmudXR+SF6rbD3PLozhc0=,tag:Hmfz+bji/tUYkUkVB0nrhQ==,type:str]
           pagerduty_configs:
-            - routing_key: ENC[AES256_GCM,data:BxX0s1YTUO/h4O6dUs58TsAZkxqwNDMyeYx4bzuJifk=,iv:qTLH5bteyUEXvduIVzy5GtdLv+3GDBHwhj2VAs0+bQ4=,tag:sSgBts/RsWUOoifuJYevPg==,type:str]
-              send_resolved: ENC[AES256_GCM,data:ZpteoqQ=,iv:0TrJjEF7tB2BrhbN2QAoHY3JssCr2NksfycYI6I9zyM=,tag:3zjKfmsMaycpnRrpwTKbLg==,type:bool]
+            - routing_key: ENC[AES256_GCM,data:9vUBomoVOIyvWv/5BCppAc5FWrib2+dm4U+5iY9tIUA=,iv:yX7BbeCvaD7wc6KAQvACJxHw4h4PXdbvHXHf/Rr0H40=,tag:EBfMsgePe62iKBFdtpDniw==,type:str]
         - name: ENC[AES256_GCM,data:3H1uE0zM5/dTpnjrJOOhdznBziZlNPBl,iv:o0oqunEPVep9txxxlqrdeTqwQJKC9tYOpa6piTg8RWw=,tag:nq9tBEFGhgFwF2hR1v0ZqQ==,type:str]
           pagerduty_configs:
             - routing_key: ENC[AES256_GCM,data:D2JPkGj8Ai6pa+0K78f4IeVIpwPAI+uh77JTMp0Mt04=,iv:G2GWWp7IXWV+m//CjYe134wnWuEQiJHxDehKmN59UyY=,tag:JoBVxQrCrL4sF/5bBmautA==,type:str]
@@ -27,7 +26,7 @@ sops:
     - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
       created_at: "2024-12-18T10:09:21Z"
       enc: CiUA4OM7eOThrN3NqDXYd4belMxUdYfF8XpeUSYfiefKkYutN2KoEkkAnGhyNqzzrXyUXTvtS4xl4IjlJpo6hwm9FVbQAyh2Vw9dcXH6h1+NNUHGPYj6KQJvvWZzgXHFLiWdqOfUgTJME0YDx/DYgXgm
-  lastmodified: "2025-09-09T18:53:04Z"
-  mac: ENC[AES256_GCM,data:0O0krdj8krVKq0uwlLJhfvRG/phcQRyh/hYkpVV90t0QCvEa+zH2sfRYZrFL3ZvB20Qf9eqQiZloCiOzLP06hcoJGqjPnq+gR4aKvRIaPkmbopg3l6frxaUxiFg+fAvQ53AUtumDnCnN0XHlu95UOKiOfMTRCgIrWfxquZ7gR/Y=,iv:G++9Q+w7o5z1uiCCclg9W+W2rxTrkVTBV8Cp/ZrCbhM=,tag:Xm/d3VnX4u5k6pHeM4nH7w==,type:str]
+  lastmodified: "2025-11-27T14:44:45Z"
+  mac: ENC[AES256_GCM,data:c4fhwv0CejF1CdFQj4xqf9RGmxpgMpBtB5whKBzA80+7i1nWGmgRBdKhauiOmFj48BZBpvKpknoq4Mx3aZrlW+2wSCYXtCKfmJOYkpo5Je6QpyuA0IhhNxrk6M+svMBWauC252cIyp7beDWj9/OtxtQzyOLt8tsduxYGhnTaOBo=,iv:/Yav3UoGeaIpHuv/LKErrQ1cbkUX1DXveqDpK9QK7JI=,tag:41c0NCYVY5c1g/BTfORIDw==,type:str]
   unencrypted_suffix: _unencrypted
   version: 3.10.2

helm-charts/support/values.jsonnet

@@ -154,6 +154,15 @@ function(VARS_2I2C_AWS_ACCOUNT_ID=null)
             ],
             repeat_interval: '3h',
             routes: [
+              {
+                receiver: 'known-storage-outage-pager',
+                matchers: [
+                  'cluster =~ .*',
+                  'alertname =~ ".*0% of space left.*"',
+                ],
+                // if this one matches, don't check sub-sequent routes
+                continue: false,
+              },
               {
                 receiver: 'persistent-storage-pager',
                 matchers: [
@@ -186,13 +195,13 @@ function(VARS_2I2C_AWS_ACCOUNT_ID=null)
               name: 'PVC available capacity',
               rules: [
                 makePVCApproachingFullAlert(
-                  'Take action! Home Directory Disk very close to full: cluster:%s hub:{{ $labels.namespace }}' % [cluster_name],
+                  'Home Directory Disk very close to full: cluster:%s hub:{{ $labels.namespace }}' % [cluster_name],
                   'home-nfs',
                   10,
                   'same day action needed',
                 ),
                 makePVCApproachingFullAlert(
-                  'Take action! Home Directory Disk very close to full: cluster:%s hub:{{ $labels.namespace }}' % [cluster_name],
+                  'Home Directory Disk is full: cluster:%s hub:{{ $labels.namespace }}' % [cluster_name],
                   'home-nfs',
                   0,
                   'take immediate action',
@@ -204,12 +213,24 @@ function(VARS_2I2C_AWS_ACCOUNT_ID=null)
                   10,
                   'same day action needed'
                 ),
+                makePVCApproachingFullAlert(
+                  'Hub Database Disk is full: cluster:%s hub:{{ $labels.namespace }}' % [cluster_name],
+                  'hub-db-dir',
+                  0,
+                  'take immediate action'
+                ),
                 makePVCApproachingFullAlert(
                   'Prometheus Disk about to be full: cluster:%s' % [cluster_name],
                   'support-prometheus-server',
                   10,
                   'same day action needed'
                 ),
+                makePVCApproachingFullAlert(
+                  'Prometheus Disk is full: cluster:%s' % [cluster_name],
+                  'support-prometheus-server',
+                  0,
+                  'take immediate action'
+                ),
               ],
             },
             {