{"meta":{"date":"2026-10-22","timezone":"UTC","generated_at":"2026-10-30T01:16:43Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":99,"failed_runs":58,"runs_with_occurrences":58,"runs_with_failure_pattern_matches":54,"runs_with_unmatched_occurrences":4,"runs_without_occurrence_rows":41,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113419724510466048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7665,"pr_state":"closed","pr_sha":"68e3bd7d9d2bd53293e48687fa6dfc7b9249ea7b","final_merged_sha":"b40bc5cbb80a171bde8ca5e40c43b439603f5493","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T23:58:22Z","started_at":"2026-10-22T23:58:22Z","completed_at":"2026-10-23T03:00:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T03:10:48.06502841Z","timing_metadata_checked_at":"2026-10-23T03:10:48.06502841Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T03:10:48.067662004Z","region_metadata_checked_at":"2026-10-23T03:10:48.067662004Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"83303ae18f044a536fc424deb4118fc6434531837cb9ccc4f371868e62d3737a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113419724510466048","occurred_at":"2026-10-22T23:58:22Z","failed_at":"e2e","signature_id":"0fc78f7bb3db5f1e5303acea86ceb6e26fde838e0d4e3a4175410040cf2516f9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-b9bhxn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x83317de340\u003e: \n DNS for route host agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x8331542d60\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-b9bhxn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x83317de340\u003e: \n DNS for route host agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x8331542d60\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113406116942581760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"43a9a1c9a1caa8d8332e5913fbd7c37518e0d4b0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T23:04:18Z","started_at":"2026-10-22T23:04:18Z","completed_at":"2026-10-22T23:51:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T00:01:27.954650162Z","timing_metadata_checked_at":"2026-10-23T00:01:27.954650162Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T00:01:27.957996651Z","region_metadata_checked_at":"2026-10-23T00:01:27.957996651Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"53da98fefd755a5ce0884affaa290b152885c80825ae0cbdb9db115501939dff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113406116942581760","occurred_at":"2026-10-22T23:04:18Z","failed_at":"provision","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T23:38:58.792Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-22T23:38:59.096Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-22T23:38:59.096Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-22T23:38:59.335Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-22T23:38:59.485Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-22T23:38:59.485Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-22T23:38:59.547Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-22T23:44:03.236Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-22T23:44:03.237Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113387154783604736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7548,"pr_state":"open","pr_sha":"47f81f80646559a4433eb3e0cdad367eaebf7c0e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T21:48:57Z","started_at":"2026-10-22T21:48:57Z","completed_at":"2026-10-23T00:07:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T00:22:26.046540538Z","timing_metadata_checked_at":"2026-10-23T00:22:26.046540538Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T00:22:26.053584712Z","region_metadata_checked_at":"2026-10-23T00:22:26.053584712Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"274654e3af8d878411fde45332f6a82fa5ebff402b7559398ca3daf82bfb2ae3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113387154783604736","occurred_at":"2026-10-22T21:48:57Z","failed_at":"e2e","signature_id":"200dd7d90d12350bb3d5692f2441c7f6fbacb47c87dbb8b771b20f5be28bb4f1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-4-23-h5dw5xj4gsz5/cluster-candidate-4-23-q8dfp4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x202fbf30a20\u003e: \n failed to create HCP cluster cluster-candidate-4-23-q8dfp4: failed waiting for cluster=\"cluster-candidate-4-23-q8dfp4\" in resourcegroup=\"rg-candidate-4-23-h5dw5xj4gsz5\" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\",\n \"name\": \"1daf02db-3317-4617-a13e-d6a72278331e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T22:42:59.591011904Z\",\n \"endTime\": \"2026-10-22T23:12:07.791955181Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 20m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\n correlation ID: 5bc687aa-4948-47f9-abde-ed1eae592853\n client request ID: 990e5092-5d1b-4b02-8e7c-312e0ee47aa5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-4-23-h5dw5xj4gsz5/cluster-candidate-4-23-q8dfp4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x202fbf30a20\u003e: \n failed to create HCP cluster cluster-candidate-4-23-q8dfp4: failed waiting for cluster=\"cluster-candidate-4-23-q8dfp4\" in resourcegroup=\"rg-candidate-4-23-h5dw5xj4gsz5\" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\",\n \"name\": \"1daf02db-3317-4617-a13e-d6a72278331e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T22:42:59.591011904Z\",\n \"endTime\": \"2026-10-22T23:12:07.791955181Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 20m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\n correlation ID: 5bc687aa-4948-47f9-abde-ed1eae592853\n client request ID: 990e5092-5d1b-4b02-8e7c-312e0ee47aa5\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e538c1de0b1457bef16d4d85afcd040c7c016eeeab3e830942f2fa6a27371a65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113387154783604736","occurred_at":"2026-10-22T21:48:57Z","failed_at":"e2e","signature_id":"58649267055e4c23aa219ef1853e3253b93ce6e21498804b8d24cf5a47de3412","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-5-0-7v6qdnlhwvvb/cluster-candidate-5-0-9zpb54 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a093dfe4480\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9zpb54: failed waiting for cluster=\"cluster-candidate-5-0-9zpb54\" in resourcegroup=\"rg-candidate-5-0-7v6qdnlhwvvb\" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\",\n \"name\": \"23ba9cb2-0511-4188-a063-cf6d4e7f1fde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T22:43:09.924601415Z\",\n \"endTime\": \"2026-10-22T23:12:18.105452223Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 21m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\n correlation ID: 1ba9603d-b743-4b6a-ad8e-362bed8e2691\n client request ID: 5ec430fe-78af-4487-8945-3607302f78b7\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-5-0-7v6qdnlhwvvb/cluster-candidate-5-0-9zpb54 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a093dfe4480\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9zpb54: failed waiting for cluster=\"cluster-candidate-5-0-9zpb54\" in resourcegroup=\"rg-candidate-5-0-7v6qdnlhwvvb\" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\",\n \"name\": \"23ba9cb2-0511-4188-a063-cf6d4e7f1fde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T22:43:09.924601415Z\",\n \"endTime\": \"2026-10-22T23:12:18.105452223Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 21m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\n correlation ID: 1ba9603d-b743-4b6a-ad8e-362bed8e2691\n client request ID: 5ec430fe-78af-4487-8945-3607302f78b7\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e54218362914a42719d7909a07a46d902b9d29461399b8b9db4595254c455df3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113387154783604736","occurred_at":"2026-10-22T21:48:57Z","failed_at":"alert","signature_id":"dec7456054680cbb2852e9fb295189edbf59f26b803821d887eace607491b311","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T22:57:13Z\n Ended: 2026-10-22T23:24:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3604736-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tk46t38vvtso9smm7df5vj1o9gclv0j-b7a9d8e5d1m5r3a\", pod=\"router-b47fc45-hpmlk\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tk46t38vvtso9smm7df5vj1o9gclv0j-b7a9d8e5d1m5r3a/router-b47fc45-hpmlk has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-22T22:58:13Z\n Ended: 2026-10-22T23:24:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3604736-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tk46vmso1j9nnf1s1kifbbej15nl6s5-u2x7s9z5l4p4r7w\", pod=\"router-7c878788f5-hkdfr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tk46vmso1j9nnf1s1kifbbej15nl6s5-u2x7s9z5l4p4r7w/router-7c878788f5-hkdfr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113386303964844032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7119,"pr_state":"closed","pr_sha":"cc0c5ef047b5bd0a2bf6e1ddc04dbd009721f1ba","final_merged_sha":"cc0c5ef047b5bd0a2bf6e1ddc04dbd009721f1ba","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T21:45:32Z","started_at":"2026-10-22T21:45:32Z","completed_at":"2026-10-23T00:05:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T00:22:25.902156477Z","timing_metadata_checked_at":"2026-10-23T00:22:25.902156477Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T00:22:25.924500694Z","region_metadata_checked_at":"2026-10-23T00:22:25.924500694Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7703/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113386278404755456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7703,"pr_state":"closed","pr_sha":"333aa14c3d9c2e3fcabc6d393e814daf2efffb2b","final_merged_sha":"333aa14c3d9c2e3fcabc6d393e814daf2efffb2b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T21:45:28Z","started_at":"2026-10-22T21:45:28Z","completed_at":"2026-10-23T00:12:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T00:22:26.181603534Z","timing_metadata_checked_at":"2026-10-23T00:22:26.181603534Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T00:22:26.191151599Z","region_metadata_checked_at":"2026-10-23T00:22:26.191151599Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113385464588144640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7547,"pr_state":"closed","pr_sha":"28c20c5eff68a4950667d98fdfc5163f1798bdff","final_merged_sha":"cd3c21e53a196ba0ad2c38b6f73f33b744c9e3fa","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T21:42:14Z","started_at":"2026-10-22T21:42:14Z","completed_at":"2026-10-22T22:38:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T22:46:54.588605322Z","timing_metadata_checked_at":"2026-10-22T22:46:54.588605322Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T22:46:54.593397609Z","region_metadata_checked_at":"2026-10-22T22:46:54.593397609Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"68b418fca67d0a12c42d47060e4983c49718d17d744e0d077e663f9c0ae6dcc5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113385464588144640","occurred_at":"2026-10-22T21:42:14Z","failed_at":"alert","signature_id":"0a5c75aa0c2fe74940638b97aa581fdd9a1b679cd6ad35f756ff074c101cad2c","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 21 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-qm65f\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-qm65f has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-646rk\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-646rk has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-97mz4\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-97mz4 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-ztvz7\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-ztvz7 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-6vncz\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-6vncz has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-vw4s5\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-vw4s5 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-8tcjd\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-8tcjd has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-rf2c7\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-rf2c7 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-tv8lq\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-tv8lq has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-74f8k\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-74f8k has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-x5msf\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-x5msf has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-hfcqc\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-hfcqc has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-nhww7\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-nhww7 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-99qll\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-99qll has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-8zp7x\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-8zp7x has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-6lchz\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-6lchz has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-m29wl\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-m29wl has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-tvhb4\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-tvhb4 has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-t9zc6\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-t9zc6 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-47967\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-47967 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Fired (not resolved)\n Started: 2026-10-22T22:23:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8144640-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-2bsv7\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-2bsv7 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"fcfb56a8e4acf6a05fac4d0620e1e73e14d25da1158324219bb4ffee83b9b3e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113385464588144640","occurred_at":"2026-10-22T21:42:14Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T22:17:12.835Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-22T22:17:14.824Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8144640-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-22T22:17:14.824Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8144640-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-22T22:17:16.343Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8144640-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-22T22:17:16.343Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8144640-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T22:17:16.344Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j8144640-svc\ntime=2026-10-22T22:32:19.729Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113384890496978944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7650,"pr_state":"closed","pr_sha":"ea414f13a62125e03b134d3877a8002ce4b6b014","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T21:39:57Z","started_at":"2026-10-22T21:39:57Z","completed_at":"2026-10-22T23:54:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T00:11:51.973736815Z","timing_metadata_checked_at":"2026-10-23T00:11:51.973736815Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T00:11:51.977643805Z","region_metadata_checked_at":"2026-10-23T00:11:51.977643805Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7667/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113384044728160256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7667,"pr_state":"closed","pr_sha":"59ae68cb1d04712c5175790a72ad028c91c17203","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T21:36:35Z","started_at":"2026-10-22T21:36:35Z","completed_at":"2026-10-22T23:55:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T00:11:52.246230205Z","timing_metadata_checked_at":"2026-10-23T00:11:52.246230205Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T00:11:52.249380396Z","region_metadata_checked_at":"2026-10-23T00:11:52.249380396Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a410ff955469313ca93ed0fdb5f2f8a7b4d1a643786855965682a3d5c3676db4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7667/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113384044728160256","occurred_at":"2026-10-22T21:36:35Z","failed_at":"e2e","signature_id":"d6bbcd1d10bc7d627f007c65311d55d2d53b06f7912e2f2c8d382d44b94fe9f5","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x32b05a79ea40\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x32b05c5389f0\u003e{\n s: \"dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x32b05a79ea40\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x32b05c5389f0\u003e{\n s: \"dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7cd226b1ef7b8222b38971a112303a8516ff1f0851d469499d37f592a43adc2a","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7cd226b1ef7b8222b38971a112303a8516ff1f0851d469499d37f592a43adc2a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113374411854712832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7495,"pr_state":"closed","pr_sha":"7c8f0c1cecbb740555ccb24e3f292792ac64e0ea","final_merged_sha":"c8ebae4103c841cfb1590b88cc329be1a9829823","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T20:58:18Z","started_at":"2026-10-22T20:58:18Z","completed_at":"2026-10-22T23:26:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T23:40:08.576447783Z","timing_metadata_checked_at":"2026-10-22T23:40:08.576447783Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T23:40:08.581180471Z","region_metadata_checked_at":"2026-10-22T23:40:08.581180471Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"21b2eabb12f85cee20ca1e07c4bbe8083e5e60041a51cfa973c7d99bf388cc29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113374411854712832","occurred_at":"2026-10-22T20:58:18Z","failed_at":"alert","signature_id":"3845ce22d709f7d6f88c6b78d29a932df4b7e3435bd9d98ded715b56bb04b04e","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j4712832 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T22:34:51Z\n Ended: 2026-10-22T22:41:02Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired"}},{"row_id":"d3a2eebf7e801df107eeadb80a8f2c1ed077bcfb02a5e16cbc2074931ac3c9c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113374411854712832","occurred_at":"2026-10-22T20:58:18Z","failed_at":"alert","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j4712832 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T22:23:37Z\n Ended: 2026-10-22T22:51:36Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113373941157335040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7177,"pr_state":"closed","pr_sha":"27fe8fe46cb154c4b4ff583dea5a4d9ad0f93dd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T20:56:26Z","started_at":"2026-10-22T20:56:26Z","completed_at":"2026-10-22T23:06:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T23:19:03.863101197Z","timing_metadata_checked_at":"2026-10-22T23:19:03.863101197Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T23:19:03.868561579Z","region_metadata_checked_at":"2026-10-22T23:19:03.868561579Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"83bbb62a01a528553dc6c273a48a2cc97a3679715cc0930337f0ef396a3402d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113373941157335040","occurred_at":"2026-10-22T20:56:26Z","failed_at":"e2e","signature_id":"e0dcd02397ab4d23ba24e7c57448f2841b01c0fa54fb3bb9faad619af0f10aa6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-8s2f2l\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x30cafd6d0a20\u003e: \n DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x30cafd44e600\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-8s2f2l\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x30cafd6d0a20\u003e: \n DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x30cafd44e600\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113371217632169984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7119,"pr_state":"closed","pr_sha":"cc0c5ef047b5bd0a2bf6e1ddc04dbd009721f1ba","final_merged_sha":"cc0c5ef047b5bd0a2bf6e1ddc04dbd009721f1ba","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T20:45:37Z","started_at":"2026-10-22T20:45:37Z","completed_at":"2026-10-22T21:21:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T21:32:52.676582069Z","timing_metadata_checked_at":"2026-10-22T21:32:52.676582069Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T21:32:52.679844758Z","region_metadata_checked_at":"2026-10-22T21:32:52.679844758Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"64ddd992b36d803c2dde3750a3630a4a848478db5ea7d7ae169257351c8fc5e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113371217632169984","occurred_at":"2026-10-22T20:45:37Z","failed_at":"provision","signature_id":"7dba303763564f512f9c5cae72718a1ea571c1a5f6ff70b0d0efd7d6d4c2edaa","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T21:15:19.830Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n Kind: Shell\\n Command: ./swift-vnet.sh\\n\"\ntime=2026-10-22T21:15:40.577Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Launching container group swift-vnet-aks-net as globalMSI...\\nWaiting for swift-vnet-aks-net to finish (state: pending, elapsed 2s)...\\nāœ— Container group swift-vnet-aks-net failed to provision\\n\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c3bac01c69d371179a684cc643681dbf473300fa462d0ddc425d19fb5a7bff64","failure_pattern":"āœ— Container group swift-vnet-aks-net failed to provision","search_query":"āœ— Container group swift-vnet-aks-net failed to provision"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c3bac01c69d371179a684cc643681dbf473300fa462d0ddc425d19fb5a7bff64"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7703/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113366191513800704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7703,"pr_state":"closed","pr_sha":"333aa14c3d9c2e3fcabc6d393e814daf2efffb2b","final_merged_sha":"333aa14c3d9c2e3fcabc6d393e814daf2efffb2b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T20:25:38Z","started_at":"2026-10-22T20:25:38Z","completed_at":"2026-10-22T21:41:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T21:53:23.808250065Z","timing_metadata_checked_at":"2026-10-22T21:53:23.808250065Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T21:53:23.812409451Z","region_metadata_checked_at":"2026-10-22T21:53:23.812409451Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"23c3deda310c6bd8c0e9f580dc48657fb4c7976fd12296d5645d28c0e3bb98b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7703/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113366191513800704","occurred_at":"2026-10-22T20:25:38Z","failed_at":"alert","signature_id":"8622c4f05e530429c2435ac6bd159aff27f3109548086ec61b7cd9ecf8f87649","test_name":"[aro-hcp-observability] [svc] alert PrometheusJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T21:19:53Z\n Severity: Sev2\n Labels: alertname=\"PrometheusJobUp\", cluster=\"ci01-j3800704-mgmt-1\", component=\"monitoring-infrastructure\", severity=\"critical\"\n Description: Prometheus has not been reachable for the past 10 minutes.\nThis may indicate that the Prometheus server is down, unreachable due to network issues, or experiencing a crash loop.\nCheck the status of the Prometheus pods, service endpoints, and network connectivity.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fb0cbad5198bdae4515e07599d2b1b6ff3ac6cb7da7109069ce8cd804ef1fb4b","failure_pattern":"alert [svc] PrometheusJobUp fired","search_query":"alert [svc] PrometheusJobUp fired"}},{"row_id":"4887317f1ee5fc414e16f86767ec387495260c593628d0af3da4bbf8c18dced0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7703/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113366191513800704","occurred_at":"2026-10-22T20:25:38Z","failed_at":"alert","signature_id":"f42018e30e12f894f1629fdf1b2b27fda4bc7322b3ae6f164b98305381f5a376","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T21:30:04Z\n Severity: Sev3\n Labels: alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j3800704-mgmt-1\", component=\"monitoring-infrastructure\", severity=\"warning\"\n Description: The Arobit forwarder metrics endpoint on cluster ci01-j3800704-mgmt-1 has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"7e88d534bb05b49b90dd7953af1efa701a6c5b7f5089077ec8fb364733b46741","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7703/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113366191513800704","occurred_at":"2026-10-22T20:25:38Z","failed_at":"alert","signature_id":"21ef629d9815e1be0aa9ab1b5d0b1208b139615a20d5ce194bf36c9d123c0ea8","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T21:15:12Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j3800704-mgmt-2\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T21:24:13Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j3800704-mgmt-1\", component=\"monitoring-infrastructure\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}},{"row_id":"b28514b01413107e2c14f2f3bfb25961984debc1608fa12f3e931f9a4cc08574","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7703/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113366191513800704","occurred_at":"2026-10-22T20:25:38Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T21:04:17.903Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-10-22T21:04:18.672Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-10-22T21:04:19.255Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=6a5336e24f93f554fd8b0d65c59b856837a40a2fdae6e7d0fbc40c8888d8d17e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3800704-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6a5336e24f93f554fd8b0d65c59b856837a40a2fdae6e7d0fbc40c8888d8d17e\ntime=2026-10-22T21:07:49.884Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3800704-mgmt-1/providers/Microsoft.Resources/deployments/6a5336e24f93f554fd8b0d65c59b856837a40a2fdae6e7d0fbc40c8888d8d17e/operationStatuses/08584089038265307890\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3800704-mgmt-1/providers/Microsoft.Resources/deployments/fe73d360-7d74-562a-b1ee-44865af0a11a' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/22/2026 9:06:20 PM' with correlationId 'fe6d7e33-2867-4647-8fe8-d9c4bbb9af94', and will expire at '10/29/2026 9:06:20 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e9cd130a5a7f9bb66c6119ae4848ee76685bc008fdd6564ac11ea576600fa92","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/\u003cuuid\u003e' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["4e9cd130a5a7f9bb66c6119ae4848ee76685bc008fdd6564ac11ea576600fa92","5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","fb0cbad5198bdae4515e07599d2b1b6ff3ac6cb7da7109069ce8cd804ef1fb4b"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113366014895853568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7698,"pr_state":"closed","pr_sha":"debf842270f7b7563708c23457398f40ecdce214","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T20:24:57Z","started_at":"2026-10-22T20:24:57Z","completed_at":"2026-10-22T22:39:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T22:46:54.719101466Z","timing_metadata_checked_at":"2026-10-22T22:46:54.719101466Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T22:46:54.722993656Z","region_metadata_checked_at":"2026-10-22T22:46:54.722993656Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113360508831666176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7177,"pr_state":"closed","pr_sha":"63d94bccfcc49b2a7c67fa37521431489810121b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T20:03:04Z","started_at":"2026-10-22T20:03:04Z","completed_at":"2026-10-22T20:03:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T20:09:43.880899473Z","timing_metadata_checked_at":"2026-10-22T20:09:43.880899473Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-22T20:09:43.88732865Z","region_metadata_checked_at":"2026-10-22T21:11:27.081885549Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0d4aa476e34e09e0bc77152dbdc7c51051025485ac0adadbc967fcd22ec773f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113360508831666176","occurred_at":"2026-10-22T20:03:04Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113355573662584832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7119,"pr_state":"closed","pr_sha":"cc0c5ef047b5bd0a2bf6e1ddc04dbd009721f1ba","final_merged_sha":"cc0c5ef047b5bd0a2bf6e1ddc04dbd009721f1ba","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T19:43:11Z","started_at":"2026-10-22T19:43:11Z","completed_at":"2026-10-22T20:15:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T20:30:20.745680437Z","timing_metadata_checked_at":"2026-10-22T20:30:20.745680437Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-22T20:30:20.749329527Z","region_metadata_checked_at":"2026-10-22T21:32:52.513497387Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2f3048c4f5d2bad07213ea20cae585db0f510887d19aa47babd1037d975f2328","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113355573662584832","occurred_at":"2026-10-22T19:43:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113344226350600192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7700,"pr_state":"closed","pr_sha":"abd5cbbc075c4b61d0a1de4c05b93c3e6f5649fb","final_merged_sha":"50bf39bf591aa2b64e5bedba4ba2f4443d76d279","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T18:58:22Z","started_at":"2026-10-22T18:58:22Z","completed_at":"2026-10-22T21:07:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T21:22:27.039123953Z","timing_metadata_checked_at":"2026-10-22T21:22:27.039123953Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T21:22:27.056556506Z","region_metadata_checked_at":"2026-10-22T21:22:27.056556506Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"closed","pr_sha":"e159b4bb81965488c7898a51e008061eee591672","final_merged_sha":"afd3417980c4b5052b54fdf377e648bf6df383fc","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T18:39:26Z","started_at":"2026-10-22T18:39:26Z","completed_at":"2026-10-22T20:53:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T21:01:00.805865267Z","timing_metadata_checked_at":"2026-10-22T21:01:00.805865267Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T21:01:00.809819654Z","region_metadata_checked_at":"2026-10-22T21:01:00.809819654Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"1778f4c3a9c2a5c6821eb1a179187f0c10b5dd7b60710eadb9b3a08dd7c8a0c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","failed_at":"e2e","signature_id":"3a71b87c724dd30609861eaf297fd7c712b6b149707e839f6721959f2a1a471d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0xc8f6d4d40a8\u003e: \n resource group \"rg-candidate-4-23-wv4m7dxwvslv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-llx74k\" in resourcegroup=\"rg-candidate-4-23-wv4m7dxwvslv\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10\",\n \"name\": \"38d6d256-b8f5-43d6-9bea-bc7db4e81a10\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T19:58:57.002004594Z\",\n \"endTime\": \"2026-10-22T20:23:04.238386188Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10\n correlation ID: 89679606-4473-4b98-af12-ab459a5b124c\n client request ID: e90666ef-7f96-44ec-8272-834002e9a78d\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0xc8f6d4d40a8\u003e: \n resource group \"rg-candidate-4-23-wv4m7dxwvslv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-llx74k\" in resourcegroup=\"rg-candidate-4-23-wv4m7dxwvslv\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10\",\n \"name\": \"38d6d256-b8f5-43d6-9bea-bc7db4e81a10\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T19:58:57.002004594Z\",\n \"endTime\": \"2026-10-22T20:23:04.238386188Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10\n correlation ID: 89679606-4473-4b98-af12-ab459a5b124c\n client request ID: e90666ef-7f96-44ec-8272-834002e9a78d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1f3315bcd622bda48ffc70ef1de67a2ebdc10051144f137d89ac8697373e9673","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","failed_at":"e2e","signature_id":"bfa94f3d303830ea50f4ae4f2810d453787700edaab1bd594d0d227e0fd34552","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x3e4a7eaec120\u003e: \n resource group \"kusto-logs-bjqbcmj9fh99\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bjqbcmj9fh99\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\",\n \"name\": \"133a0c79-b758-4022-b9d3-abe3437fdb7e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T20:01:21.775547749Z\",\n \"endTime\": \"2026-10-22T20:25:24.154021571Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\n correlation ID: 876f3f40-3d0a-4f9d-b0a1-13067115d7f2\n client request ID: a6519fef-aba9-4692-8b59-2cc44c90a17d\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x3e4a7eaec120\u003e: \n resource group \"kusto-logs-bjqbcmj9fh99\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bjqbcmj9fh99\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\",\n \"name\": \"133a0c79-b758-4022-b9d3-abe3437fdb7e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T20:01:21.775547749Z\",\n \"endTime\": \"2026-10-22T20:25:24.154021571Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\n correlation ID: 876f3f40-3d0a-4f9d-b0a1-13067115d7f2\n client request ID: a6519fef-aba9-4692-8b59-2cc44c90a17d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5d0b750937ad9c8f0f72861db1d442d01b73ecb0297469b6926108a17b0ca13b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","failed_at":"e2e","signature_id":"8cfd4f8b1047556e1120af67bd86078b72621252b89d930d39333adaf2164a5c","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x5f7edbba210\u003e: \n resource group \"feature-aggregation-9bffc7pfd9cf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9bffc7pfd9cf\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\",\n \"name\": \"80845250-6487-4bd8-bc2d-fcba49147d69\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T20:00:56.002177295Z\",\n \"endTime\": \"2026-10-22T20:25:04.157804188Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\n correlation ID: b47a1ca2-3039-4616-a8a8-424022677398\n client request ID: da4cb843-dd82-426b-ae29-8754ca365c47\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x5f7edbba210\u003e: \n resource group \"feature-aggregation-9bffc7pfd9cf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9bffc7pfd9cf\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\",\n \"name\": \"80845250-6487-4bd8-bc2d-fcba49147d69\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T20:00:56.002177295Z\",\n \"endTime\": \"2026-10-22T20:25:04.157804188Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\n correlation ID: b47a1ca2-3039-4616-a8a8-424022677398\n client request ID: da4cb843-dd82-426b-ae29-8754ca365c47\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6eb333cda2364c3447e130d1f1851e1939994c7c46dc6e1f1523202906c09b96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","failed_at":"e2e","signature_id":"37fb4527b0dca28de2500cb1a873ec11aeee90e63a01f0b3975e6a14eb0ed6fc","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-kb7qmr with version 4.21.39\nUnexpected error:\n \u003c*fmt.wrapError | 0x28069869e800\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-kb7qmr: failed waiting for cluster=\"np-version-upgrade-cluster-kb7qmr\" in resourcegroup=\"rg-np-version-upgrade-kb7qmr-c47gf99np5dv\" to finish creating: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0\",\n \"name\": \"9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T19:49:25.431008286Z\",\n \"endTime\": \"2026-10-22T20:18:34.116415534Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0\n correlation ID: 0f045ea6-0c55-4d5b-bd0c-acde888327dd\n client request ID: cee3d7d5-1361-4cfc-b548-f2d63668a2b7\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-kb7qmr with version 4.21.39\nUnexpected error:\n \u003c*fmt.wrapError | 0x28069869e800\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-kb7qmr: failed waiting for cluster=\"np-version-upgrade-cluster-kb7qmr\" in resourcegroup=\"rg-np-version-upgrade-kb7qmr-c47gf99np5dv\" to finish creating: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0\",\n \"name\": \"9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T19:49:25.431008286Z\",\n \"endTime\": \"2026-10-22T20:18:34.116415534Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0\n correlation ID: 0f045ea6-0c55-4d5b-bd0c-acde888327dd\n client request ID: cee3d7d5-1361-4cfc-b548-f2d63668a2b7\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8b2d36683ddaa6437e21bd4aa9ceeb21012f3c8068fd69a43bb2423b6694f52e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","failed_at":"e2e","signature_id":"a6a6d7ef95b9bce2823ee44c5133f92a6c026911f7cba3ed9bfc9fb6134bd9b7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x1445b90de228\u003e: \n resource group \"rg-candidate-4-21-td4mhfs4j8v5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-grnnwp\" in resourcegroup=\"rg-candidate-4-21-td4mhfs4j8v5\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e\",\n \"name\": \"cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T19:56:06.651640707Z\",\n \"endTime\": \"2026-10-22T20:20:14.650326027Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e\n correlation ID: ecab9896-2020-4354-a4df-380a8a550a70\n client request ID: 3bfbbe06-8120-4d72-8516-3e9a6561ddcd\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x1445b90de228\u003e: \n resource group \"rg-candidate-4-21-td4mhfs4j8v5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-grnnwp\" in resourcegroup=\"rg-candidate-4-21-td4mhfs4j8v5\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e\",\n \"name\": \"cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T19:56:06.651640707Z\",\n \"endTime\": \"2026-10-22T20:20:14.650326027Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e\n correlation ID: ecab9896-2020-4354-a4df-380a8a550a70\n client request ID: 3bfbbe06-8120-4d72-8516-3e9a6561ddcd\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"92a484578af76811d83b3e1ac6736841a217e96bc2709246b671746fbfb76af0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","failed_at":"e2e","signature_id":"f6273eed30538960aa15eba547ebb2f777f14cefb915530b47ab177adf25601c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x1e7a4ac381c8\u003e: \n resource group \"rg-candidate-5-0-54ll7ff5h97p\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-bdl7b6\" in resourcegroup=\"rg-candidate-5-0-54ll7ff5h97p\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588\",\n \"name\": \"56d97582-5397-4afa-ae32-c0de69fc1588\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T20:00:21.519712195Z\",\n \"endTime\": \"2026-10-22T20:24:24.151438373Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588\n correlation ID: 086aa11a-7ea6-463b-a979-85ac6ded58d4\n client request ID: 7e8f297b-44a9-439d-b3c8-54c4cf3d8847\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x1e7a4ac381c8\u003e: \n resource group \"rg-candidate-5-0-54ll7ff5h97p\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-bdl7b6\" in resourcegroup=\"rg-candidate-5-0-54ll7ff5h97p\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588\",\n \"name\": \"56d97582-5397-4afa-ae32-c0de69fc1588\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T20:00:21.519712195Z\",\n \"endTime\": \"2026-10-22T20:24:24.151438373Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588\n correlation ID: 086aa11a-7ea6-463b-a979-85ac6ded58d4\n client request ID: 7e8f297b-44a9-439d-b3c8-54c4cf3d8847\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113337883904446464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7633,"pr_state":"closed","pr_sha":"1b7d89e2b17f9288ea983c06d8e9ece5e53c383e","final_merged_sha":"1b7d89e2b17f9288ea983c06d8e9ece5e53c383e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T18:33:10Z","started_at":"2026-10-22T18:33:10Z","completed_at":"2026-10-22T20:47:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T21:01:01.265774212Z","timing_metadata_checked_at":"2026-10-22T21:01:01.265774212Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T21:01:01.282561459Z","region_metadata_checked_at":"2026-10-22T21:01:01.282561459Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113337766480711680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"5db7c10ae1b400f88402e4f17ef0ac84aba3e9d1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T18:32:42Z","started_at":"2026-10-22T18:32:42Z","completed_at":"2026-10-22T20:49:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T21:01:01.414414742Z","timing_metadata_checked_at":"2026-10-22T21:01:01.414414742Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T21:01:01.417843131Z","region_metadata_checked_at":"2026-10-22T21:01:01.417843131Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"17730e957297d76b280d86d80b499930017d5399f4dac7daa13f9fa7650a8e2a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113337766480711680","occurred_at":"2026-10-22T18:32:42Z","failed_at":"e2e","signature_id":"e89a1c1574690bc16c1102c73369b4e5d51aa6d1b3d8433e36b4576eb66adc6f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0xb52cc494270\u003e: \n resource group \"rg-candidate-4-22-dbz2f2dgxvsz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-5sf4vm\" in resourcegroup=\"rg-candidate-4-22-dbz2f2dgxvsz\" to finish deleting: GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c\",\n \"name\": \"e97180c3-ec2d-41de-ba05-4c97d3788b0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T19:51:01.626448313Z\",\n \"endTime\": \"2026-10-22T20:15:04.909018235Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c\n correlation ID: 06dd90eb-d763-414b-9dbc-6cbbd01f522d\n client request ID: 134d05fc-4c49-4a17-ae46-0db6a5b33c79\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0xb52cc494270\u003e: \n resource group \"rg-candidate-4-22-dbz2f2dgxvsz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-5sf4vm\" in resourcegroup=\"rg-candidate-4-22-dbz2f2dgxvsz\" to finish deleting: GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c\",\n \"name\": \"e97180c3-ec2d-41de-ba05-4c97d3788b0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T19:51:01.626448313Z\",\n \"endTime\": \"2026-10-22T20:15:04.909018235Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c\n correlation ID: 06dd90eb-d763-414b-9dbc-6cbbd01f522d\n client request ID: 134d05fc-4c49-4a17-ae46-0db6a5b33c79\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3c45de080237f587d232f9fd65fd34434f62ee8df75e69bf7a53177f04899ad5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113337766480711680","occurred_at":"2026-10-22T18:32:42Z","failed_at":"alert","signature_id":"6ec942e8ee5fad5d5c4fba3fbbe4fe73778eae33cb5e869a5d69169f1367dfe6","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T19:39:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0711680-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.242:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0711680/providers/microsoft.monitor/accounts/services-j0711680\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-86mcm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"44790cfe-d60d-4b27-a39b-93e0f60af9be\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-86mcm (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T19:40:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0711680-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.234:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0711680/providers/microsoft.monitor/accounts/services-j0711680\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-8wz92\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d334bde9-8444-442b-a710-d97c48525e23\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-8wz92 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-22T19:41:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0711680-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.242:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0711680/providers/microsoft.monitor/accounts/services-j0711680\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-86mcm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"44790cfe-d60d-4b27-a39b-93e0f60af9be\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-86mcm (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-22T19:42:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0711680-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.234:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0711680/providers/microsoft.monitor/accounts/services-j0711680\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-8wz92\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d334bde9-8444-442b-a710-d97c48525e23\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-8wz92 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"87b11013daa5a0320337e9b078d41821b1f97e85d4dc268d95a614209073b60a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113337766480711680","occurred_at":"2026-10-22T18:32:42Z","failed_at":"e2e","signature_id":"7d48d0783a7d6fa8287cfe8694ee010e0fe2aa87656f0647076142aac123bae1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x20e75a338120\u003e: \n resource group \"rg-candidate-4-23-wm8dczw7qltp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-q6d2hl\" in resourcegroup=\"rg-candidate-4-23-wm8dczw7qltp\" to finish deleting: GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683\",\n \"name\": \"7182a919-4fbb-49ce-83aa-dd6eb45ba683\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T19:49:32.282155892Z\",\n \"endTime\": \"2026-10-22T20:13:36.499422253Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683\n correlation ID: ccfc9272-4802-4cdf-a65e-9aaeb8062ade\n client request ID: e0488b7a-701b-4302-8c93-fb4506094516\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x20e75a338120\u003e: \n resource group \"rg-candidate-4-23-wm8dczw7qltp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-q6d2hl\" in resourcegroup=\"rg-candidate-4-23-wm8dczw7qltp\" to finish deleting: GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683\",\n \"name\": \"7182a919-4fbb-49ce-83aa-dd6eb45ba683\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T19:49:32.282155892Z\",\n \"endTime\": \"2026-10-22T20:13:36.499422253Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683\n correlation ID: ccfc9272-4802-4cdf-a65e-9aaeb8062ade\n client request ID: e0488b7a-701b-4302-8c93-fb4506094516\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bd1adf9855dd1c3b8f5d8f78675b9a61c420ccb075155424a7ecd47a5fce0c10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113337766480711680","occurred_at":"2026-10-22T18:32:42Z","failed_at":"e2e","signature_id":"4249d51f56b51a2a4f36f216d451e5000bdfdbe8cc19e478c64f0ed465dd4943","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3863a797e030\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3863a797e030\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd8293942caca74f6fdf5f2f77a272ddca84daf5e523e4bbdebdc477d7c3ee71","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","dd8293942caca74f6fdf5f2f77a272ddca84daf5e523e4bbdebdc477d7c3ee71"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7697/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113336312449732608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7697,"pr_state":"open","pr_sha":"689b0add8ab2ae379ff2f50ab7524d3c692900d1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T18:26:55Z","started_at":"2026-10-22T18:26:55Z","completed_at":"2026-10-22T18:56:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T19:06:35.996531503Z","timing_metadata_checked_at":"2026-10-22T19:06:35.996531503Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T19:06:36.000049893Z","region_metadata_checked_at":"2026-10-22T19:06:36.000049893Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2b37e7063210dcf1b6b2a61043541f6dd0e2886e1f0539f42ffd823dae2c031e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7697/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113336312449732608","occurred_at":"2026-10-22T18:26:55Z","failed_at":"provision","signature_id":"980c914d74e054017fdcac7ea3a9f64034c21dfbdbe2586220b7dbcbd475b756","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T18:49:27.700Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 description=\"Step swift-vnet\\n Kind: Shell\\n Command: ./swift-vnet.sh\\n\"\ntime=2026-10-22T18:49:46.822Z level=ERROR msg=\"failed to remove step Azure CLI config directory\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 err=\"unlinkat /tmp/azure-cli-step-config-1336382318: directory not empty\" dir=/tmp/azure-cli-step-config-1336382318\ntime=2026-10-22T18:49:46.822Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: Launching container group swift-vnet-aks-net as globalMSI...\\nWaiting for swift-vnet-aks-net to finish (state: pending, elapsed 1s)...\\nāœ— Container group swift-vnet-aks-net failed to provision\\n\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c3bac01c69d371179a684cc643681dbf473300fa462d0ddc425d19fb5a7bff64","failure_pattern":"āœ— Container group swift-vnet-aks-net failed to provision","search_query":"āœ— Container group swift-vnet-aks-net failed to provision"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c3bac01c69d371179a684cc643681dbf473300fa462d0ddc425d19fb5a7bff64"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7699/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113334356226347008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7699,"pr_state":"closed","pr_sha":"e1714c92c0d92587c6d182104614f1005a270c49","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T18:19:08Z","started_at":"2026-10-22T18:19:08Z","completed_at":"2026-10-22T20:39:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T20:50:44.773682587Z","timing_metadata_checked_at":"2026-10-22T20:50:44.773682587Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T20:50:44.77680628Z","region_metadata_checked_at":"2026-10-22T20:50:44.77680628Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113333668549234688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7650,"pr_state":"closed","pr_sha":"55bbf252ffdf4363fe4f0969a2b8e33b68cb3ea8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T18:16:25Z","started_at":"2026-10-22T18:16:25Z","completed_at":"2026-10-22T20:25:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T20:30:21.140840271Z","timing_metadata_checked_at":"2026-10-22T20:30:21.140840271Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T20:30:21.144078562Z","region_metadata_checked_at":"2026-10-22T20:30:21.144078562Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113332093210595328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7133,"pr_state":"closed","pr_sha":"45037000851deb39176b3b62f286f06464c87985","final_merged_sha":"45037000851deb39176b3b62f286f06464c87985","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T18:10:09Z","started_at":"2026-10-22T18:10:09Z","completed_at":"2026-10-22T20:31:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T20:40:36.454437353Z","timing_metadata_checked_at":"2026-10-22T20:40:36.454437353Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T20:40:36.457563339Z","region_metadata_checked_at":"2026-10-22T20:40:36.457563339Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7692/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113331670458306560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7692,"pr_state":"closed","pr_sha":"6b6f1019dd6509a28f35ea8fb265dadbee990b39","final_merged_sha":"6b6f1019dd6509a28f35ea8fb265dadbee990b39","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T18:08:28Z","started_at":"2026-10-22T18:08:28Z","completed_at":"2026-10-22T20:35:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T20:50:44.324637413Z","timing_metadata_checked_at":"2026-10-22T20:50:44.324637413Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T20:50:44.328220205Z","region_metadata_checked_at":"2026-10-22T20:50:44.328220205Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7667/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113331501809537024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7667,"pr_state":"closed","pr_sha":"cfe1fda9ac6c78f61e8a940001dede83afcedf13","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T18:07:48Z","started_at":"2026-10-22T18:07:48Z","completed_at":"2026-10-22T20:29:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T20:40:36.843112692Z","timing_metadata_checked_at":"2026-10-22T20:40:36.843112692Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T20:40:36.845910079Z","region_metadata_checked_at":"2026-10-22T20:40:36.845910079Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7394/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113329770732195840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7394,"pr_state":"open","pr_sha":"54d072f652cd6978a2b9b8dc5ac71929281811db","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T18:00:21Z","started_at":"2026-10-22T18:00:21Z","completed_at":"2026-10-22T20:28:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T20:40:36.68019143Z","timing_metadata_checked_at":"2026-10-22T20:40:36.68019143Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T20:40:36.683252216Z","region_metadata_checked_at":"2026-10-22T20:40:36.683252216Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113319859344379904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7665,"pr_state":"closed","pr_sha":"1c6e3257072bbcaaffb95f99a79650cb360720b2","final_merged_sha":"b40bc5cbb80a171bde8ca5e40c43b439603f5493","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T17:21:32Z","started_at":"2026-10-22T17:21:32Z","completed_at":"2026-10-22T17:41:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T17:54:15.163437463Z","timing_metadata_checked_at":"2026-10-22T17:54:15.163437463Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-22T17:54:15.168629046Z","region_metadata_checked_at":"2026-10-22T18:55:54.222618345Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9cf66df1dfe08833377dacfe8c64d5bd0f61099cf6bcfc0ba1c38d9aa9c5edd7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113319859344379904","occurred_at":"2026-10-22T17:21:32Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7696/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113319770949423104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7696,"pr_state":"open","pr_sha":"265b839d54b5faee78c6af1a6cc3183e15b16659","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T17:21:11Z","started_at":"2026-10-22T17:21:11Z","completed_at":"2026-10-22T19:33:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T19:48:50.898262254Z","timing_metadata_checked_at":"2026-10-22T19:48:50.898262254Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T19:48:50.908722066Z","region_metadata_checked_at":"2026-10-22T19:48:50.908722066Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ab3e76d409760b6ed853397525de8b89577d85acd584d7d9b2edc08b7220b9c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7696/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113319770949423104","occurred_at":"2026-10-22T17:21:11Z","failed_at":"e2e","signature_id":"96f835c54af083d55baeaf346ab65584abf7bcf116d908e62aac2a124faa4aac","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x18d647b87940\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8q2cclgqxhlk\" to finish creating: GET https://rp.j9423104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53\",\n \"name\": \"0d5e7b9b-6357-498c-88f8-d878e3a54b53\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T18:25:27.469650053Z\",\n \"endTime\": \"2026-10-22T18:44:36.112402686Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4 to exist\\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4 to exist\\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9423104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53\n correlation ID: de487052-0457-4f7a-86bf-23162f549db5\n client request ID: 8a9f42ff-db11-48e8-af3f-fae5e92dcff9\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x18d647b87940\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8q2cclgqxhlk\" to finish creating: GET https://rp.j9423104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53\",\n \"name\": \"0d5e7b9b-6357-498c-88f8-d878e3a54b53\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T18:25:27.469650053Z\",\n \"endTime\": \"2026-10-22T18:44:36.112402686Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4 to exist\\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4 to exist\\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9423104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53\n correlation ID: de487052-0457-4f7a-86bf-23162f549db5\n client request ID: 8a9f42ff-db11-48e8-af3f-fae5e92dcff9","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-twhbflrbjdzn\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7691/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113319722232582144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7691,"pr_state":"closed","pr_sha":"04fe40a8f6b4e30bc41dec49a5fd6015dceb708d","final_merged_sha":"04fe40a8f6b4e30bc41dec49a5fd6015dceb708d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T17:20:59Z","started_at":"2026-10-22T17:20:59Z","completed_at":"2026-10-22T19:44:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T19:59:05.267360908Z","timing_metadata_checked_at":"2026-10-22T19:59:05.267360908Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T19:59:05.288519056Z","region_metadata_checked_at":"2026-10-22T19:59:05.288519056Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113318213356883968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7598,"pr_state":"closed","pr_sha":"62341e172ca0c298be2d7b6050c97d16ad90d504","final_merged_sha":"be965db2bf14d58e471ed21952ec3e210e01ecba","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T17:15:00Z","started_at":"2026-10-22T17:15:00Z","completed_at":"2026-10-22T19:24:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T19:38:41.329197152Z","timing_metadata_checked_at":"2026-10-22T19:38:41.329197152Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T19:38:41.332548243Z","region_metadata_checked_at":"2026-10-22T19:38:41.332548243Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113315750507712512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7119,"pr_state":"closed","pr_sha":"cc0c5ef047b5bd0a2bf6e1ddc04dbd009721f1ba","final_merged_sha":"cc0c5ef047b5bd0a2bf6e1ddc04dbd009721f1ba","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T17:05:13Z","started_at":"2026-10-22T17:05:13Z","completed_at":"2026-10-22T19:29:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T19:38:41.301459321Z","timing_metadata_checked_at":"2026-10-22T19:38:41.301459321Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T19:38:41.304956312Z","region_metadata_checked_at":"2026-10-22T19:38:41.304956312Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ca4158a8c7242be8bd13b15f32b8c164a2b6c3adda6b3de965c31b4ac1ea7c50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113315750507712512","occurred_at":"2026-10-22T17:05:13Z","failed_at":"e2e","signature_id":"4df1aac1e0f3c92964f69b87100586a1ffec75be0df556e93f996e421e8d3ddf","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x12f37f64c920\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7h867b\" in resourcegroup=\"rg-np-version-upgrade-7h867b-ccdxwvgmcw5k\" to finish creating: GET https://rp.j7712512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68\",\n \"name\": \"800b48a8-8e04-4c8c-a76f-848043b4ca68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T18:35:17.000023517Z\",\n \"endTime\": \"2026-10-22T18:54:25.220281009Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7h867b-ccdxwvgmcw5k-np-upgrade-7h-d37091ba/providers/Microsoft.Compute/virtualMachines/m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7h867b-ccdxwvgmcw5k-np-upgrade-7h-d37091ba/providers/Microsoft.Compute/virtualMachines/m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7712512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68\n correlation ID: 4baff6fc-a012-46cd-a498-5faf00b70f7d\n client request ID: 6aa50b83-5102-4e98-a262-a0727625bcc5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x12f37f64c920\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7h867b\" in resourcegroup=\"rg-np-version-upgrade-7h867b-ccdxwvgmcw5k\" to finish creating: GET https://rp.j7712512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68\",\n \"name\": \"800b48a8-8e04-4c8c-a76f-848043b4ca68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T18:35:17.000023517Z\",\n \"endTime\": \"2026-10-22T18:54:25.220281009Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7h867b-ccdxwvgmcw5k-np-upgrade-7h-d37091ba/providers/Microsoft.Compute/virtualMachines/m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7h867b-ccdxwvgmcw5k-np-upgrade-7h-d37091ba/providers/Microsoft.Compute/virtualMachines/m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7712512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68\n correlation ID: 4baff6fc-a012-46cd-a498-5faf00b70f7d\n client request ID: 6aa50b83-5102-4e98-a262-a0727625bcc5\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7395/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113313496413245440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7395,"pr_state":"closed","pr_sha":"975e82b0f0c27ebfb8c30fbed634e639d48c6b92","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T16:56:15Z","started_at":"2026-10-22T16:56:15Z","completed_at":"2026-10-22T19:19:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T19:27:47.391986689Z","timing_metadata_checked_at":"2026-10-22T19:27:47.391986689Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T19:27:47.396197875Z","region_metadata_checked_at":"2026-10-22T19:27:47.396197875Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113307009825640448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7653,"pr_state":"closed","pr_sha":"76da4621827373d9267b68cfd55f1d91f45267d3","final_merged_sha":"1c5f99177052e49fb875d9503c65dd8f8ccd901f","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T16:30:28Z","started_at":"2026-10-22T16:30:28Z","completed_at":"2026-10-22T18:56:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T19:06:35.815164006Z","timing_metadata_checked_at":"2026-10-22T19:06:35.815164006Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T19:06:35.819608694Z","region_metadata_checked_at":"2026-10-22T19:06:35.819608694Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113305954912047104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7688,"pr_state":"open","pr_sha":"45d6c3fe0536cc334af902ba0fd530eaafa7ebc3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T16:26:17Z","started_at":"2026-10-22T16:26:17Z","completed_at":"2026-10-22T17:24:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T17:33:26.517734226Z","timing_metadata_checked_at":"2026-10-22T17:33:26.517734226Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T17:33:26.521467611Z","region_metadata_checked_at":"2026-10-22T17:33:26.521467611Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"64ae760673978f549faa89176731681b2166509956e987da8edf2403cc102995","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113305954912047104","occurred_at":"2026-10-22T16:26:17Z","failed_at":"alert","signature_id":"a400f34dd5d9c6546d3c51f97584489f4f8247fb2e77291e277aa95514d5afd7","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T17:16:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"multicluster-engine-operator-79c768d748-gtmch\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/multicluster-engine-operator-79c768d748-gtmch has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T17:17:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-58787656fc-vpngf\", region=\"westus3\", severity=\"warning\"\n Description: Pod mgmt-agent/mgmt-agent-58787656fc-vpngf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-22T17:17:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-applier\", pod=\"kube-applier-6d4c66b65c-cjgt4\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-applier/kube-applier-6d4c66b65c-cjgt4 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-22T17:17:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"secret-sync-controller\", pod=\"secrets-store-sync-controller-manager-7f4b9bc5df-pcxv2\", region=\"westus3\", severity=\"warning\"\n Description: Pod secret-sync-controller/secrets-store-sync-controller-manager-7f4b9bc5df-pcxv2 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-22T17:17:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"hypershift-sharedingress\", pod=\"router-65664496f6-j6f62\", region=\"westus3\", severity=\"warning\"\n Description: Pod hypershift-sharedingress/router-65664496f6-j6f62 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-22T17:18:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"ocm-webhook-68676899fd-plgpk\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/ocm-webhook-68676899fd-plgpk has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-22T17:18:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"managedcluster-import-controller-v2-6cb46794f7-kfk7p\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/managedcluster-import-controller-v2-6cb46794f7-kfk7p has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-22T17:18:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-58f4f6fb5b-8mhhk\", region=\"westus3\", severity=\"warning\"\n Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-58f4f6fb5b-8mhhk has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-22T17:18:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"ocm-controller-859484fc49-6spbd\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/ocm-controller-859484fc49-6spbd has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-22T17:18:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"cluster-manager-c5d6c587f-cx7xv\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/cluster-manager-c5d6c587f-cx7xv has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-22T17:19:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent\", pod=\"klusterlet-agent-54d5df574-f7jps\", region=\"westus3\", severity=\"warning\"\n Description: Pod open-cluster-management-agent/klusterlet-agent-54d5df574-f7jps has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-22T17:19:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2047104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent\", pod=\"klusterlet-9574f7d67-8mdfx\", region=\"westus3\", severity=\"warning\"\n Description: Pod open-cluster-management-agent/klusterlet-9574f7d67-8mdfx has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"a0a32a8d103c04e18ba3574c5977df491bb2ace7b4e7107ccb2268a99d149548","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113305954912047104","occurred_at":"2026-10-22T16:26:17Z","failed_at":"provision","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T17:11:01.934Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-22T17:11:02.617Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:72abeb657cfbdd6a5891fc102a16429875d70c5932ad5994524d0246db914013 imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:55a0ad895a8f10193bb070883194449f5252c057353887af1ba0767760a0d2f0 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-22T17:11:02.617Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-22T17:11:03.344Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-22T17:11:03.700Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-22T17:11:03.700Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-22T17:11:03.826Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-22T17:16:10.104Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-22T17:16:10.104Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113305872997289984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6823,"pr_state":"closed","pr_sha":"7675cd70bbd032c6518e23950b0738453ba0352f","final_merged_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T16:25:57Z","started_at":"2026-10-22T16:25:57Z","completed_at":"2026-10-22T18:45:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T18:55:53.971411696Z","timing_metadata_checked_at":"2026-10-22T18:55:53.971411696Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T18:55:53.974999588Z","region_metadata_checked_at":"2026-10-22T18:55:53.974999588Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0923e04fe68ffa16bba98f9a20efb8235b2070330c85e0098981eb05e7b88d69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113305872997289984","occurred_at":"2026-10-22T16:25:57Z","failed_at":"e2e","signature_id":"efbf32a99997756a2750ff3841eee191c8cd690170bd70992c2237917b3bee18","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x20c3ca48efe0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n err: \u003c*verifiers.pollObservationError | 0x20c3ca48ef80\u003e{\n observation: \u003c*errors.errorString | 0x20c3cb6d0a90\u003e{\n s: \"dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n },\n interruption: \u003c*fmt.wrapError | 0x20c3ca48ef60\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x20c3c94b2370\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x20c3ca48ef40\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x20c3cb422450\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x20c3ca48efe0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n err: \u003c*verifiers.pollObservationError | 0x20c3ca48ef80\u003e{\n observation: \u003c*errors.errorString | 0x20c3cb6d0a90\u003e{\n s: \"dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n },\n interruption: \u003c*fmt.wrapError | 0x20c3ca48ef60\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x20c3c94b2370\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x20c3ca48ef40\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x20c3cb422450\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e61311760ae7623cb7fd4cb9def8b7649d07be11756598455821961566fe69c5","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e61311760ae7623cb7fd4cb9def8b7649d07be11756598455821961566fe69c5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113305665899335680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7686,"pr_state":"closed","pr_sha":"b51dc9af4b3ad5d537fa203c98049fd2ba1496e3","final_merged_sha":"b51dc9af4b3ad5d537fa203c98049fd2ba1496e3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T16:25:08Z","started_at":"2026-10-22T16:25:08Z","completed_at":"2026-10-22T18:46:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T18:55:54.371901017Z","timing_metadata_checked_at":"2026-10-22T18:55:54.371901017Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T18:55:54.37500941Z","region_metadata_checked_at":"2026-10-22T18:55:54.37500941Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7674/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113303768312320000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7674,"pr_state":"closed","pr_sha":"d57ce0fbf902f2a6d5ca99616dd6ba10c4d5b134","final_merged_sha":"d57ce0fbf902f2a6d5ca99616dd6ba10c4d5b134","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T16:17:31Z","started_at":"2026-10-22T16:17:31Z","completed_at":"2026-10-22T18:40:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T18:45:49.72123761Z","timing_metadata_checked_at":"2026-10-22T18:45:49.72123761Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T18:45:49.724403501Z","region_metadata_checked_at":"2026-10-22T18:45:49.724403501Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7421/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113302477775310848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7421,"pr_state":"closed","pr_sha":"2155a16b17e047da9c0b48494c0961d4bfb921cc","final_merged_sha":"2155a16b17e047da9c0b48494c0961d4bfb921cc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T16:12:23Z","started_at":"2026-10-22T16:12:23Z","completed_at":"2026-10-22T18:46:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T18:55:54.113756184Z","timing_metadata_checked_at":"2026-10-22T18:55:54.113756184Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T18:55:54.116919477Z","region_metadata_checked_at":"2026-10-22T18:55:54.116919477Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113301360286896128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"closed","pr_sha":"5c4c6584c41117757b6f57f16a2d54abe386ea89","final_merged_sha":"6405f41e9d626eac01c2304a4b0fb1eefd6c1e81","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T16:08:02Z","started_at":"2026-10-22T16:08:02Z","completed_at":"2026-10-22T18:40:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T18:45:49.451538134Z","timing_metadata_checked_at":"2026-10-22T18:45:49.451538134Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T18:45:49.455694523Z","region_metadata_checked_at":"2026-10-22T18:45:49.455694523Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3d7795c52f2c57b0b407363f9466b46fd5a852dc1126b04ad33d6f7b50f4b55b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113301360286896128","occurred_at":"2026-10-22T16:08:02Z","failed_at":"e2e","signature_id":"3e9aaa4d1d813a4709137ed455a6a028fd10e68d2b9d24f9a1cbc8601d7c3757","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0x35ac7e2dd140\u003e: \n image pull not confirmed; cause undetermined (insufficient evidence)\n polling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n - pod pull-secret-test container pull-test: ContainerCreating ()\n {\n msg: \"image pull not confirmed; cause undetermined (insufficient evidence)\\npolling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n - pod pull-secret-test container pull-test: ContainerCreating ()\",\n err: \u003c*fmt.wrapError | 0x35ac7e2dd120\u003e{\n msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n - pod pull-secret-test container pull-test: ContainerCreating ()\",\n err: \u003c*errors.errorString | 0x35ac7fce7bf0\u003e{\n s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n - pod pull-secret-test container pull-test: ContainerCreating ()\",\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0x35ac7e2dd140\u003e: \n image pull not confirmed; cause undetermined (insufficient evidence)\n polling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n - pod pull-secret-test container pull-test: ContainerCreating ()\n {\n msg: \"image pull not confirmed; cause undetermined (insufficient evidence)\\npolling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n - pod pull-secret-test container pull-test: ContainerCreating ()\",\n err: \u003c*fmt.wrapError | 0x35ac7e2dd120\u003e{\n msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n - pod pull-secret-test container pull-test: ContainerCreating ()\",\n err: \u003c*errors.errorString | 0x35ac7fce7bf0\u003e{\n s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n - pod pull-secret-test container pull-test: ContainerCreating ()\",\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5561db6163db8e6282579645be22b666c3caa655cd5984b6509f9f92368e2bd6","failure_pattern":"image pull not confirmed; cause undetermined (insufficient evidence)","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["5561db6163db8e6282579645be22b666c3caa655cd5984b6509f9f92368e2bd6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113296902714822656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7611,"pr_state":"open","pr_sha":"d0b8c2115124d89310796a2231d4b9582d7762fc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T15:50:19Z","started_at":"2026-10-22T15:50:19Z","completed_at":"2026-10-22T17:48:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T18:04:41.508957005Z","timing_metadata_checked_at":"2026-10-22T18:04:41.508957005Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T18:04:41.512571195Z","region_metadata_checked_at":"2026-10-22T18:04:41.512571195Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113289616629960704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7646,"pr_state":"closed","pr_sha":"41c70366aac66b69d46eea04dd8464b994e38b2a","final_merged_sha":"41c70366aac66b69d46eea04dd8464b994e38b2a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T15:20:57Z","started_at":"2026-10-22T15:20:57Z","completed_at":"2026-10-22T17:54:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T18:04:41.731564191Z","timing_metadata_checked_at":"2026-10-22T18:04:41.731564191Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T18:04:41.76086321Z","region_metadata_checked_at":"2026-10-22T18:04:41.76086321Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1fe71aaf289e0bf329f0c9aa2e201f2adc9bc5144d1a7fc84c5045d4c58d399c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113289616629960704","occurred_at":"2026-10-22T15:20:57Z","failed_at":"e2e","signature_id":"fe24240d8784318f59cd835bc925e1e72ca0ec51e04dd27b76e1c9a75ad9c2f0","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xed2ec9e01c0\u003e: \n failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-989gmc\" resourcegroup=\"rg-np-version-upgrade-989gmc-8hflngjf7pjq\" to finish updating: GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\",\n \"name\": \"33eeff39-7b04-4af1-8912-9297ff607fdb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T16:42:14.996433941Z\",\n \"endTime\": \"2026-10-22T17:26:16.905655326Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool condition AllNodesHealthy is False: 1 of 4 machines are not healthy\\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; hypershift NodePool condition AllMachinesReady is False: 1 of 4 machines are not ready\\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\n correlation ID: 49293550-ec1c-44c3-83a4-ef44b94443ce\n client request ID: 143bee29-fe61-4aa3-b798-569832148329\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xed2ec9e01c0\u003e: \n failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-989gmc\" resourcegroup=\"rg-np-version-upgrade-989gmc-8hflngjf7pjq\" to finish updating: GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\",\n \"name\": \"33eeff39-7b04-4af1-8912-9297ff607fdb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T16:42:14.996433941Z\",\n \"endTime\": \"2026-10-22T17:26:16.905655326Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool condition AllNodesHealthy is False: 1 of 4 machines are not healthy\\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; hypershift NodePool condition AllMachinesReady is False: 1 of 4 machines are not ready\\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\n correlation ID: 49293550-ec1c-44c3-83a4-ef44b94443ce\n client request ID: 143bee29-fe61-4aa3-b798-569832148329\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2bd348e2f759f173403310693a20ab58f53f43ab8426ca2e577a35357f75f147","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; hypershift NodePool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2bd348e2f759f173403310693a20ab58f53f43ab8426ca2e577a35357f75f147"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7690/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113289300694011904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7690,"pr_state":"closed","pr_sha":"93bc62d45a30b98e68a0c6f1e066286e7a07a18b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T15:20:06Z","started_at":"2026-10-22T15:20:06Z","completed_at":"2026-10-22T17:26:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T17:33:26.937576482Z","timing_metadata_checked_at":"2026-10-22T17:33:26.937576482Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T17:33:26.942268964Z","region_metadata_checked_at":"2026-10-22T17:33:26.942268964Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"eb9fd131a542efd0ccc739c8f95ad37ea50bc22de2b9a1fe957017bb1969466a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7690/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113289300694011904","occurred_at":"2026-10-22T15:20:06Z","failed_at":"e2e","signature_id":"904e103fc4d240f7d05c2e2efb5e56be203c4cc87483a19c18cf88198cb1b1d6","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xf9bfe384ae0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xf9bfe384ae0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c1c8b39ba5d3ca3c70c30864a182ac9ff6f4361297590b1691f87bbb39aadd6d","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c1c8b39ba5d3ca3c70c30864a182ac9ff6f4361297590b1691f87bbb39aadd6d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113288746173468672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"closed","pr_sha":"9af097ada0a187a98f45aa51a8d8a1b962874a5c","final_merged_sha":"afd3417980c4b5052b54fdf377e648bf6df383fc","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T15:17:54Z","started_at":"2026-10-22T15:17:54Z","completed_at":"2026-10-22T17:32:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T17:43:33.127346269Z","timing_metadata_checked_at":"2026-10-22T17:43:33.127346269Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T17:43:33.130779961Z","region_metadata_checked_at":"2026-10-22T17:43:33.130779961Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113285901743296512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7608,"pr_state":"closed","pr_sha":"f8135fd806c5ededeeff856816d373186933c603","final_merged_sha":"f8135fd806c5ededeeff856816d373186933c603","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T15:06:36Z","started_at":"2026-10-22T15:06:36Z","completed_at":"2026-10-22T17:19:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T17:33:26.233857538Z","timing_metadata_checked_at":"2026-10-22T17:33:26.233857538Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T17:33:26.236874726Z","region_metadata_checked_at":"2026-10-22T17:33:26.236874726Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113285425358442496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7371,"pr_state":"closed","pr_sha":"a4597de5adbc5d597aaac059f210c327fcc93f8a","final_merged_sha":"42b177e82e37e27d732edd9dee43c759e521c0b1","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T15:04:42Z","started_at":"2026-10-22T15:04:42Z","completed_at":"2026-10-22T17:26:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T17:33:25.949973749Z","timing_metadata_checked_at":"2026-10-22T17:33:25.949973749Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T17:33:25.955729727Z","region_metadata_checked_at":"2026-10-22T17:33:25.955729727Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113283987379064832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"5db7c10ae1b400f88402e4f17ef0ac84aba3e9d1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T14:58:49Z","started_at":"2026-10-22T14:58:49Z","completed_at":"2026-10-22T17:35:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T17:43:33.283911117Z","timing_metadata_checked_at":"2026-10-22T17:43:33.283911117Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T17:43:33.28714301Z","region_metadata_checked_at":"2026-10-22T17:43:33.28714301Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"60b053400a4a6ce6fdcef8c76163c9b680f38e31127cc3c4e54e669e76ba1651","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113283987379064832","occurred_at":"2026-10-22T14:58:49Z","failed_at":"alert","signature_id":"6610592e09db876ec6c37c80a3d1988cc8a66321429ef0c56dab5208368f0124","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T17:10:49Z\n Ended: 2026-10-22T17:16:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/config-policy-controller is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"6dff6ce3e1d1a8832dee6c7d36fe51f5c7e35ea56d3c5cc6e72be9250d3e092d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113283987379064832","occurred_at":"2026-10-22T14:58:49Z","failed_at":"alert","signature_id":"609a4e6d360d5c5dfe5d49e536501c3a4adecf726fdbd6fbad2ca3bce77a4013","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 35 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T16:41:51Z\n Ended: 2026-10-22T16:47:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-22T16:41:51Z\n Ended: 2026-10-22T16:48:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-22T16:41:51Z\n Ended: 2026-10-22T16:48:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-22T16:41:51Z\n Ended: 2026-10-22T16:47:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-22T16:41:51Z\n Ended: 2026-10-22T16:47:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-22T16:41:51Z\n Ended: 2026-10-22T16:48:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-22T16:41:51Z\n Ended: 2026-10-22T16:48:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-22T16:41:51Z\n Ended: 2026-10-22T16:48:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-22T16:42:48Z\n Ended: 2026-10-22T16:49:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-22T16:42:48Z\n Ended: 2026-10-22T16:48:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-22T16:42:48Z\n Ended: 2026-10-22T16:49:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-22T16:42:48Z\n Ended: 2026-10-22T16:49:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-22T16:53:57Z\n Ended: 2026-10-22T16:59:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjuk4ke8vrnppho65r5l1u1hcmcf117\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjuk4ke8vrnppho65r5l1u1hcmcf117/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-22T16:56:49Z\n Ended: 2026-10-22T17:06:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-22T16:56:50Z\n Ended: 2026-10-22T17:06:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-22T16:56:50Z\n Ended: 2026-10-22T17:05:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-22T16:56:50Z\n Ended: 2026-10-22T17:05:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-22T16:56:50Z\n Ended: 2026-10-22T17:06:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-22T16:56:50Z\n Ended: 2026-10-22T17:06:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-22T17:01:50Z\n Ended: 2026-10-22T17:16:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-22T17:01:50Z\n Ended: 2026-10-22T17:16:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-22T17:01:50Z\n Ended: 2026-10-22T17:16:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-22T17:01:50Z\n Ended: 2026-10-22T17:15:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-22T17:01:50Z\n Ended: 2026-10-22T17:15:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-22T17:01:50Z\n Ended: 2026-10-22T17:16:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-22T17:02:47Z\n Ended: 2026-10-22T17:14:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-22T17:02:47Z\n Ended: 2026-10-22T17:14:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-22T17:02:47Z\n Ended: 2026-10-22T17:12:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-22T17:02:47Z\n Ended: 2026-10-22T17:14:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-22T17:02:47Z\n Ended: 2026-10-22T17:12:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-22T17:02:47Z\n Ended: 2026-10-22T17:14:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-22T17:03:49Z\n Ended: 2026-10-22T17:09:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-22T17:03:49Z\n Ended: 2026-10-22T17:09:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-22T17:03:49Z\n Ended: 2026-10-22T17:09:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-22T17:03:49Z\n Ended: 2026-10-22T17:09:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"c61067918d31054840258ce54e1938faef99f591c3d0df0b5ce5b35a0427dd5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113283987379064832","occurred_at":"2026-10-22T14:58:49Z","failed_at":"alert","signature_id":"b5976c7e247238d154539db68b0098f770efffac46ccd361fcbee9ff68b1cf23","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T16:20:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-q84cd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"33074520-cd79-4e68-90bd-146270a8dbf6\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-q84cd (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T16:25:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9064832-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-dtrgd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d46836a3-1737-4f96-a0d8-b3ef8af19f45\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-dtrgd (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-22T16:25:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9064832-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-q84cd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"33074520-cd79-4e68-90bd-146270a8dbf6\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-q84cd (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-22T16:28:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9064832-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-dtrgd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d46836a3-1737-4f96-a0d8-b3ef8af19f45\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-dtrgd (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113271747368194048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7688,"pr_state":"open","pr_sha":"471d85333d9f55e20b3f3c91bf8243d3f9943031","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T14:10:21Z","started_at":"2026-10-22T14:10:21Z","completed_at":"2026-10-22T14:22:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T14:27:57.6182923Z","timing_metadata_checked_at":"2026-10-22T14:27:57.6182923Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-22T14:27:57.622332185Z","region_metadata_checked_at":"2026-10-22T15:29:46.555124757Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9a2766b2989e8f08d41e6a5af7b6ffd058c2ed54d40b2f61be72d7351b6d3a58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113271747368194048","occurred_at":"2026-10-22T14:10:21Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113269162364112896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7340,"pr_state":"open","pr_sha":"dc3b53a1fbaf166578b115d916c6a425188524ee","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T14:00:01Z","started_at":"2026-10-22T14:00:01Z","completed_at":"2026-10-22T16:15:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T16:31:07.241438909Z","timing_metadata_checked_at":"2026-10-22T16:31:07.241438909Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T16:31:07.245476899Z","region_metadata_checked_at":"2026-10-22T16:31:07.245476899Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7685/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113265280208605184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7685,"pr_state":"closed","pr_sha":"134b977bc26f3d45efc2352cb21a7665bcb0a482","final_merged_sha":"134b977bc26f3d45efc2352cb21a7665bcb0a482","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T13:44:39Z","started_at":"2026-10-22T13:44:39Z","completed_at":"2026-10-22T15:46:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T16:00:17.645343379Z","timing_metadata_checked_at":"2026-10-22T16:00:17.645343379Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T16:00:17.648581972Z","region_metadata_checked_at":"2026-10-22T16:00:17.648581972Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113262158069370880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7653,"pr_state":"closed","pr_sha":"76da4621827373d9267b68cfd55f1d91f45267d3","final_merged_sha":"1c5f99177052e49fb875d9503c65dd8f8ccd901f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T13:32:15Z","started_at":"2026-10-22T13:32:15Z","completed_at":"2026-10-22T15:50:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T16:00:17.240372504Z","timing_metadata_checked_at":"2026-10-22T16:00:17.240372504Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T16:00:17.243616396Z","region_metadata_checked_at":"2026-10-22T16:00:17.243616396Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"30e8d8bae8a6002cd16bd692411769b2a9a9e3c2a952271fa37de90bc737d381","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113262158069370880","occurred_at":"2026-10-22T13:32:15Z","failed_at":"alert","signature_id":"1acc704dec9da24c5c7fa9f35ac9616ae74f220571f4a3427e7537e172296446","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T14:37:59Z\n Ended: 2026-10-22T15:06:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9370880-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjss4leeh7ul4ugbdhur3figtfoc2ie-t8i0b6j2d1r3s2j\", pod=\"router-85d447744d-ghchd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tjss4leeh7ul4ugbdhur3figtfoc2ie-t8i0b6j2d1r3s2j/router-85d447744d-ghchd has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"3297e4c6a077765011f3aeacec15a801b36cb66176e111fa3eb274bb09cc79c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113262158069370880","occurred_at":"2026-10-22T13:32:15Z","failed_at":"e2e","signature_id":"c38efd6ad70105db427bd0e6d9a57dde87e6596812ce842dbcdfefe00ee77962","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:179]: failed to capture node release images before upgrade\nUnexpected error:\n \u003c*fmt.wrapError | 0x390d19dbe00\u003e: \n list nodes (nodePool=npupgrade-4-20): Get \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\n {\n msg: \"list nodes (nodePool=npupgrade-4-20): Get \\\"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\\\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0x390d1c076e0\u003e{\n Op: \"Get\",\n URL: \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\",\n Err: \u003c*net.OpError | 0x390d1655400\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0x390d16552c0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:179]: failed to capture node release images before upgrade\nUnexpected error:\n \u003c*fmt.wrapError | 0x390d19dbe00\u003e: \n list nodes (nodePool=npupgrade-4-20): Get \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\n {\n msg: \"list nodes (nodePool=npupgrade-4-20): Get \\\"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\\\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0x390d1c076e0\u003e{\n Op: \"Get\",\n URL: \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\",\n Err: \u003c*net.OpError | 0x390d1655400\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0x390d16552c0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e20314db8c9a8fa7947e904463dad9423d8152bc1bcd1896b51f5df0a9fd524f","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"list nodes (nodePool=npupgrade-4-20): Get \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout"}},{"row_id":"f1e8c6d2d2697a5e896b766ddbd6dbffd1b356293836aab4c34e22bd0c80602e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113262158069370880","occurred_at":"2026-10-22T13:32:15Z","failed_at":"e2e","signature_id":"a9050521680fae4d294a8aeac103dcc74b0cc4f6524dabc513da8d4e5d2a6777","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-h9tmsmbgdtrf/cluster-candidate-4-22-c8mmrm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e5e1de42400\u003e: \n failed to create HCP cluster cluster-candidate-4-22-c8mmrm: failed waiting for cluster=\"cluster-candidate-4-22-c8mmrm\" in resourcegroup=\"rg-candidate-4-22-h9tmsmbgdtrf\" to finish creating: GET https://rp.j9370880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545\",\n \"name\": \"3c4c9b08-e05d-4f74-a38c-fd407e61a545\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:22:09.348889033Z\",\n \"endTime\": \"2026-10-22T14:51:16.010319539Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.19 is Partial (want Completed), started 21m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9370880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545\n correlation ID: 5f95ddbb-902c-4016-8fa5-c1736f7561ec\n client request ID: 47f590fe-4c8d-4f3b-bde5-687e697310cd\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-h9tmsmbgdtrf/cluster-candidate-4-22-c8mmrm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e5e1de42400\u003e: \n failed to create HCP cluster cluster-candidate-4-22-c8mmrm: failed waiting for cluster=\"cluster-candidate-4-22-c8mmrm\" in resourcegroup=\"rg-candidate-4-22-h9tmsmbgdtrf\" to finish creating: GET https://rp.j9370880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545\",\n \"name\": \"3c4c9b08-e05d-4f74-a38c-fd407e61a545\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:22:09.348889033Z\",\n \"endTime\": \"2026-10-22T14:51:16.010319539Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.19 is Partial (want Completed), started 21m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9370880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545\n correlation ID: 5f95ddbb-902c-4016-8fa5-c1736f7561ec\n client request ID: 47f590fe-4c8d-4f3b-bde5-687e697310cd\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","e20314db8c9a8fa7947e904463dad9423d8152bc1bcd1896b51f5df0a9fd524f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113260518343970816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7641,"pr_state":"closed","pr_sha":"4aac3114a556696faf4cfae98197999801b0c225","final_merged_sha":"4aac3114a556696faf4cfae98197999801b0c225","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T13:25:41Z","started_at":"2026-10-22T13:25:41Z","completed_at":"2026-10-22T15:45:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T16:00:16.907562668Z","timing_metadata_checked_at":"2026-10-22T16:00:16.907562668Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T16:00:16.911911158Z","region_metadata_checked_at":"2026-10-22T16:00:16.911911158Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113256572250492928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7633,"pr_state":"closed","pr_sha":"1b7d89e2b17f9288ea983c06d8e9ece5e53c383e","final_merged_sha":"1b7d89e2b17f9288ea983c06d8e9ece5e53c383e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T13:10:03Z","started_at":"2026-10-22T13:10:03Z","completed_at":"2026-10-22T15:25:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T15:29:46.454700299Z","timing_metadata_checked_at":"2026-10-22T15:29:46.454700299Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T15:29:46.460576185Z","region_metadata_checked_at":"2026-10-22T15:29:46.460576185Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"311b4401bb771e47647f014b8944a237fb1d01f51f350c1adf7e95b5fe68f175","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113256572250492928","occurred_at":"2026-10-22T13:10:03Z","failed_at":"alert","signature_id":"278d0b8c7c257830ef45bc0ace5ac701b95b0ade44e6e541f1c00a6415a0baaf","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 19 time(s) (1 unknown, 18 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T14:09:33Z\n Ended: 2026-10-22T14:38:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0492928-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjsfrjpd12fvkk41s1vdc57g83sl0d9-a8i0g4h2e2c7b8y\", pod=\"router-6cb766858b-xgt9t\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tjsfrjpd12fvkk41s1vdc57g83sl0d9-a8i0g4h2e2c7b8y/router-6cb766858b-xgt9t has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"8a82941c8e7ad6563c0a73facc93cc612f2e0794cfeaaec93ca51fb01510456a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113256572250492928","occurred_at":"2026-10-22T13:10:03Z","failed_at":"e2e","signature_id":"a5b2daf3dd240c9a4da33b2d49221615c7d48c11644f231e38b5041bfd0487d6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-bdn4zcvwkzhf/cluster-candidate-4-23-86tqbt should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x26f2228144e0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-86tqbt: failed waiting for cluster=\"cluster-candidate-4-23-86tqbt\" in resourcegroup=\"rg-candidate-4-23-bdn4zcvwkzhf\" to finish creating: GET https://rp.j0492928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca\",\n \"name\": \"d723b871-78c8-4adc-9137-f20dd790b9ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T13:55:51.795775898Z\",\n \"endTime\": \"2026-10-22T14:24:54.794749941Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 22m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0492928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca\n correlation ID: 134b5d7e-1756-4b7d-a04c-9d549c52db55\n client request ID: 73ca7188-a99d-4194-8ce5-029bf47125b1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-bdn4zcvwkzhf/cluster-candidate-4-23-86tqbt should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x26f2228144e0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-86tqbt: failed waiting for cluster=\"cluster-candidate-4-23-86tqbt\" in resourcegroup=\"rg-candidate-4-23-bdn4zcvwkzhf\" to finish creating: GET https://rp.j0492928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca\",\n \"name\": \"d723b871-78c8-4adc-9137-f20dd790b9ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T13:55:51.795775898Z\",\n \"endTime\": \"2026-10-22T14:24:54.794749941Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 22m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0492928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca\n correlation ID: 134b5d7e-1756-4b7d-a04c-9d549c52db55\n client request ID: 73ca7188-a99d-4194-8ce5-029bf47125b1\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254849540788224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7682,"pr_state":"closed","pr_sha":"fc2671b3b269a1655ed87c23d807979f5a82950f","final_merged_sha":"f1986ec811a1588158df61aabf80ee48537cf900","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T13:03:13Z","started_at":"2026-10-22T13:03:13Z","completed_at":"2026-10-22T14:10:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T14:16:58.053372996Z","timing_metadata_checked_at":"2026-10-22T14:16:58.053372996Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T14:16:58.057214587Z","region_metadata_checked_at":"2026-10-22T14:16:58.057214587Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"848d22bab9b83b737ddd91743715a55a4eeb22e07a8db10762e2076747114ef2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254849540788224","occurred_at":"2026-10-22T13:03:13Z","failed_at":"provision","signature_id":"6226765797caa992638fa67211f61137351872a74ebe70cc5200677de47ce171","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T13:58:37.009Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-10-22T13:58:37.626Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-10-22T13:58:37.626Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-22T13:58:37.853Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 namespace=multicluster-engine\ntime=2026-10-22T13:58:38.094Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 dryRun=false\ntime=2026-10-22T13:58:38.094Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-22T13:58:38.280Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-22T14:03:45.649Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 err=\"resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-10-22T14:03:45.650Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c2e7122f012fcda98e11e9f11c6eb72558008d2e2cc0835ce25a3969a080ede6","failure_pattern":"failed to roll out Helm release: resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded"}},{"row_id":"ce9582fce1d3d92e4fe10f4e3e18a9b6b54196e38ba2264fae62212cbb70104f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254849540788224","occurred_at":"2026-10-22T13:03:13Z","failed_at":"alert","signature_id":"51f0bcb0b5ece926328642189c37e30f72dcfc0519d66c414435e5df060a778f","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 22 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-54f9c547d8-j6wx5\", region=\"westus3\", severity=\"warning\"\n Description: Pod mgmt-agent/mgmt-agent-54f9c547d8-j6wx5 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-kzbt8\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-kzbt8 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"multicluster-engine-operator-79c768d748-rrgxh\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/multicluster-engine-operator-79c768d748-rrgxh has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-vmtzh\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-vmtzh has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-lk66c\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-lk66c has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"kube-applier\", pod=\"kube-applier-c5cf7d96-lrcd2\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-applier/kube-applier-c5cf7d96-lrcd2 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-tp6wq\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-tp6wq has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-gp8mb\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-gp8mb has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-2qnxq\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-2qnxq has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-8n4qd\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-8n4qd has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-6vxsk\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-6vxsk has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"secret-sync-controller\", pod=\"secrets-store-sync-controller-manager-7f4b9bc5df-h8csf\", region=\"westus3\", severity=\"warning\"\n Description: Pod secret-sync-controller/secrets-store-sync-controller-manager-7f4b9bc5df-h8csf has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-vgdgb\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-vgdgb has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-kwkqc\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-kwkqc has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-fh9lr\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-fh9lr has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-10-22T14:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-jbwkn\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-jbwkn has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-10-22T14:05:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"managedcluster-import-controller-v2-6cb46794f7-hqqwq\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/managedcluster-import-controller-v2-6cb46794f7-hqqwq has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Fired (not resolved)\n Started: 2026-10-22T14:05:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"ocm-webhook-68676899fd-srvpq\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/ocm-webhook-68676899fd-srvpq has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Fired (not resolved)\n Started: 2026-10-22T14:05:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"ocm-controller-859484fc49-hpp4f\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/ocm-controller-859484fc49-hpp4f has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Fired (not resolved)\n Started: 2026-10-22T14:05:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"cluster-manager-c5d6c587f-nf87h\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/cluster-manager-c5d6c587f-nf87h has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Fired (not resolved)\n Started: 2026-10-22T14:05:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"cluster-manager-c5d6c587f-xw9f9\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/cluster-manager-c5d6c587f-xw9f9 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Fired (not resolved)\n Started: 2026-10-22T14:05:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0788224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"clusterlifecycle-state-metrics-v2-7bb4664cf8-trs6j\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/clusterlifecycle-state-metrics-v2-7bb4664cf8-trs6j has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","c2e7122f012fcda98e11e9f11c6eb72558008d2e2cc0835ce25a3969a080ede6"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254479959691264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7133,"pr_state":"closed","pr_sha":"45037000851deb39176b3b62f286f06464c87985","final_merged_sha":"45037000851deb39176b3b62f286f06464c87985","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T13:01:44Z","started_at":"2026-10-22T13:01:44Z","completed_at":"2026-10-22T15:31:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T15:50:01.841161307Z","timing_metadata_checked_at":"2026-10-22T15:50:01.841161307Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T15:50:01.845639469Z","region_metadata_checked_at":"2026-10-22T15:50:01.845639469Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"af15257b998da57b8e72d4e1cc23044a0ae898ba881483a2c639690ed36f8fe5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254479959691264","occurred_at":"2026-10-22T13:01:44Z","failed_at":"alert","signature_id":"5f46b56ac477343a6a11acbdfacf4c4c593c390c995318b85beadcc73d4ed5ba","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T14:40:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T14:41:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"f2ef8938b3db8ced99192e30994629e1912a04f7dcc943b7dcedf056c72f0d80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254479959691264","occurred_at":"2026-10-22T13:01:44Z","failed_at":"alert","signature_id":"8b832a372ecfb2ace8d8d9be81e7f2b7575ca31564d8dc52d41ae47c9b508996","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 11 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T14:30:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T14:31:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-22T14:41:37Z\n Ended: 2026-10-22T14:47:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"klusterlet-2tjsln0321cqtvn8430ocpgltipiguvb\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjsln0321cqtvn8430ocpgltipiguvb/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-22T14:41:37Z\n Ended: 2026-10-22T14:47:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"klusterlet-2tjsln0321cqtvn8430ocpgltipiguvb\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjsln0321cqtvn8430ocpgltipiguvb/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-22T14:57:36Z\n Ended: 2026-10-22T15:06:35Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"klusterlet-2tjsu1de5klk2l9lntbe3b6lp63khc9q\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjsu1de5klk2l9lntbe3b6lp63khc9q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-22T14:58:40Z\n Ended: 2026-10-22T15:06:35Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"klusterlet-2tjsu1de5klk2l9lntbe3b6lp63khc9q\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjsu1de5klk2l9lntbe3b6lp63khc9q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-22T15:00:34Z\n Ended: 2026-10-22T15:06:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"klusterlet-2tjsuurcgq00jdskheubnv1t28341vhu\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjsuurcgq00jdskheubnv1t28341vhu/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-22T15:01:34Z\n Ended: 2026-10-22T15:10:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"klusterlet-2tjsvjhboesm03krfvrue116vlqlgtfk\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjsvjhboesm03krfvrue116vlqlgtfk/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-22T15:01:34Z\n Ended: 2026-10-22T15:10:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"klusterlet-2tjsvjhboesm03krfvrue116vlqlgtfk\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjsvjhboesm03krfvrue116vlqlgtfk/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-22T15:03:36Z\n Ended: 2026-10-22T15:11:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"klusterlet-2tjt0i26iaqpuk2cotnfqoqva7ea5k9q\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjt0i26iaqpuk2cotnfqoqva7ea5k9q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-22T15:03:36Z\n Ended: 2026-10-22T15:11:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9691264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264\", namespace=\"klusterlet-2tjt0i26iaqpuk2cotnfqoqva7ea5k9q\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjt0i26iaqpuk2cotnfqoqva7ea5k9q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254320458698752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7548,"pr_state":"open","pr_sha":"02e2c03a9a9e59fe826b308583a1ed6de8b6645a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T13:01:04Z","started_at":"2026-10-22T13:01:04Z","completed_at":"2026-10-22T15:30:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T15:50:01.975940576Z","timing_metadata_checked_at":"2026-10-22T15:50:01.975940576Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T15:50:01.979534146Z","region_metadata_checked_at":"2026-10-22T15:50:01.979534146Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8990b1aec91d465234578646ca9b08126c8b7f77077167705d756b89d26860fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254320458698752","occurred_at":"2026-10-22T13:01:04Z","failed_at":"e2e","signature_id":"f11e42bee52762323ac7f2a29ccc1f770e75b1825e14bc3e92057295a1ae7fb3","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x2fdcc5c1a920\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-jhgf55\" in resourcegroup=\"rg-np-noedge-jhgf55-jn7vd7sqrbwj\" to finish creating: GET https://rp.j8698752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2\",\n \"name\": \"053b6bb5-afd0-4ef3-ba63-cc853312d8d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:32:42.03798521Z\",\n \"endTime\": \"2026-10-22T14:51:43.440263423Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v6g6m6x8l5j1c5f-npnoedge-4-20-mrthg-lqgf9: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v6g6m6x8l5j1c5f-npnoedge-4-20-mrthg-lqgf9: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8698752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2\n correlation ID: 71832900-0fbb-4552-a679-c7de683bfa6e\n client request ID: 82d54b68-f5bf-46a3-9e65-10064ff56089\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x2fdcc5c1a920\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-jhgf55\" in resourcegroup=\"rg-np-noedge-jhgf55-jn7vd7sqrbwj\" to finish creating: GET https://rp.j8698752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2\",\n \"name\": \"053b6bb5-afd0-4ef3-ba63-cc853312d8d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:32:42.03798521Z\",\n \"endTime\": \"2026-10-22T14:51:43.440263423Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v6g6m6x8l5j1c5f-npnoedge-4-20-mrthg-lqgf9: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v6g6m6x8l5j1c5f-npnoedge-4-20-mrthg-lqgf9: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8698752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2\n correlation ID: 71832900-0fbb-4552-a679-c7de683bfa6e\n client request ID: 82d54b68-f5bf-46a3-9e65-10064ff56089\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7686,"pr_state":"closed","pr_sha":"b51dc9af4b3ad5d537fa203c98049fd2ba1496e3","final_merged_sha":"b51dc9af4b3ad5d537fa203c98049fd2ba1496e3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T12:57:54Z","started_at":"2026-10-22T12:57:54Z","completed_at":"2026-10-22T15:34:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T15:50:01.998259289Z","timing_metadata_checked_at":"2026-10-22T15:50:01.998259289Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T15:50:02.001034065Z","region_metadata_checked_at":"2026-10-22T15:50:02.001034065Z"},"failed_at":["e2e"],"failed_test_count":7,"occurrences":[{"row_id":"070f5d19e020fbebe21c93a00329253e552583b6c9ec4e0221bb71ea148442f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","failed_at":"e2e","signature_id":"21381fe06efc73c3442517b08384543c00261e1938f3ca66dc65ef940b6459da","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x2a775c43e180\u003e: \n resource group \"complex-cilium-kv-8hx8sgkq9px6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8hx8sgkq9px6\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\",\n \"name\": \"28cf5da8-4bf8-49ef-8a24-44a6ab872d73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:21:07.679040667Z\",\n \"endTime\": \"2026-10-22T14:45:09.479893276Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsi0li3c3ivjdhg5u6etupj5flrs9m still exists (deletion dispatched at 2026-10-22T14:21:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\n correlation ID: e13b5021-89b3-4639-913d-d5270a5dc946\n client request ID: 15895da4-10f4-47b9-b08f-39becbd952c2\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x2a775c43e180\u003e: \n resource group \"complex-cilium-kv-8hx8sgkq9px6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8hx8sgkq9px6\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\",\n \"name\": \"28cf5da8-4bf8-49ef-8a24-44a6ab872d73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:21:07.679040667Z\",\n \"endTime\": \"2026-10-22T14:45:09.479893276Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsi0li3c3ivjdhg5u6etupj5flrs9m still exists (deletion dispatched at 2026-10-22T14:21:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\n correlation ID: e13b5021-89b3-4639-913d-d5270a5dc946\n client request ID: 15895da4-10f4-47b9-b08f-39becbd952c2\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"274ea668badb049d3c116767e5c35a7da203707b1ec1d5ff2c7c83e84ec45a41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","failed_at":"e2e","signature_id":"bd7bb04cbc3d1b80cca9e3082743d7f6fb56d7eba3aaf104da4eae35eb46e10c","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x27b60e3061f8\u003e: \n resource group \"pullsecret-test-2qfk9tbgwjpl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2qfk9tbgwjpl\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\",\n \"name\": \"0e34d898-c038-4aae-864f-7bc35e33d52f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:22:39.714077689Z\",\n \"endTime\": \"2026-10-22T14:46:39.512324074Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6m9stttuajn7f257929ebjijis still exists (deletion dispatched at 2026-10-22T14:22:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\n correlation ID: fdd89538-540d-4f74-94cb-acf48e5dcf5d\n client request ID: f1af71fe-def5-470f-a1f8-0006a3290e20\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x27b60e3061f8\u003e: \n resource group \"pullsecret-test-2qfk9tbgwjpl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2qfk9tbgwjpl\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\",\n \"name\": \"0e34d898-c038-4aae-864f-7bc35e33d52f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:22:39.714077689Z\",\n \"endTime\": \"2026-10-22T14:46:39.512324074Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6m9stttuajn7f257929ebjijis still exists (deletion dispatched at 2026-10-22T14:22:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\n correlation ID: fdd89538-540d-4f74-94cb-acf48e5dcf5d\n client request ID: f1af71fe-def5-470f-a1f8-0006a3290e20\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6ea6b73dce393d75f88cefb834bea45c7fb434b3a27e97ff165a82d7c9b54d31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","failed_at":"e2e","signature_id":"aeb431818b5c6b442b52161098d914e998eb3001d4b357dac24274924a88a237","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x14030b5a2150\u003e: \n resource group \"arm64-vm-cluster-mkmhd5qnk9lx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mkmhd5qnk9lx\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c\",\n \"name\": \"4a8e901d-725b-4e43-a6ab-53b69527c98c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:16:46.278169996Z\",\n \"endTime\": \"2026-10-22T14:40:50.034223752Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshbm80e288ae77lmgjasg7lrkdr39 still exists (deletion dispatched at 2026-10-22T14:16:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c\n correlation ID: 75809158-6027-4075-b7d1-2543b49090e7\n client request ID: 76f25ad0-6660-420c-abae-8208cba3c40f\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x14030b5a2150\u003e: \n resource group \"arm64-vm-cluster-mkmhd5qnk9lx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mkmhd5qnk9lx\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c\",\n \"name\": \"4a8e901d-725b-4e43-a6ab-53b69527c98c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:16:46.278169996Z\",\n \"endTime\": \"2026-10-22T14:40:50.034223752Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshbm80e288ae77lmgjasg7lrkdr39 still exists (deletion dispatched at 2026-10-22T14:16:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c\n correlation ID: 75809158-6027-4075-b7d1-2543b49090e7\n client request ID: 76f25ad0-6660-420c-abae-8208cba3c40f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7594b2f850ed1c75d596f21627ea655894c27cb935228133bae61a1298637ae8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","failed_at":"e2e","signature_id":"9ee611bb2a3e02c3800fcd63a0f4ec2bc6c1c6da5f8de8fd0a89b2b935289a9e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x2ed9255f4018\u003e: \n resource group \"rg-candidate-4-22-f565h2jdmqgh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-4j4s8h\" in resourcegroup=\"rg-candidate-4-22-f565h2jdmqgh\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\",\n \"name\": \"837480d6-dd8d-4baa-bb95-0329fd032c3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:27:18.140344478Z\",\n \"endTime\": \"2026-10-22T14:51:19.380950943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6vlktl4mo3ik8bbsikf69bgmlu still exists (deletion dispatched at 2026-10-22T14:27:21Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\n correlation ID: c5cd1b4c-a04e-4f40-9b2c-1eb227504489\n client request ID: 3fa15ac4-aadc-4a06-b7db-144805d68e5c\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x2ed9255f4018\u003e: \n resource group \"rg-candidate-4-22-f565h2jdmqgh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-4j4s8h\" in resourcegroup=\"rg-candidate-4-22-f565h2jdmqgh\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\",\n \"name\": \"837480d6-dd8d-4baa-bb95-0329fd032c3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:27:18.140344478Z\",\n \"endTime\": \"2026-10-22T14:51:19.380950943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6vlktl4mo3ik8bbsikf69bgmlu still exists (deletion dispatched at 2026-10-22T14:27:21Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\n correlation ID: c5cd1b4c-a04e-4f40-9b2c-1eb227504489\n client request ID: 3fa15ac4-aadc-4a06-b7db-144805d68e5c\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8988a3af0c7241fa90e732f73c769b06ece706c479bf91076a6443400512c483","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","failed_at":"e2e","signature_id":"9470a39f6b3757be7a80681fe7b8ac52c6881c846eff19ecc805839f186e8123","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x1661e2326120\u003e: \n resource group \"external-auth-cluster-km6bz4ns8fzk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-km6bz4ns8fzk\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\",\n \"name\": \"cc17306c-6d65-4b24-bd6f-a1392d064e72\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:27:28.417107798Z\",\n \"endTime\": \"2026-10-22T14:51:29.402583281Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshs28plfeftpnhlpn245qqpnk9ut6 still exists (deletion dispatched at 2026-10-22T14:27:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\n correlation ID: 2d284687-9204-4517-b640-41ab53fbee0b\n client request ID: 9ef671a3-662c-492d-a9df-693795b1e91a\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x1661e2326120\u003e: \n resource group \"external-auth-cluster-km6bz4ns8fzk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-km6bz4ns8fzk\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\",\n \"name\": \"cc17306c-6d65-4b24-bd6f-a1392d064e72\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:27:28.417107798Z\",\n \"endTime\": \"2026-10-22T14:51:29.402583281Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshs28plfeftpnhlpn245qqpnk9ut6 still exists (deletion dispatched at 2026-10-22T14:27:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\n correlation ID: 2d284687-9204-4517-b640-41ab53fbee0b\n client request ID: 9ef671a3-662c-492d-a9df-693795b1e91a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9f7e07be8465bcdcee2a2e74cde2d559125d9970b9d5e295014e8ba84bd57c82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","failed_at":"e2e","signature_id":"a420d5a9e3f855f48c2651db4f74b1059d2169fa2a73c735b46816a88994b8a8","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x291accb9c480\u003e: \n resource group \"rg-cluster-back-version-n5m774zrgzv9\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x291acc2cd520\u003e{\n msg: \"resource group \\\"rg-cluster-back-version-n5m774zrgzv9\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x291acc2cd500\u003e{\n msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n err: \u003c*fmt.wrapErrors | 0x291acd993a70\u003e{\n msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n errs: [\n \u003c*fmt.wrapErrors | 0x291acd993a40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x291acd69fed0\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n \u003c*fmt.wrapErrors | 0x291acd993a40\u003e0x291acd993a40 (cyclic reference),\n ],\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x291accb9c480\u003e: \n resource group \"rg-cluster-back-version-n5m774zrgzv9\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x291acc2cd520\u003e{\n msg: \"resource group \\\"rg-cluster-back-version-n5m774zrgzv9\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x291acc2cd500\u003e{\n msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n err: \u003c*fmt.wrapErrors | 0x291acd993a70\u003e{\n msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n errs: [\n \u003c*fmt.wrapErrors | 0x291acd993a40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x291acd69fed0\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n \u003c*fmt.wrapErrors | 0x291acd993a40\u003e0x291acd993a40 (cyclic reference),\n ],\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"resource group \"rg-cluster-back-version-n5m774zrgzv9\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded"}},{"row_id":"e292fd48115eb22dda731afbcb1c1ed7cc3588212a98ca2d0488d3ef0d1272fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","failed_at":"e2e","signature_id":"78240a56007cbdb142d1fe54126202e27b95da39d24a277343ee78126e9f1890","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x239068445020\u003e: \n resource group \"cni-cilium-w6t5xjcmq5lh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-w6t5xjcmq5lh\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\",\n \"name\": \"47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:19:53.04024209Z\",\n \"endTime\": \"2026-10-22T14:43:59.86158375Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshch4bfh839ftim6rnschfdun3698 still exists (deletion dispatched at 2026-10-22T14:19:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\n correlation ID: 7aea69f5-9144-48be-be55-309aae2de7a4\n client request ID: db330e23-e554-4902-a60c-dc62e09132f3\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x239068445020\u003e: \n resource group \"cni-cilium-w6t5xjcmq5lh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-w6t5xjcmq5lh\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\",\n \"name\": \"47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T14:19:53.04024209Z\",\n \"endTime\": \"2026-10-22T14:43:59.86158375Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshch4bfh839ftim6rnschfdun3698 still exists (deletion dispatched at 2026-10-22T14:19:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\n correlation ID: 7aea69f5-9144-48be-be55-309aae2de7a4\n client request ID: db330e23-e554-4902-a60c-dc62e09132f3\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113251417044029440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7547,"pr_state":"closed","pr_sha":"555992cf0ab38905574fa94b8fcfb2f04415f733","final_merged_sha":"cd3c21e53a196ba0ad2c38b6f73f33b744c9e3fa","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T12:49:34Z","started_at":"2026-10-22T12:49:34Z","completed_at":"2026-10-22T14:50:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T14:59:48.408118257Z","timing_metadata_checked_at":"2026-10-22T14:59:48.408118257Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T14:59:48.414193929Z","region_metadata_checked_at":"2026-10-22T14:59:48.414193929Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f90627aa2779c37b2e4fa025d7fdd17b0aa86e13e633f1782b85923c4d30db96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113251417044029440","occurred_at":"2026-10-22T12:49:34Z","failed_at":"e2e","signature_id":"1925c4ff43c7c8127f497ba14b75cd670537fb5f70933201f320fd96cbf2c0c3","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1bbfb1338400\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1bbfb13383e0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1bbfb092c1e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1bbfb13383c0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1bbfb280d260\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1bbfb1338400\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1bbfb13383e0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1bbfb092c1e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1bbfb13383c0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1bbfb280d260\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7680/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113251396953313280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7680,"pr_state":"closed","pr_sha":"e61dfe1b9de4be05a37bdfb4f61dff922b3f272b","final_merged_sha":"e61dfe1b9de4be05a37bdfb4f61dff922b3f272b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T12:49:29Z","started_at":"2026-10-22T12:49:29Z","completed_at":"2026-10-22T15:02:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T15:10:10.538058829Z","timing_metadata_checked_at":"2026-10-22T15:10:10.538058829Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T15:10:10.541827518Z","region_metadata_checked_at":"2026-10-22T15:10:10.541827518Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4aa2fadaee17b96f455078d32b910fa2ebf428f29d80fcf2a1e0b4930934a260","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7680/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113251396953313280","occurred_at":"2026-10-22T12:49:29Z","failed_at":"e2e","signature_id":"0d53882633db1d5048190df3d0bfbb3d29e675e5960c5c44e74bd8e56f0ba890","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-2m9m9h\" for cluster \"cluster-candidate-4-21-2m9m9h\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x27893fb0e000\u003e: \n failed to create NodePool np-2m9m9h: failed waiting for nodepool=\"np-2m9m9h\" for cluster \"cluster-candidate-4-21-2m9m9h\" in resourcegroup=\"rg-candidate-4-21-6lszx59zjddq\" to finish creating: GET https://rp.j3313280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb\",\n \"name\": \"f57af8c9-c3cf-4aea-87b5-7b743fe62efb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T13:54:06.562943785Z\",\n \"endTime\": \"2026-10-22T14:13:14.924879335Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6lszx59zjddq-candidate-2m9m9h--managed/providers/Microsoft.Compute/virtualMachines/u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6lszx59zjddq-candidate-2m9m9h--managed/providers/Microsoft.Compute/virtualMachines/u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3313280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb\n correlation ID: b876de21-3542-4a1c-bc13-5b0b17d4888a\n client request ID: f374bcdf-2d88-4b72-b317-4d2ad1fe477a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-2m9m9h\" for cluster \"cluster-candidate-4-21-2m9m9h\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x27893fb0e000\u003e: \n failed to create NodePool np-2m9m9h: failed waiting for nodepool=\"np-2m9m9h\" for cluster \"cluster-candidate-4-21-2m9m9h\" in resourcegroup=\"rg-candidate-4-21-6lszx59zjddq\" to finish creating: GET https://rp.j3313280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb\",\n \"name\": \"f57af8c9-c3cf-4aea-87b5-7b743fe62efb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T13:54:06.562943785Z\",\n \"endTime\": \"2026-10-22T14:13:14.924879335Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6lszx59zjddq-candidate-2m9m9h--managed/providers/Microsoft.Compute/virtualMachines/u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6lszx59zjddq-candidate-2m9m9h--managed/providers/Microsoft.Compute/virtualMachines/u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3313280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb\n correlation ID: b876de21-3542-4a1c-bc13-5b0b17d4888a\n client request ID: f374bcdf-2d88-4b72-b317-4d2ad1fe477a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7684/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113243453683929088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7684,"pr_state":"closed","pr_sha":"4df4b482f7006c3bcd247c54d1d80bf312ffc310","final_merged_sha":"4df4b482f7006c3bcd247c54d1d80bf312ffc310","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T12:17:56Z","started_at":"2026-10-22T12:17:56Z","completed_at":"2026-10-22T15:00:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T15:10:10.88462137Z","timing_metadata_checked_at":"2026-10-22T15:10:10.88462137Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T15:10:10.892453548Z","region_metadata_checked_at":"2026-10-22T15:10:10.892453548Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113240554585198592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7687,"pr_state":"open","pr_sha":"ab2787fc652bae7ace1a0d29de076ca0b076b00b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T12:06:24Z","started_at":"2026-10-22T12:06:24Z","completed_at":"2026-10-22T14:38:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T14:48:58.613821833Z","timing_metadata_checked_at":"2026-10-22T14:48:58.613821833Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T14:48:58.620545316Z","region_metadata_checked_at":"2026-10-22T14:48:58.620545316Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5c97db0ecc9c6e58c34f2e8151317c6d3536cfe80d9981c0dbb31ea18a0fc6a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113240554585198592","occurred_at":"2026-10-22T12:06:24Z","failed_at":"e2e","signature_id":"0c413ba7400d39619499c18f717f654ad4d24d7d3d25c4fd556162f570f4d3c4","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1971c8790ec0\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-qg2q7d744n2t\" to finish creating: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-qg2q7d744n2t\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1971c8790ec0\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-qg2q7d744n2t\" to finish creating: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-qg2q7d744n2t\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-qg2q7d744n2t\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113238110056747008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7683,"pr_state":"closed","pr_sha":"b2049d610b0f44f4441e4e06f0dfd134505df89d","final_merged_sha":"b2049d610b0f44f4441e4e06f0dfd134505df89d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T11:56:42Z","started_at":"2026-10-22T11:56:42Z","completed_at":"2026-10-22T14:44:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T14:48:58.441073381Z","timing_metadata_checked_at":"2026-10-22T14:48:58.441073381Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T14:48:58.446178368Z","region_metadata_checked_at":"2026-10-22T14:48:58.446178368Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113228155324993536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7641,"pr_state":"closed","pr_sha":"4aac3114a556696faf4cfae98197999801b0c225","final_merged_sha":"4aac3114a556696faf4cfae98197999801b0c225","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T11:17:08Z","started_at":"2026-10-22T11:17:08Z","completed_at":"2026-10-22T12:17:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T12:31:59.103657805Z","timing_metadata_checked_at":"2026-10-22T12:31:59.103657805Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T12:31:59.109243592Z","region_metadata_checked_at":"2026-10-22T12:31:59.109243592Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"57b12c511d206dc8d91c6f3cdab2cbe7122b109c0edf29c00a9330d21276a17b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113228155324993536","occurred_at":"2026-10-22T11:17:08Z","failed_at":"alert","signature_id":"447150ec04c57cb895574cbf66f0db98c26d67b4a9ca7846df88f7bff32541fa","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T12:11:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4993536-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-applier\", pod=\"kube-applier-695c66cfc4-vx26j\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-applier/kube-applier-695c66cfc4-vx26j has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T12:11:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4993536-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"multicluster-engine-operator-79c768d748-8qq8b\", region=\"westus3\", severity=\"warning\"\n Description: Pod multicluster-engine/multicluster-engine-operator-79c768d748-8qq8b has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-22T12:11:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4993536-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-86dc97764f-kp7qm\", region=\"westus3\", severity=\"warning\"\n Description: Pod mgmt-agent/mgmt-agent-86dc97764f-kp7qm has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-22T12:11:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4993536-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"secret-sync-controller\", pod=\"secrets-store-sync-controller-manager-7f4b9bc5df-gtp8m\", region=\"westus3\", severity=\"warning\"\n Description: Pod secret-sync-controller/secrets-store-sync-controller-manager-7f4b9bc5df-gtp8m has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"69d7388fca825453d928912aaf57e385cf34cc0fea325c789c3b1306915d8b7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113228155324993536","occurred_at":"2026-10-22T11:17:08Z","failed_at":"provision","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T12:05:37.122Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-22T12:05:37.861Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:72abeb657cfbdd6a5891fc102a16429875d70c5932ad5994524d0246db914013 imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:55a0ad895a8f10193bb070883194449f5252c057353887af1ba0767760a0d2f0 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-22T12:05:37.861Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-22T12:05:38.038Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-22T12:05:38.279Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-22T12:05:38.279Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-22T12:05:38.409Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-22T12:10:45.336Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-22T12:10:45.336Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113227265989939200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7665,"pr_state":"closed","pr_sha":"983cfba68b2c170cc433573b5d9c0c72f2a1405a","final_merged_sha":"b40bc5cbb80a171bde8ca5e40c43b439603f5493","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T11:13:36Z","started_at":"2026-10-22T11:13:36Z","completed_at":"2026-10-22T13:27:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T13:35:03.307119884Z","timing_metadata_checked_at":"2026-10-22T13:35:03.307119884Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T13:35:03.310957574Z","region_metadata_checked_at":"2026-10-22T13:35:03.310957574Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"8202fd57b590099bf42267bedb3e3c3330dd03b9f65864043d25a49b8ef35c07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113227265989939200","occurred_at":"2026-10-22T11:13:36Z","failed_at":"e2e","signature_id":"8de204c51ea1e77126cc08101aced92a4c4f1946ec3be586cda6f8a7ac4a26df","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3fcfb02035d0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3fcfb02035d0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"729ad653b1a4e1df171b5384e8c8a7d6ecb54a27a28c07489437e4661c195409","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163"}},{"row_id":"ad921fab8f9fdf601da09dda9626487a2e937faabf3c455335c17e0c7a43d726","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113227265989939200","occurred_at":"2026-10-22T11:13:36Z","failed_at":"e2e","signature_id":"7ce7ec004ca06c2f4f394ab96fd260d7f8d5523221c536dc3b5faf13439d18c9","test_name":"Engineering want metrics from SVC and MGMT clusters to be present in Azure Monitor WOrkspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.\nnot all expected service metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:\nexpected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: []\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.\nnot all expected service metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:\nexpected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: []\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b1d68134ee1583cdf3cf29ccf5f52a00bbda8ef7bbf33a93cfb29ce106c7b3bb","failure_pattern":"not all expected service metrics appeared in Azure Monitor","search_query":"not all expected service metrics appeared in Azure Monitor"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["729ad653b1a4e1df171b5384e8c8a7d6ecb54a27a28c07489437e4661c195409","b1d68134ee1583cdf3cf29ccf5f52a00bbda8ef7bbf33a93cfb29ce106c7b3bb"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113223180033200128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7073,"pr_state":"closed","pr_sha":"71040fb7c1d7f90960c4e22f55cbcc4df121eb18","final_merged_sha":"9c4815df62c125480cb9a6a711a2a7ca92f94b47","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T10:57:22Z","started_at":"2026-10-22T10:57:22Z","completed_at":"2026-10-22T13:09:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T13:14:19.063881626Z","timing_metadata_checked_at":"2026-10-22T13:14:19.063881626Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T13:14:19.067700016Z","region_metadata_checked_at":"2026-10-22T13:14:19.067700016Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113222453214842880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7679,"pr_state":"closed","pr_sha":"9ad11c04ca7d2bf65a7a65ecca3176ba5f3912f9","final_merged_sha":"aef55fb48491e559a7a2689dd16c32b57b3baced","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T10:54:29Z","started_at":"2026-10-22T10:54:29Z","completed_at":"2026-10-22T13:11:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T13:14:19.215664304Z","timing_metadata_checked_at":"2026-10-22T13:14:19.215664304Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T13:14:19.218814995Z","region_metadata_checked_at":"2026-10-22T13:14:19.218814995Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113216309620641792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7598,"pr_state":"closed","pr_sha":"62341e172ca0c298be2d7b6050c97d16ad90d504","final_merged_sha":"be965db2bf14d58e471ed21952ec3e210e01ecba","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T10:30:03Z","started_at":"2026-10-22T10:30:03Z","completed_at":"2026-10-22T13:06:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T13:14:19.084507569Z","timing_metadata_checked_at":"2026-10-22T13:14:19.084507569Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T13:14:19.087244561Z","region_metadata_checked_at":"2026-10-22T13:14:19.087244561Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"85d8a1162b9cf5b09aa1892393b057b437ad72817ce387c8a39d4113e971e03c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113216309620641792","occurred_at":"2026-10-22T10:30:03Z","failed_at":"e2e","signature_id":"4efbb868e90e0c981d31b779e618a262978897a74c8152d02034b9b5aced9b10","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xfc12762a440\u003e: \n DNS for route host agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xfc127c420e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xfc12762a440\u003e: \n DNS for route host agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xfc127c420e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"b932a53b571dbe2eeca04fe0dfdcbb234fc9de03860883094295d480a50708d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113216309620641792","occurred_at":"2026-10-22T10:30:03Z","failed_at":"e2e","signature_id":"535db7791df0e62f6c566e8206080df859bf280615373999df4bcace6f1dde18","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.039s.\npod on cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should remain Pending\nThe function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:\npod targeting cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should not be assigned a node\nExpected\n \u003cstring\u003e: a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.039s.\npod on cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should remain Pending\nThe function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:\npod targeting cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should not be assigned a node\nExpected\n \u003cstring\u003e: a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36ebd23c937dc29d944a27853b2ad54818920906b2bd4877347b2a0825abe2fc","failure_pattern":"pod targeting cordoned node \u003cnode\u003e should not be assigned a node","search_query":"pod targeting cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should not be assigned a node"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36ebd23c937dc29d944a27853b2ad54818920906b2bd4877347b2a0825abe2fc","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7578/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113211866976096256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7578,"pr_state":"open","pr_sha":"844a090ee8a80e9f1a87402b8c90883bba049578","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T10:12:25Z","started_at":"2026-10-22T10:12:25Z","completed_at":"2026-10-22T12:49:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T12:53:29.32440195Z","timing_metadata_checked_at":"2026-10-22T12:53:29.32440195Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T12:53:29.328759541Z","region_metadata_checked_at":"2026-10-22T12:53:29.328759541Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5801a47440a88e34e7e6e92e80e513661f8a4e682f24f8d090d060a8c371fd2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7578/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113211866976096256","occurred_at":"2026-10-22T10:12:25Z","failed_at":"e2e","signature_id":"2efde096b96e33606cefab787452a20bfbeac53b5b67e5735832387f8619958e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x268662498700\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zczv78nf9ddg\" to finish creating: GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\",\n \"name\": \"dfac02ec-d27d-4ef0-b523-7cb0c0010faa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T11:20:30.318762088Z\",\n \"endTime\": \"2026-10-22T11:33:05.673152156Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\n correlation ID: f47f9f8e-d365-4823-b09b-4ccb1c4128f4\n client request ID: 32649701-fb7c-48c8-ac3b-ebfc52571839\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x268662498700\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zczv78nf9ddg\" to finish creating: GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\",\n \"name\": \"dfac02ec-d27d-4ef0-b523-7cb0c0010faa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T11:20:30.318762088Z\",\n \"endTime\": \"2026-10-22T11:33:05.673152156Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\n correlation ID: f47f9f8e-d365-4823-b09b-4ccb1c4128f4\n client request ID: 32649701-fb7c-48c8-ac3b-ebfc52571839\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a7b7935e0fbe062a83ac94a424d97876ea2ab08c4e53e37b439fcd600a57bd3","failure_pattern":"ERROR CODE: InvalidResource; detail message ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidResource"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1a7b7935e0fbe062a83ac94a424d97876ea2ab08c4e53e37b439fcd600a57bd3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113210447921418240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7373,"pr_state":"closed","pr_sha":"5051cc737d55c4b05cc92fa959d0f44fe99d3db6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T10:06:46Z","started_at":"2026-10-22T10:06:46Z","completed_at":"2026-10-22T12:27:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T12:42:43.411666559Z","timing_metadata_checked_at":"2026-10-22T12:42:43.411666559Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T12:42:43.414562548Z","region_metadata_checked_at":"2026-10-22T12:42:43.414562548Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113210171659390976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6823,"pr_state":"closed","pr_sha":"0106f6c2637f06cfc4f8384a6320a16f8332a691","final_merged_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T10:05:39Z","started_at":"2026-10-22T10:05:39Z","completed_at":"2026-10-22T12:25:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T12:42:43.249550201Z","timing_metadata_checked_at":"2026-10-22T12:42:43.249550201Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T12:42:43.252943988Z","region_metadata_checked_at":"2026-10-22T12:42:43.252943988Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"aa7c2e690781c66aa0dcefd50f4a23b637e5c232edc3f47c6b5924c793300bbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113210171659390976","occurred_at":"2026-10-22T10:05:39Z","failed_at":"e2e","signature_id":"f2dcdc168e6795af87723ba65fde77f18a504cb3af7a59e39b6eea863e2044ba","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.38\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f3cf438a400\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-2fr64m\" in resourcegroup=\"rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn\" to finish creating: GET https://rp.j9390976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269\",\n \"name\": \"ee32fd12-b37a-4cd9-99a9-d6c6135b4269\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T11:18:21.118418049Z\",\n \"endTime\": \"2026-10-22T11:37:27.814332759Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn-np-upgrade-2f-dc28d34b/providers/Microsoft.Compute/virtualMachines/c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn-np-upgrade-2f-dc28d34b/providers/Microsoft.Compute/virtualMachines/c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9390976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269\n correlation ID: 53e16191-0db4-40cf-9191-639da6db8ba5\n client request ID: ecd27427-b9cf-4b3a-aecb-2037c8f3702e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.38\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f3cf438a400\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-2fr64m\" in resourcegroup=\"rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn\" to finish creating: GET https://rp.j9390976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269\",\n \"name\": \"ee32fd12-b37a-4cd9-99a9-d6c6135b4269\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T11:18:21.118418049Z\",\n \"endTime\": \"2026-10-22T11:37:27.814332759Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn-np-upgrade-2f-dc28d34b/providers/Microsoft.Compute/virtualMachines/c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn-np-upgrade-2f-dc28d34b/providers/Microsoft.Compute/virtualMachines/c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9390976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269\n correlation ID: 53e16191-0db4-40cf-9191-639da6db8ba5\n client request ID: ecd27427-b9cf-4b3a-aecb-2037c8f3702e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113208030127460352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"b61b5c2dbd944b3d6073c8c05ccc06c95ef2b8e5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T09:57:10Z","started_at":"2026-10-22T09:57:10Z","completed_at":"2026-10-22T12:29:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T12:42:43.609210478Z","timing_metadata_checked_at":"2026-10-22T12:42:43.609210478Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T12:42:43.612121066Z","region_metadata_checked_at":"2026-10-22T12:42:43.612121066Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"1d5dc2698dc858e4f9f9651c856523302a3b4ff48e31f20d0265eaf1c2d019dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113208030127460352","occurred_at":"2026-10-22T09:57:10Z","failed_at":"alert","signature_id":"8217ab2784f36f979beaba44454a1e2325f5d27c7a000ec1e76e7eb24e5b4ea1","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T11:18:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7460352-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.205:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-t67kq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d7472bae-c27c-4ab8-baa0-3a814678cda6\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-t67kq (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T11:21:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7460352-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.205:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-t67kq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d7472bae-c27c-4ab8-baa0-3a814678cda6\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-t67kq (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-22T11:24:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-lqfpq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e223dfef-7a26-486c-ad94-ac549a60abd4\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-lqfpq (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-22T11:25:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-lqfpq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e223dfef-7a26-486c-ad94-ac549a60abd4\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-lqfpq (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"d4f3bf1181773c89b9dc429125620cbe6031a1ab88fc1662cdc06707ad25b0da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113208030127460352","occurred_at":"2026-10-22T09:57:10Z","failed_at":"alert","signature_id":"d9f7e014ea9393bf34d8e64426450dfc14c29fac5bb560d8f15e8301b474387c","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 29 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T11:35:49Z\n Ended: 2026-10-22T11:41:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.205:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-q4hm8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-22T11:35:49Z\n Ended: 2026-10-22T11:41:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.205:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-q4hm8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-22T11:35:50Z\n Ended: 2026-10-22T11:41:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.205:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-q4hm8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-22T11:35:50Z\n Ended: 2026-10-22T11:41:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjpuk6i2kb2t6qpc0cd60o1eif4bpro\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjpuk6i2kb2t6qpc0cd60o1eif4bpro/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-22T11:35:50Z\n Ended: 2026-10-22T11:41:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.205:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-q4hm8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-22T11:53:53Z\n Ended: 2026-10-22T11:59:56Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-22T11:53:53Z\n Ended: 2026-10-22T12:00:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-22T11:53:53Z\n Ended: 2026-10-22T12:00:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-22T11:53:53Z\n Ended: 2026-10-22T12:00:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-22T11:53:53Z\n Ended: 2026-10-22T12:00:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-22T11:53:53Z\n Ended: 2026-10-22T12:00:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-22T11:54:49Z\n Ended: 2026-10-22T12:01:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-22T11:54:49Z\n Ended: 2026-10-22T12:01:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-22T11:54:49Z\n Ended: 2026-10-22T12:00:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-22T11:54:49Z\n Ended: 2026-10-22T12:01:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-22T11:55:52Z\n Ended: 2026-10-22T12:02:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-22T11:55:52Z\n Ended: 2026-10-22T12:04:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-22T11:55:52Z\n Ended: 2026-10-22T12:03:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-22T11:55:52Z\n Ended: 2026-10-22T12:02:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-22T11:56:51Z\n Ended: 2026-10-22T12:03:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-22T11:56:51Z\n Ended: 2026-10-22T12:04:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-22T11:56:51Z\n Ended: 2026-10-22T12:05:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-22T11:56:51Z\n Ended: 2026-10-22T12:04:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-22T11:56:51Z\n Ended: 2026-10-22T12:05:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-22T11:56:51Z\n Ended: 2026-10-22T12:05:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-22T11:56:51Z\n Ended: 2026-10-22T12:05:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-22T11:56:51Z\n Ended: 2026-10-22T12:03:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-22T11:58:55Z\n Ended: 2026-10-22T12:04:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq9f63j3j2qj8oncuvau7tpnkhg8ub\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq9f63j3j2qj8oncuvau7tpnkhg8ub/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-22T11:58:55Z\n Ended: 2026-10-22T12:04:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7460352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352\", namespace=\"klusterlet-2tjq9f63j3j2qj8oncuvau7tpnkhg8ub\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjq9f63j3j2qj8oncuvau7tpnkhg8ub/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113203785768636416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7665,"pr_state":"closed","pr_sha":"983cfba68b2c170cc433573b5d9c0c72f2a1405a","final_merged_sha":"b40bc5cbb80a171bde8ca5e40c43b439603f5493","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T09:40:18Z","started_at":"2026-10-22T09:40:18Z","completed_at":"2026-10-22T10:48:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T11:06:46.517005842Z","timing_metadata_checked_at":"2026-10-22T11:06:46.517005842Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T11:06:46.521080531Z","region_metadata_checked_at":"2026-10-22T11:06:46.521080531Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2774e41582e91cc74feecd5d1ab04ef677a7952295528a570be2a3e9bb6925b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113203785768636416","occurred_at":"2026-10-22T09:40:18Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T10:26:56.653Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-22T10:26:57.976Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8636416-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-22T10:26:57.976Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8636416-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-22T10:26:58.804Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8636416-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-22T10:26:58.804Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8636416-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T10:26:58.804Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j8636416-svc\ntime=2026-10-22T10:42:01.308Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113198633527545856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7683,"pr_state":"closed","pr_sha":"b2049d610b0f44f4441e4e06f0dfd134505df89d","final_merged_sha":"b2049d610b0f44f4441e4e06f0dfd134505df89d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T09:19:50Z","started_at":"2026-10-22T09:19:50Z","completed_at":"2026-10-22T11:32:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T11:48:42.288899381Z","timing_metadata_checked_at":"2026-10-22T11:48:42.288899381Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T11:48:42.292956272Z","region_metadata_checked_at":"2026-10-22T11:48:42.292956272Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"3f5d7907db6d7e8d5ee585e5519ee0daeab1a8026888606f0533bdbe0e17c22e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113198633527545856","occurred_at":"2026-10-22T09:19:50Z","failed_at":"alert","signature_id":"0b4904eba35f2ce4fb4e61a5ba5bc50afccce305efed6874c4167b62c103c66b","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T10:34:09Z\n Ended: 2026-10-22T10:47:10Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7545856-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.184:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7545856/providers/microsoft.monitor/accounts/hcps-j7545856\", namespace=\"ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-bxqfq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"480916008285346af7edebef3e3ba00e18dc129236d2cb353c65746c56a692b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113198633527545856","occurred_at":"2026-10-22T09:19:50Z","failed_at":"alert","signature_id":"ba4aa3115f35ce0988c053065b4de6eb875806daaccb54e29cea6b49686f0130","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T10:19:14Z\n Ended: 2026-10-22T10:39:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7545856-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.184:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7545856/providers/microsoft.monitor/accounts/hcps-j7545856\", namespace=\"ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-bxqfq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-22T10:19:14Z\n Ended: 2026-10-22T10:38:10Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7545856-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.184:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7545856/providers/microsoft.monitor/accounts/hcps-j7545856\", namespace=\"ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-bxqfq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"81b147dd77f204546809f482f6cdb10d1e64cd9733e35e63205fabb81127bf3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113198633527545856","occurred_at":"2026-10-22T09:19:50Z","failed_at":"e2e","signature_id":"38380caeb49ac20d9aa623cb9f89aeb6a3a057f42abe6a005618f92bc4b435b1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-464j6rgg5p8f/cluster-candidate-4-23-c98khv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x15ffaf8e6ac0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-c98khv: failed waiting for cluster=\"cluster-candidate-4-23-c98khv\" in resourcegroup=\"rg-candidate-4-23-464j6rgg5p8f\" to finish creating: GET https://rp.j7545856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15\",\n \"name\": \"914ff296-32b8-41bd-9581-3a19f7c15e15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T09:57:41.668026777Z\",\n \"endTime\": \"2026-10-22T10:26:45.058545936Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7545856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15\n correlation ID: ba6da990-0b2a-402c-b063-c501f6bad04e\n client request ID: 2565518b-4e23-4410-8a4e-8f021c515e5f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-464j6rgg5p8f/cluster-candidate-4-23-c98khv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x15ffaf8e6ac0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-c98khv: failed waiting for cluster=\"cluster-candidate-4-23-c98khv\" in resourcegroup=\"rg-candidate-4-23-464j6rgg5p8f\" to finish creating: GET https://rp.j7545856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15\",\n \"name\": \"914ff296-32b8-41bd-9581-3a19f7c15e15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T09:57:41.668026777Z\",\n \"endTime\": \"2026-10-22T10:26:45.058545936Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7545856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15\n correlation ID: ba6da990-0b2a-402c-b063-c501f6bad04e\n client request ID: 2565518b-4e23-4410-8a4e-8f021c515e5f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c53be91e63d1913a4c2a7c3d41fa0d96992121ee25b6974edca970f96d7723d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113198633527545856","occurred_at":"2026-10-22T09:19:50Z","failed_at":"alert","signature_id":"d11f1ee63b9c35d1ba118fc95457b088065166fe3c0142c76f58ceb21d2523bc","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T10:28:07Z\n Ended: 2026-10-22T10:42:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7545856-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.184:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7545856/providers/microsoft.monitor/accounts/hcps-j7545856\", namespace=\"ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2bda07d3-27fb-4164-a4c3-6cfe1f72c141\"\n Description: Pod ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-22T10:28:07Z\n Ended: 2026-10-22T10:42:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7545856-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.184:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7545856/providers/microsoft.monitor/accounts/hcps-j7545856\", namespace=\"ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2bda07d3-27fb-4164-a4c3-6cfe1f72c141\"\n Description: Pod ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113192920306683904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7672,"pr_state":"closed","pr_sha":"8bcd43f4a5d222942cd8b739e8c0f4763d2621fa","final_merged_sha":"8bcd43f4a5d222942cd8b739e8c0f4763d2621fa","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T08:57:08Z","started_at":"2026-10-22T08:57:08Z","completed_at":"2026-10-22T11:26:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T11:38:37.72660355Z","timing_metadata_checked_at":"2026-10-22T11:38:37.72660355Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T11:38:37.729860142Z","region_metadata_checked_at":"2026-10-22T11:38:37.729860142Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113188493898289152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7681,"pr_state":"closed","pr_sha":"682e036e46a2a9cbc6a99f631fdcdd737ced03e7","final_merged_sha":"682e036e46a2a9cbc6a99f631fdcdd737ced03e7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T08:39:32Z","started_at":"2026-10-22T08:39:32Z","completed_at":"2026-10-22T10:57:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T11:06:46.92436254Z","timing_metadata_checked_at":"2026-10-22T11:06:46.92436254Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T11:06:46.927136133Z","region_metadata_checked_at":"2026-10-22T11:06:46.927136133Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113177722531352576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7592,"pr_state":"closed","pr_sha":"c9a469c0a42acf6482344c2525dfc92f27189b30","final_merged_sha":"c9a469c0a42acf6482344c2525dfc92f27189b30","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T07:56:44Z","started_at":"2026-10-22T07:56:44Z","completed_at":"2026-10-22T10:40:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T10:56:27.7688016Z","timing_metadata_checked_at":"2026-10-22T10:56:27.7688016Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T10:56:27.772830693Z","region_metadata_checked_at":"2026-10-22T10:56:27.772830693Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113171725192531968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7373,"pr_state":"closed","pr_sha":"5051cc737d55c4b05cc92fa959d0f44fe99d3db6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T07:32:54Z","started_at":"2026-10-22T07:32:54Z","completed_at":"2026-10-22T09:57:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T10:15:07.83928553Z","timing_metadata_checked_at":"2026-10-22T10:15:07.83928553Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T10:15:07.84317842Z","region_metadata_checked_at":"2026-10-22T10:15:07.84317842Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b974b4e66650d95d71a91ae1e80fcad513a9123a0fdf4bc17435a3a8faf80d73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113171725192531968","occurred_at":"2026-10-22T07:32:54Z","failed_at":"alert","signature_id":"e8bab8e8bd10f29c86040fc7faea61d020bbc0e4e4e89f6d21a0bf4619b7714f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T09:11:20Z\n Ended: 2026-10-22T09:17:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2531968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjnp7iabfu8t4bg9odobu4iitn1rm2a-v20260901\", pod=\"etcd-0\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tjnp7iabfu8t4bg9odobu4iitn1rm2a-v20260901/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113166434187087872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"closed","pr_sha":"ce91a6e03b6b92619c585957df7276d393359389","final_merged_sha":"4cf164ecff36585c6e67a93232af75a00a26bf26","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T07:11:53Z","started_at":"2026-10-22T07:11:53Z","completed_at":"2026-10-22T09:16:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T09:28:59.879244485Z","timing_metadata_checked_at":"2026-10-22T09:28:59.879244485Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T09:28:59.886221969Z","region_metadata_checked_at":"2026-10-22T09:28:59.886221969Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113165525352714240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"b61b5c2dbd944b3d6073c8c05ccc06c95ef2b8e5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T07:08:16Z","started_at":"2026-10-22T07:08:16Z","completed_at":"2026-10-22T09:21:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T09:29:00.076342444Z","timing_metadata_checked_at":"2026-10-22T09:29:00.076342444Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T09:29:00.093755605Z","region_metadata_checked_at":"2026-10-22T09:29:00.093755605Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2f70f1b06285a359bc1841d0ebe3b0a9a53806a6c0a2f5c78bd8650dfa55724f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113165525352714240","occurred_at":"2026-10-22T07:08:16Z","failed_at":"e2e","signature_id":"bb30606bde067b70e417b1ab7034bb648603ed8a2a28fe33e585197379c8df85","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:361]: failed to create console OAuth client secret for external auth via VM\nUnexpected error:\n \u003c*errors.errorString | 0x339af03e78d0\u003e: \n timed out waiting for VM run command \"e2e-runcommand-1792657320988467170\" on VM \"cidr-connectivity-test-test-vm\"; canceled run command\n {\n s: \"timed out waiting for VM run command \\\"e2e-runcommand-1792657320988467170\\\" on VM \\\"cidr-connectivity-test-test-vm\\\"; canceled run command\",\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:361]: failed to create console OAuth client secret for external auth via VM\nUnexpected error:\n \u003c*errors.errorString | 0x339af03e78d0\u003e: \n timed out waiting for VM run command \"e2e-runcommand-1792657320988467170\" on VM \"cidr-connectivity-test-test-vm\"; canceled run command\n {\n s: \"timed out waiting for VM run command \\\"e2e-runcommand-1792657320988467170\\\" on VM \\\"cidr-connectivity-test-test-vm\\\"; canceled run command\",\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36030c2f6530fdbd07d634405c15916c6fea00f3911df6853eed3a48b722f69c","failure_pattern":"timed out waiting for VM run command \"e2e-runcommand-1792657320988467170\" on VM \"\u003cvm\u003e\"; canceled run command","search_query":"occurred"}},{"row_id":"83111ac475b8ac6b4e92d6850a26374fb298fb743afde074a7265b7bdcad3197","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113165525352714240","occurred_at":"2026-10-22T07:08:16Z","failed_at":"alert","signature_id":"7a54f01ba59cb3554a88a9d5a612ad65fae371543aa6288021249b5c68542874","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 30 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T08:33:21Z\n Ended: 2026-10-22T08:39:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjn8k6j55ugd7dnkujc325j0v4cs6pr\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-6tpfj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjn8k6j55ugd7dnkujc325j0v4cs6pr/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-22T08:50:23Z\n Ended: 2026-10-22T08:56:20Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjnh74e75j44d0qjilt2dadh163si99\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjnh74e75j44d0qjilt2dadh163si99/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-22T08:50:24Z\n Ended: 2026-10-22T08:57:20Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjnh74e75j44d0qjilt2dadh163si99\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjnh74e75j44d0qjilt2dadh163si99/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-22T08:50:24Z\n Ended: 2026-10-22T08:57:20Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjnh74e75j44d0qjilt2dadh163si99\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjnh74e75j44d0qjilt2dadh163si99/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-22T08:51:19Z\n Ended: 2026-10-22T08:58:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-22T08:51:19Z\n Ended: 2026-10-22T08:57:20Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjnh74e75j44d0qjilt2dadh163si99\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjnh74e75j44d0qjilt2dadh163si99/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-22T08:51:19Z\n Ended: 2026-10-22T08:57:20Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjnh74e75j44d0qjilt2dadh163si99\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjnh74e75j44d0qjilt2dadh163si99/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-22T08:51:19Z\n Ended: 2026-10-22T08:57:20Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-22T08:51:19Z\n Ended: 2026-10-22T08:58:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-22T08:51:19Z\n Ended: 2026-10-22T08:59:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-22T08:51:19Z\n Ended: 2026-10-22T08:57:20Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-22T08:51:19Z\n Ended: 2026-10-22T08:58:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:00:21Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:01:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:00:21Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:05:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:05:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:04:20Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:01:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:05:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:05:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:04:19Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:01:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:04:19Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:05:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:01:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:05:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:05:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:04:19Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-22T08:53:18Z\n Ended: 2026-10-22T09:05:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"c4656099be856eb8ee708b1bfbb227d53465c7b4f3ecb317b03f70175c84c27b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113165525352714240","occurred_at":"2026-10-22T07:08:16Z","failed_at":"alert","signature_id":"a8a3c6a30c55341f0d2bcacbcbbd3a03912b87ddffb2ee938913151c553a557f","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T08:13:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-2f299\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f456e270-14f4-4101-89f7-ecf21d7b98ce\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-2f299 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T08:15:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2714240-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-2f299\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f456e270-14f4-4101-89f7-ecf21d7b98ce\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-2f299 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-22T08:16:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2714240-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-8jwvp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f1b0a0ab-3f08-4426-a498-92feced750e6\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-8jwvp (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-22T08:17:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2714240-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-8jwvp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f1b0a0ab-3f08-4426-a498-92feced750e6\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-8jwvp (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["36030c2f6530fdbd07d634405c15916c6fea00f3911df6853eed3a48b722f69c","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7675/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113164811889020928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7675,"pr_state":"closed","pr_sha":"1e6a37af898da7289b27fc9334ee222d4d28469f","final_merged_sha":"1e6a37af898da7289b27fc9334ee222d4d28469f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T07:05:26Z","started_at":"2026-10-22T07:05:26Z","completed_at":"2026-10-22T09:22:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T09:29:00.289859266Z","timing_metadata_checked_at":"2026-10-22T09:29:00.289859266Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T09:29:00.296595551Z","region_metadata_checked_at":"2026-10-22T09:29:00.296595551Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113164536595877888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7665,"pr_state":"closed","pr_sha":"641d7c05eefb41ea106b494d8be6c4d2b3824d01","final_merged_sha":"b40bc5cbb80a171bde8ca5e40c43b439603f5493","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T07:04:20Z","started_at":"2026-10-22T07:04:20Z","completed_at":"2026-10-22T09:11:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T09:18:36.079790325Z","timing_metadata_checked_at":"2026-10-22T09:18:36.079790325Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T09:18:36.083216613Z","region_metadata_checked_at":"2026-10-22T09:18:36.083216613Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"59f432549537b775104254f187f88b1aa57e94e684cd51e0fe558f7d503f5faf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113164536595877888","occurred_at":"2026-10-22T07:04:20Z","failed_at":"e2e","signature_id":"3b8862f53308a60b2af78a86bb8911332deef646ed130d7021031cd841820f0b","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x291503b87830\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x291503b87830\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3aa90bdf47475703df3b110c727dbfb72dbdbd02636c9ba15471acd53cf0021f","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225"}},{"row_id":"63c31c95e85bb95bdf3d53a085782f1fc0e56d3f701bd3c7e727d3d7b29c4fc0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113164536595877888","occurred_at":"2026-10-22T07:04:20Z","failed_at":"e2e","signature_id":"2a31cf35da94c36d9c9a054be4d1fc4a8f126cdb012d93530f303b9096768ee7","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d7c44593440\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-h4jhcq\" in resourcegroup=\"rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb\" to finish creating: GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2\",\n \"name\": \"5b440666-a8bb-43ad-8b4a-822ed5350ff2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T08:17:17.757679885Z\",\n \"endTime\": \"2026-10-22T08:36:24.274326303Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb-np-upgrade-h4-a1d46754/providers/Microsoft.Compute/virtualMachines/n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb-np-upgrade-h4-a1d46754/providers/Microsoft.Compute/virtualMachines/n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2\n correlation ID: 1da8ee73-93dd-4106-994f-5306d5b29df5\n client request ID: b60ab242-ab9c-4c04-b16f-c44b09af50eb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d7c44593440\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-h4jhcq\" in resourcegroup=\"rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb\" to finish creating: GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2\",\n \"name\": \"5b440666-a8bb-43ad-8b4a-822ed5350ff2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T08:17:17.757679885Z\",\n \"endTime\": \"2026-10-22T08:36:24.274326303Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb-np-upgrade-h4-a1d46754/providers/Microsoft.Compute/virtualMachines/n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb-np-upgrade-h4-a1d46754/providers/Microsoft.Compute/virtualMachines/n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2\n correlation ID: 1da8ee73-93dd-4106-994f-5306d5b29df5\n client request ID: b60ab242-ab9c-4c04-b16f-c44b09af50eb\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9a27037d13a29a7b85c7db8710cab6f72bceea01947de3b798f5e79a05c8add2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113164536595877888","occurred_at":"2026-10-22T07:04:20Z","failed_at":"e2e","signature_id":"7ce7ec004ca06c2f4f394ab96fd260d7f8d5523221c536dc3b5faf13439d18c9","test_name":"Engineering want metrics from SVC and MGMT clusters to be present in Azure Monitor WOrkspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.\nnot all expected service metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:\nexpected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: []\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.\nnot all expected service metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:\nexpected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: []\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b1d68134ee1583cdf3cf29ccf5f52a00bbda8ef7bbf33a93cfb29ce106c7b3bb","failure_pattern":"not all expected service metrics appeared in Azure Monitor","search_query":"not all expected service metrics appeared in Azure Monitor"}},{"row_id":"bda9939b45ec3766a1b434e71dd44539b681f6dabdbf669bd0a1180f7ae034d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113164536595877888","occurred_at":"2026-10-22T07:04:20Z","failed_at":"e2e","signature_id":"5aa430a71f1e1be5ff2471115ee092e684da6d03063af5eec54d8c3b6e8e964a","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xed2a17d5560\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-kr8h8mgbckw7\" to finish creating: GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\",\n \"name\": \"a192dd85-8881-406d-a906-718d3d901c7e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T08:02:04.742645812Z\",\n \"endTime\": \"2026-10-22T08:21:04.297195938Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Compute/virtualMachines/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-kr8h8mgbckw7--managed/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Network/networkInterfaces/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\n correlation ID: 72b107e0-3135-44c9-989b-a2246b2d1d75\n client request ID: db0337ce-fd31-490b-ad1f-f2e98766f20c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xed2a17d5560\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-kr8h8mgbckw7\" to finish creating: GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\",\n \"name\": \"a192dd85-8881-406d-a906-718d3d901c7e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T08:02:04.742645812Z\",\n \"endTime\": \"2026-10-22T08:21:04.297195938Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Compute/virtualMachines/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-kr8h8mgbckw7--managed/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Network/networkInterfaces/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\n correlation ID: 72b107e0-3135-44c9-989b-a2246b2d1d75\n client request ID: db0337ce-fd31-490b-ad1f-f2e98766f20c\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b7a7316d7bc9a13546759ba543c0393779c1adcbaf7eafe296163143bdb55b2c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["3aa90bdf47475703df3b110c727dbfb72dbdbd02636c9ba15471acd53cf0021f","b1d68134ee1583cdf3cf29ccf5f52a00bbda8ef7bbf33a93cfb29ce106c7b3bb","b7a7316d7bc9a13546759ba543c0393779c1adcbaf7eafe296163143bdb55b2c","d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7668/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113161145354293248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7668,"pr_state":"closed","pr_sha":"98cd233347471b6bbb1e8bf223eabbfd5025ba99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T06:50:52Z","started_at":"2026-10-22T06:50:52Z","completed_at":"2026-10-22T09:00:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T09:08:14.852632799Z","timing_metadata_checked_at":"2026-10-22T09:08:14.852632799Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T09:08:14.856470392Z","region_metadata_checked_at":"2026-10-22T09:08:14.856470392Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113158281953284096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7669,"pr_state":"closed","pr_sha":"e5d396359943f1d53aea2fc812b45403fbbb8f23","final_merged_sha":"e5d396359943f1d53aea2fc812b45403fbbb8f23","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T06:39:29Z","started_at":"2026-10-22T06:39:29Z","completed_at":"2026-10-22T08:58:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T09:08:14.874461056Z","timing_metadata_checked_at":"2026-10-22T09:08:14.874461056Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T09:08:14.87728065Z","region_metadata_checked_at":"2026-10-22T09:08:14.87728065Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113151915083698176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7672,"pr_state":"closed","pr_sha":"8bcd43f4a5d222942cd8b739e8c0f4763d2621fa","final_merged_sha":"8bcd43f4a5d222942cd8b739e8c0f4763d2621fa","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T06:14:11Z","started_at":"2026-10-22T06:14:11Z","completed_at":"2026-10-22T08:31:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T08:47:06.767286172Z","timing_metadata_checked_at":"2026-10-22T08:47:06.767286172Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T08:47:06.770921364Z","region_metadata_checked_at":"2026-10-22T08:47:06.770921364Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"82bae8dabf57595a443140ccfbc12b296fd8a091ac27620b38f85331514df5be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113151915083698176","occurred_at":"2026-10-22T06:14:11Z","failed_at":"e2e","signature_id":"dc531b14207c6f576efeb1d0b689a61a1f6e47c8a24d2b02b6c10b19d54c8208","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-1-8599c9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e7ad2d862a0\u003e: \n DNS for route host agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2e7ad2a24480\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-1-8599c9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e7ad2d862a0\u003e: \n DNS for route host agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2e7ad2a24480\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113147384883056640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7573,"pr_state":"closed","pr_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","final_merged_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T05:56:11Z","started_at":"2026-10-22T05:56:11Z","completed_at":"2026-10-22T08:47:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T08:57:39.417421668Z","timing_metadata_checked_at":"2026-10-22T08:57:39.417421668Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T08:57:39.421203255Z","region_metadata_checked_at":"2026-10-22T08:57:39.421203255Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"50dd004c1c4180798da65556846743e7b35cad15f43ee21a985d0f4c2ff29105","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113147384883056640","occurred_at":"2026-10-22T05:56:11Z","failed_at":"alert","signature_id":"f45f5c96ff8e6ceb8e57b7dda4c8c05437e4ee5fb4582519685c9718a6396ed6","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T07:46:50Z\n Ended: 2026-10-22T08:14:51Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3056640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.145:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3056640/providers/microsoft.monitor/accounts/hcps-j3056640\", namespace=\"ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-f624w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-22T07:46:50Z\n Ended: 2026-10-22T08:13:50Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3056640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.145:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3056640/providers/microsoft.monitor/accounts/hcps-j3056640\", namespace=\"ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-f624w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"95e5ce6318fcf5c5a8ac78c5d43c3083cedcf55713c58e17a576acdc3140bf5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113147384883056640","occurred_at":"2026-10-22T05:56:11Z","failed_at":"e2e","signature_id":"11bb1183f5742505aad8eb8162c6762f75c32bdf1650bcffd71b6e3d63f7d81c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-8whfwd9s2cfc/cluster-candidate-5-0-rsdsmg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2868f20425c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-rsdsmg: failed waiting for cluster=\"cluster-candidate-5-0-rsdsmg\" in resourcegroup=\"rg-candidate-5-0-8whfwd9s2cfc\" to finish creating: GET https://rp.j3056640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058\",\n \"name\": \"624deb2b-f6be-4ff9-9d3a-453b1d0fd058\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T07:26:11.110423067Z\",\n \"endTime\": \"2026-10-22T07:55:18.619815497Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3056640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058\n correlation ID: 2a0d721c-fc9d-4caa-9738-de4581a559a8\n client request ID: 087da192-6efb-4007-8ca8-464d6f6bf167\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-8whfwd9s2cfc/cluster-candidate-5-0-rsdsmg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2868f20425c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-rsdsmg: failed waiting for cluster=\"cluster-candidate-5-0-rsdsmg\" in resourcegroup=\"rg-candidate-5-0-8whfwd9s2cfc\" to finish creating: GET https://rp.j3056640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058\",\n \"name\": \"624deb2b-f6be-4ff9-9d3a-453b1d0fd058\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T07:26:11.110423067Z\",\n \"endTime\": \"2026-10-22T07:55:18.619815497Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3056640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058\n correlation ID: 2a0d721c-fc9d-4caa-9738-de4581a559a8\n client request ID: 087da192-6efb-4007-8ca8-464d6f6bf167\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c012435dbd7f2084cd872f65126fba7464bd1773600a64577658179eac5f421f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113147384883056640","occurred_at":"2026-10-22T05:56:11Z","failed_at":"alert","signature_id":"2387f1827a61b63e936473078ea28c130b72c4d492dfe00685aff4e57613ecb7","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T07:48:50Z\n Ended: 2026-10-22T08:18:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3056640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.145:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3056640/providers/microsoft.monitor/accounts/hcps-j3056640\", namespace=\"ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f844d575-0b52-40fe-a2f1-220180987256\"\n Description: Pod ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-22T07:49:47Z\n Ended: 2026-10-22T08:18:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3056640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.145:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3056640/providers/microsoft.monitor/accounts/hcps-j3056640\", namespace=\"ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f844d575-0b52-40fe-a2f1-220180987256\"\n Description: Pod ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"e3bdd65bf2f21738026066382c529e90858cd2f3923e8b0ea585fac3234a5abe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113147384883056640","occurred_at":"2026-10-22T05:56:11Z","failed_at":"alert","signature_id":"dc0b14c2e44620ae1c288ecad48b6c8dec7c73d0f88905afa45ae94448860d82","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T08:01:49Z\n Ended: 2026-10-22T08:15:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3056640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.145:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3056640/providers/microsoft.monitor/accounts/hcps-j3056640\", namespace=\"ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-f624w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113142261167427584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7653,"pr_state":"closed","pr_sha":"76da4621827373d9267b68cfd55f1d91f45267d3","final_merged_sha":"1c5f99177052e49fb875d9503c65dd8f8ccd901f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T05:35:49Z","started_at":"2026-10-22T05:35:49Z","completed_at":"2026-10-22T06:43:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T06:53:17.238900739Z","timing_metadata_checked_at":"2026-10-22T06:53:17.238900739Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T06:53:17.241950432Z","region_metadata_checked_at":"2026-10-22T06:53:17.241950432Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"731b55555d2d90617b7e9eae710d86552e3fcc541468a34e6bbb29e0287eb260","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113142261167427584","occurred_at":"2026-10-22T05:35:49Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T06:22:04.877Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-22T06:22:06.224Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j7427584-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-22T06:22:06.224Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j7427584-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-22T06:22:06.758Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j7427584-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-22T06:22:06.758Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j7427584-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T06:22:06.758Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j7427584-svc\ntime=2026-10-22T06:37:09.071Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113138915056553984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7679,"pr_state":"closed","pr_sha":"eb19f69b64845587ae65b3fbdc663d1d5a68d99b","final_merged_sha":"aef55fb48491e559a7a2689dd16c32b57b3baced","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T05:22:32Z","started_at":"2026-10-22T05:22:32Z","completed_at":"2026-10-22T07:43:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T07:55:53.560067376Z","timing_metadata_checked_at":"2026-10-22T07:55:53.560067376Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T07:55:53.563287866Z","region_metadata_checked_at":"2026-10-22T07:55:53.563287866Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"ac921f02cc11654e3ddee6e3943ea610f5fd0b00bd2ca5881bcd65f1759a9deb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113138915056553984","occurred_at":"2026-10-22T05:22:32Z","failed_at":"e2e","signature_id":"1a3274025bba0a36059024cc04b88e2285c5367a4778ef67fe03503d1eb13a3e","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:136]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0x347c86c470c8\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0x347c84da9480\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0x347c85daeb28\u003e{\n errs: [\n \u003c*errors.errorString | 0x347c86a67600\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:136]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0x347c86c470c8\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0x347c84da9480\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0x347c85daeb28\u003e{\n errs: [\n \u003c*errors.errorString | 0x347c86a67600\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1"}},{"row_id":"cbc49299b6c11e486e5bbf135aa0b286620c67be144c8e0ea9b6c1f82066e027","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113138915056553984","occurred_at":"2026-10-22T05:22:32Z","failed_at":"e2e","signature_id":"ff7400b28734fc76d943e06d455e076ff28cea9a80dbcc5f0df78307a659662d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-56rfqn\" for cluster \"cluster-candidate-4-21-56rfqn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a4be883de20\u003e: \n failed to create NodePool np-56rfqn: failed waiting for nodepool=\"np-56rfqn\" for cluster \"cluster-candidate-4-21-56rfqn\" in resourcegroup=\"rg-candidate-4-21-72hp88nff97p\" to finish creating: GET https://rp.j6553984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e\",\n \"name\": \"689ced2e-06eb-4d9a-b1eb-1be22576e59e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T06:31:11.289499044Z\",\n \"endTime\": \"2026-10-22T06:50:16.978507942Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-72hp88nff97p-candidate-56rfqn--managed/providers/Microsoft.Compute/virtualMachines/k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-72hp88nff97p-candidate-56rfqn--managed/providers/Microsoft.Compute/virtualMachines/k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6553984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e\n correlation ID: c789114c-8019-4481-bf73-689cc746416e\n client request ID: 963d7d6c-53b8-47ee-aceb-8ee1ac1d29bf\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-56rfqn\" for cluster \"cluster-candidate-4-21-56rfqn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a4be883de20\u003e: \n failed to create NodePool np-56rfqn: failed waiting for nodepool=\"np-56rfqn\" for cluster \"cluster-candidate-4-21-56rfqn\" in resourcegroup=\"rg-candidate-4-21-72hp88nff97p\" to finish creating: GET https://rp.j6553984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e\",\n \"name\": \"689ced2e-06eb-4d9a-b1eb-1be22576e59e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T06:31:11.289499044Z\",\n \"endTime\": \"2026-10-22T06:50:16.978507942Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-72hp88nff97p-candidate-56rfqn--managed/providers/Microsoft.Compute/virtualMachines/k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-72hp88nff97p-candidate-56rfqn--managed/providers/Microsoft.Compute/virtualMachines/k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6553984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e\n correlation ID: c789114c-8019-4481-bf73-689cc746416e\n client request ID: 963d7d6c-53b8-47ee-aceb-8ee1ac1d29bf\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7675/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113136134002315264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7675,"pr_state":"closed","pr_sha":"0df0c917f80ebc4c9d60d01a8f91b645a4539b44","final_merged_sha":"1e6a37af898da7289b27fc9334ee222d4d28469f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T05:11:29Z","started_at":"2026-10-22T05:11:29Z","completed_at":"2026-10-22T06:18:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T07:24:18.84810477Z","timing_metadata_checked_at":"2026-10-22T07:24:18.84810477Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T07:24:18.851467759Z","region_metadata_checked_at":"2026-10-22T07:24:18.851467759Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"20028834a407c18bfe243f958209d469c0ab08357053fe9f5d09ed164c92d696","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7675/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113136134002315264","occurred_at":"2026-10-22T05:11:29Z","failed_at":"alert","signature_id":"5022976043c0a5c67816fadc890268fa3ab94e3bd6c81af70a2cfa059aa64b1c","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T06:53:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"acm-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264\", namespace=\"klusterlet-2tjlnu5835h8mc45h83n6jc9202p3aa8\", pod=\"klusterlet-addon-workmgr-6f457dbf58-2whgw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f288f60f-8a99-4810-a368-35076f7ec62b\"\n Description: Pod klusterlet-2tjlnu5835h8mc45h83n6jc9202p3aa8/klusterlet-addon-workmgr-6f457dbf58-2whgw (acm-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T06:53:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"acm-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264\", namespace=\"klusterlet-2tjlnu5835h8mc45h83n6jc9202p3aa8\", pod=\"klusterlet-addon-workmgr-6f457dbf58-2whgw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f288f60f-8a99-4810-a368-35076f7ec62b\"\n Description: Pod klusterlet-2tjlnu5835h8mc45h83n6jc9202p3aa8/klusterlet-addon-workmgr-6f457dbf58-2whgw (acm-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"5549daf51c094bab2bb567463480e1bdd3ffebdd26f4374ffa3cbd04e36ab878","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7675/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113136134002315264","occurred_at":"2026-10-22T05:11:29Z","failed_at":"alert","signature_id":"394a5ae97331321eda54df56d575a203a99c7f35f5b568580d9d5fb95b350d96","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 72 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T06:53:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"azure-cloud-controller-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf7u6ccqaq3v26qvoollh2q9l74n2-w4b2e7w9j1z2b4x\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf7u6ccqaq3v26qvoollh2q9l74n2-w4b2e7w9j1z2b4x/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T06:53:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-22T06:53:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-22T06:53:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-22T06:53:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-22T06:53:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-state-metrics-hcp\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/kube-state-metrics-hcp has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-state-metrics-hcp\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/kube-state-metrics-hcp has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"openshift-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/openshift-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf4p1e86tn9jn6bnmvi92avpsnmr2-t5t6z2c2m7l0h8n\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf4p1e86tn9jn6bnmvi92avpsnmr2-t5t6z2c2m7l0h8n/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 18:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 19:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 20:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 21:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 22:\n State: Fired (not resolved)\n Started: 2026-10-22T06:54:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 23:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5hrm6lrr4j7n6nsn5n855udsr2j-d7v6t0q4c4t1b2j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5hrm6lrr4j7n6nsn5n855udsr2j-d7v6t0q4c4t1b2j/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 24:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 25:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 26:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 27:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 28:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 29:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 30:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf4p1e86tn9jn6bnmvi92avpsnmr2-t5t6z2c2m7l0h8n\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf4p1e86tn9jn6bnmvi92avpsnmr2-t5t6z2c2m7l0h8n/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 31:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 32:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 33:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 34:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 35:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"azure-cloud-controller-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5alfk8qk69tnqcis4t9ua0fv3bb-e6n0a3s6t4a0d8g\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5alfk8qk69tnqcis4t9ua0fv3bb-e6n0a3s6t4a0d8g/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.\nFiring 36:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 37:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"azure-cloud-controller-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf9pjn4g50ffukr1onbt218alfe4k-s5u2g5f8n8v7h1c\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf9pjn4g50ffukr1onbt218alfe4k-s5u2g5f8n8v7h1c/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.\nFiring 38:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"azure-cloud-controller-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf9pjn4g50ffukr1onbt218alfe4k-s5u2g5f8n8v7h1c\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf9pjn4g50ffukr1onbt218alfe4k-s5u2g5f8n8v7h1c/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.\nFiring 39:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 40:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 41:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 42:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 43:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 44:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 45:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 46:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 47:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 48:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 49:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"openshift-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/openshift-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 50:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 51:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"azure-cloud-controller-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5alfk8qk69tnqcis4t9ua0fv3bb-e6n0a3s6t4a0d8g\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5alfk8qk69tnqcis4t9ua0fv3bb-e6n0a3s6t4a0d8g/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.\nFiring 52:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 53:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 54:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf5hrm6lrr4j7n6nsn5n855udsr2j-d7v6t0q4c4t1b2j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf5hrm6lrr4j7n6nsn5n855udsr2j-d7v6t0q4c4t1b2j/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 55:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 56:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 57:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 58:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 59:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"openshift-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt/openshift-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 60:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 61:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 62:\n State: Fired (not resolved)\n Started: 2026-10-22T06:55:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 63:\n State: Fired (not resolved)\n Started: 2026-10-22T06:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"azure-cloud-controller-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlg6hbvd4pqme7j4ns80o2aovvt54v-q4c2c9u5g9p5a2p\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlg6hbvd4pqme7j4ns80o2aovvt54v-q4c2c9u5g9p5a2p/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.\nFiring 64:\n State: Fired (not resolved)\n Started: 2026-10-22T06:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf7u6ccqaq3v26qvoollh2q9l74n2-w4b2e7w9j1z2b4x\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf7u6ccqaq3v26qvoollh2q9l74n2-w4b2e7w9j1z2b4x/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 65:\n State: Fired (not resolved)\n Started: 2026-10-22T06:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 66:\n State: Fired (not resolved)\n Started: 2026-10-22T06:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 67:\n State: Fired (not resolved)\n Started: 2026-10-22T06:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfa1kcsndl903g11sinfeoor1pae5-p5z7m8h6i6z2o6j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfa1kcsndl903g11sinfeoor1pae5-p5z7m8h6i6z2o6j/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 68:\n State: Fired (not resolved)\n Started: 2026-10-22T06:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 69:\n State: Fired (not resolved)\n Started: 2026-10-22T06:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfa1kcsndl903g11sinfeoor1pae5-p5z7m8h6i6z2o6j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfa1kcsndl903g11sinfeoor1pae5-p5z7m8h6i6z2o6j/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 70:\n State: Fired (not resolved)\n Started: 2026-10-22T06:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"packageserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y/packageserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 71:\n State: Fired (not resolved)\n Started: 2026-10-22T06:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf2op037rdv3hdc8h1qto6lno6hal-u7g3d8u7r4l9m4n\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf2op037rdv3hdc8h1qto6lno6hal-u7g3d8u7r4l9m4n/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 72:\n State: Fired (not resolved)\n Started: 2026-10-22T06:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-controller-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264\", namespace=\"ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w/kube-controller-manager has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"ef2068984d7169f3a90b6bfac162456f9b9a3a184e6d13e873ef4e4f57511bda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7675/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113136134002315264","occurred_at":"2026-10-22T05:11:29Z","failed_at":"alert","signature_id":"7f9d9381a76886da9647cfc4773d130bc904bc278913f3ad683332e01b2b5ea2","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T06:50:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264\", namespace=\"klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T06:50:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264\", namespace=\"klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-22T06:50:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264\", namespace=\"klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-22T06:50:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264\", namespace=\"klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-22T06:50:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264\", namespace=\"klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-22T06:50:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2315264-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264\", namespace=\"klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113135233485246464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7592,"pr_state":"closed","pr_sha":"c9a469c0a42acf6482344c2525dfc92f27189b30","final_merged_sha":"c9a469c0a42acf6482344c2525dfc92f27189b30","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T05:07:54Z","started_at":"2026-10-22T05:07:54Z","completed_at":"2026-10-22T07:22:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T07:34:36.924195793Z","timing_metadata_checked_at":"2026-10-22T07:34:36.924195793Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T07:34:36.927476685Z","region_metadata_checked_at":"2026-10-22T07:34:36.927476685Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2b6142c225bb587461e3aea60df2dfa7d5fd6779476da269200925bad07af676","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113135233485246464","occurred_at":"2026-10-22T05:07:54Z","failed_at":"e2e","signature_id":"0261483674c89aeb066aafc37d75b4ad9859cf5cd7d723600d46bef0f3934bdd","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e7f906828a0\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-hlczr5rhlkx8\" to finish creating: GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\",\n \"name\": \"2d4aa15e-aedb-44e3-87cf-763478cb8fc1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T06:07:20.975549407Z\",\n \"endTime\": \"2026-10-22T06:26:22.595569254Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t to exist\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-labels-taints-hlczr5rhlkx8--managed/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Network/networkInterfaces/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\n correlation ID: 42569081-3a4c-40a5-ba81-59a78fa3c57e\n client request ID: 1bbe15c9-7618-4851-b4b5-17110a162bd6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e7f906828a0\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-hlczr5rhlkx8\" to finish creating: GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\",\n \"name\": \"2d4aa15e-aedb-44e3-87cf-763478cb8fc1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T06:07:20.975549407Z\",\n \"endTime\": \"2026-10-22T06:26:22.595569254Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t to exist\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-labels-taints-hlczr5rhlkx8--managed/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Network/networkInterfaces/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\n correlation ID: 42569081-3a4c-40a5-ba81-59a78fa3c57e\n client request ID: 1bbe15c9-7618-4851-b4b5-17110a162bd6\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"445bea11baaa5e6a0f158d83698b151143249148184d0e3c92735ff6d2548efa","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["445bea11baaa5e6a0f158d83698b151143249148184d0e3c92735ff6d2548efa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113130790136057856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7177,"pr_state":"closed","pr_sha":"32b56098d87e1f11371dcee30c31790152271af7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T04:50:15Z","started_at":"2026-10-22T04:50:15Z","completed_at":"2026-10-22T07:06:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T07:24:18.738169808Z","timing_metadata_checked_at":"2026-10-22T07:24:18.738169808Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T07:24:18.741757597Z","region_metadata_checked_at":"2026-10-22T07:24:18.741757597Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"16f9f8d561cca8aaaee5bdaedfb3ca69c72d7b2034e05a34d908f9125e95f7c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113130790136057856","occurred_at":"2026-10-22T04:50:15Z","failed_at":"e2e","signature_id":"8d094efb455f76798c9e8a4cce31f30c0ce80c0c9a0fd1d2e952804b6d1dc9b7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-7lghhv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3dee4275c660\u003e: \n DNS for route host agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3dee4349c920\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-7lghhv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3dee4275c660\u003e: \n DNS for route host agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3dee4349c920\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113129581849350144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"closed","pr_sha":"8c2779f2f687ddde332737cf706c7046bfb238d9","final_merged_sha":"4cf164ecff36585c6e67a93232af75a00a26bf26","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T04:45:26Z","started_at":"2026-10-22T04:45:26Z","completed_at":"2026-10-22T05:50:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T06:00:31.215862674Z","timing_metadata_checked_at":"2026-10-22T06:00:31.215862674Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T06:00:31.219522871Z","region_metadata_checked_at":"2026-10-22T06:00:31.219522871Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"cedb3ed36d3b7d0fd34ea4e8164120a36ba6f591b24626ed164af78c7be2aeea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113129581849350144","occurred_at":"2026-10-22T04:45:26Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T05:26:16.605Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-22T05:26:17.883Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-22T05:26:17.883Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-22T05:26:18.860Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-22T05:26:18.860Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T05:26:18.860Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j9350144-svc\ntime=2026-10-22T05:29:23.971Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-22T05:29:23.971Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=3m5s\ntime=2026-10-22T05:29:23.971Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T05:29:24.574Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-22T05:29:24.574Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T05:29:24.574Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T05:29:25.174Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-22T05:29:25.174Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T05:29:25.205Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-22T05:29:25.415Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T05:29:25.470Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T05:29:25.913Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T05:29:25.913Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T05:29:25.913Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T05:29:25.913Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-22T05:29:25.913Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j9350144-svc\ntime=2026-10-22T05:44:28.867Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230","failure_pattern":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7097/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113124474055299072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7097,"pr_state":"open","pr_sha":"29ffecb1b3517089c8aa339e8a9303239e056ddf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T04:25:09Z","started_at":"2026-10-22T04:25:09Z","completed_at":"2026-10-22T06:42:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T06:53:16.950490464Z","timing_metadata_checked_at":"2026-10-22T06:53:16.950490464Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T06:53:16.953969956Z","region_metadata_checked_at":"2026-10-22T06:53:16.953969956Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7253/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113096217243160576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7253,"pr_state":"closed","pr_sha":"72572322a275d718fe227782b5f5867f2a389729","final_merged_sha":"72572322a275d718fe227782b5f5867f2a389729","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-22T02:32:52Z","started_at":"2026-10-22T02:32:52Z","completed_at":"2026-10-22T04:49:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T04:56:49.050586653Z","timing_metadata_checked_at":"2026-10-22T04:56:49.050586653Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T04:56:49.055806135Z","region_metadata_checked_at":"2026-10-22T04:56:49.055806135Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113092279206940672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"09d80259fead3b847619e7206f95c90962c220b3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T02:17:11Z","started_at":"2026-10-22T02:17:11Z","completed_at":"2026-10-22T04:46:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T04:56:49.158498977Z","timing_metadata_checked_at":"2026-10-22T04:56:49.158498977Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T04:56:49.161650066Z","region_metadata_checked_at":"2026-10-22T04:56:49.161650066Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"a6a44b92014325abfbfa3ef0226f9cbe2c1bd77f8399e8d6b2fcdd26376d6937","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113092279206940672","occurred_at":"2026-10-22T02:17:11Z","failed_at":"alert","signature_id":"3f2c87dd944435486cb7cb53c2d27ce5177997363dcef8154e66da2bd6dddf52","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-22T03:42:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-b8qtg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5239caaa-164a-4f5a-831a-167d2502f038\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-b8qtg (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-22T03:42:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6940672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.63:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-vdkk2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"500be196-af9b-4978-b1e8-f05efb43b0f6\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-vdkk2 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-22T03:42:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6940672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.63:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-vdkk2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"500be196-af9b-4978-b1e8-f05efb43b0f6\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-vdkk2 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-22T03:42:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-b8qtg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5239caaa-164a-4f5a-831a-167d2502f038\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-b8qtg (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"adc26bdcf18b1f07f671dbe9fcc09de884365b9ec00b86503a1b4c1939475cc1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113092279206940672","occurred_at":"2026-10-22T02:17:11Z","failed_at":"alert","signature_id":"402cc04f24dfc6e5de29603c238a7b273c241bcac32793b0bf818ddb0ad63603","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 30 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T03:58:47Z\n Ended: 2026-10-22T04:06:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-22T03:58:47Z\n Ended: 2026-10-22T04:06:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-22T03:58:47Z\n Ended: 2026-10-22T04:04:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-22T03:58:47Z\n Ended: 2026-10-22T04:05:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-22T03:58:47Z\n Ended: 2026-10-22T04:05:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-22T03:58:47Z\n Ended: 2026-10-22T04:04:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-22T04:00:47Z\n Ended: 2026-10-22T04:06:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.63:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjj81ietnv2jbgeg13jpmpsis0ij8qo\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-67xzr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjj81ietnv2jbgeg13jpmpsis0ij8qo/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-22T04:17:44Z\n Ended: 2026-10-22T04:25:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-22T04:17:44Z\n Ended: 2026-10-22T04:28:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-22T04:17:44Z\n Ended: 2026-10-22T04:25:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-22T04:17:44Z\n Ended: 2026-10-22T04:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-22T04:17:44Z\n Ended: 2026-10-22T04:24:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-22T04:17:44Z\n Ended: 2026-10-22T04:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-22T04:17:45Z\n Ended: 2026-10-22T04:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-22T04:17:45Z\n Ended: 2026-10-22T04:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-22T04:17:45Z\n Ended: 2026-10-22T04:24:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-22T04:17:45Z\n Ended: 2026-10-22T04:28:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:26:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:29:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:25:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:25:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:24:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:25:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:24:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:26:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:29:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-22T04:18:47Z\n Ended: 2026-10-22T04:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6940672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.165:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672\", namespace=\"klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"e83d6b830ede1175e882fec802d880b917d87c47fdd588d1607006538a126755","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113092279206940672","occurred_at":"2026-10-22T02:17:11Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T03:40:46Z\n Ended: 2026-10-22T03:51:47Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6940672-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"f51a58ad8591340b73d596c8954886b5190919f029d99d4afda850ba13322c38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113092279206940672","occurred_at":"2026-10-22T02:17:11Z","failed_at":"e2e","signature_id":"91c10e9398dc1a118cf0ddb440a975ed51728767c12644ec941551e5b3bd3fe3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-jbvlzr\" for cluster \"cluster-candidate-5-1-jbvlzr\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x16e3bf104600\u003e: \n failed to create NodePool np-jbvlzr: failed waiting for nodepool=\"np-jbvlzr\" for cluster \"cluster-candidate-5-1-jbvlzr\" in resourcegroup=\"rg-candidate-5-1-jr4zj7tw52p6\" to finish creating: GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\",\n \"name\": \"905e6e4d-deed-4c98-88d4-7e7d915bea9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T03:36:41.547739466Z\",\n \"endTime\": \"2026-10-22T03:55:45.436892812Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-krx2m: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/providers/Microsoft.Network/networkInterfaces/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' se... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\n correlation ID: 0f6414e6-d1c6-4b3c-b1b7-40aee43cce2b\n client request ID: 57e0289e-c2ad-44a4-9341-dc67578bae73\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-jbvlzr\" for cluster \"cluster-candidate-5-1-jbvlzr\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x16e3bf104600\u003e: \n failed to create NodePool np-jbvlzr: failed waiting for nodepool=\"np-jbvlzr\" for cluster \"cluster-candidate-5-1-jbvlzr\" in resourcegroup=\"rg-candidate-5-1-jr4zj7tw52p6\" to finish creating: GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\",\n \"name\": \"905e6e4d-deed-4c98-88d4-7e7d915bea9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T03:36:41.547739466Z\",\n \"endTime\": \"2026-10-22T03:55:45.436892812Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-krx2m: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/providers/Microsoft.Network/networkInterfaces/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' se... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\n correlation ID: 0f6414e6-d1c6-4b3c-b1b7-40aee43cce2b\n client request ID: 57e0289e-c2ad-44a4-9341-dc67578bae73\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d25cf9b217ca7a1560a0ac8f986a95cd4726bd1277cdf1dc11174901dd19f0f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '10' se... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","7d25cf9b217ca7a1560a0ac8f986a95cd4726bd1277cdf1dc11174901dd19f0f","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113086770697474048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7573,"pr_state":"closed","pr_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","final_merged_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T01:55:20Z","started_at":"2026-10-22T01:55:20Z","completed_at":"2026-10-22T03:47:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T04:01:45.501719026Z","timing_metadata_checked_at":"2026-10-22T04:01:45.501719026Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T04:01:45.505162519Z","region_metadata_checked_at":"2026-10-22T04:01:45.505162519Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"4ced5d052cb84af07aa155895a35b6ecd65dfcd38dda55e35bb53ead3f0c4e56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113086770697474048","occurred_at":"2026-10-22T01:55:20Z","failed_at":"alert","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-22T02:57:07Z\n Ended: 2026-10-22T03:04:04Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7474048-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113077350928224256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6128,"pr_state":"open","pr_sha":"170f76835cc1a848764bf285f1e459b293c9fc67","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-22T01:17:54Z","started_at":"2026-10-22T01:17:54Z","completed_at":"2026-10-22T03:24:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T03:40:59.63935233Z","timing_metadata_checked_at":"2026-10-22T03:40:59.63935233Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T03:40:59.642195422Z","region_metadata_checked_at":"2026-10-22T03:40:59.642195422Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a23d6bbc51d0327c4a8f7c189348785ddb1edeaa1c2d8952f42423d3608dc0c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113077350928224256","occurred_at":"2026-10-22T01:17:54Z","failed_at":"e2e","signature_id":"e671cab1177a0493b74cb326ec698815954811c1abba3003abc61755f7f3f93b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-5h8rfd\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d0abaa07220\u003e: \n failed to create HCP cluster cluster-zstream-4-23-5h8rfd: failed waiting for cluster=\"cluster-zstream-4-23-5h8rfd\" in resourcegroup=\"rg-zstream-upgrade-4-23-5tgvrgshsr5n\" to finish creating: GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\n correlation ID: f39bfd1a-7a5d-4cc1-a489-fe05576c19eb\n client request ID: 9a94d97d-0e82-4889-a643-43d835a7d044\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-5h8rfd\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d0abaa07220\u003e: \n failed to create HCP cluster cluster-zstream-4-23-5h8rfd: failed waiting for cluster=\"cluster-zstream-4-23-5h8rfd\" in resourcegroup=\"rg-zstream-upgrade-4-23-5tgvrgshsr5n\" to finish creating: GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\n correlation ID: f39bfd1a-7a5d-4cc1-a489-fe05576c19eb\n client request ID: 9a94d97d-0e82-4889-a643-43d835a7d044\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113076587715891200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6772,"pr_state":"closed","pr_sha":"e0afb7920929a2f1a4d48fe5e209a487710d6739","final_merged_sha":"e0afb7920929a2f1a4d48fe5e209a487710d6739","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T01:14:52Z","started_at":"2026-10-22T01:14:52Z","completed_at":"2026-10-22T03:33:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T03:51:22.605311091Z","timing_metadata_checked_at":"2026-10-22T03:51:22.605311091Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T03:51:22.60920268Z","region_metadata_checked_at":"2026-10-22T03:51:22.60920268Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d27464b8184d4a1f52ef7e5d608808dbafdf0f6c4c28f88fc0df23a41ae78668","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113076587715891200","occurred_at":"2026-10-22T01:14:52Z","failed_at":"e2e","signature_id":"83d43091dcf6c3fabb39c58f779510f09c9b8b29f38f6f33dd93bee3852e5924","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1460b7c07300\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-twhbflrbjdzn\" to finish creating: GET https://rp.j5891200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6\",\n \"name\": \"b8bbf095-2cf3-4399-b879-67ffc150a1c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T02:18:02.50355295Z\",\n \"endTime\": \"2026-10-22T02:37:03.792952128Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-vg5d4-n6w85: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-n6w85 to exist\\nMachine ingress-cert-le-np-1-vg5d4-wjxjx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-wjxjx to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-vg5d4-wjxjx: Failed: availabilitysets failed to create or update. err: failed to get existing resource ingress-cert-le-twhbflrbjdzn--managed/2tji3utlagfblbjct3l9sosa37afh321_ingress-cert-le-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/availabilitySets/2tji3utlagfblbjct3l9sosa37afh321_ingress-cert-le-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '1' sec... message truncated\\nMachine ingress-cert-le-np-1-vg5d4-n6w85: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-n6w85 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5891200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6\n correlation ID: 93044a4f-4151-4014-a170-56a210eccfed\n client request ID: c5d784ea-0b44-447d-be61-01ac35e7274c\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1460b7c07300\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-twhbflrbjdzn\" to finish creating: GET https://rp.j5891200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6\",\n \"name\": \"b8bbf095-2cf3-4399-b879-67ffc150a1c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-22T02:18:02.50355295Z\",\n \"endTime\": \"2026-10-22T02:37:03.792952128Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-vg5d4-n6w85: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-n6w85 to exist\\nMachine ingress-cert-le-np-1-vg5d4-wjxjx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-wjxjx to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-vg5d4-wjxjx: Failed: availabilitysets failed to create or update. err: failed to get existing resource ingress-cert-le-twhbflrbjdzn--managed/2tji3utlagfblbjct3l9sosa37afh321_ingress-cert-le-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/availabilitySets/2tji3utlagfblbjct3l9sosa37afh321_ingress-cert-le-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '1' sec... message truncated\\nMachine ingress-cert-le-np-1-vg5d4-n6w85: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-n6w85 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5891200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6\n correlation ID: 93044a4f-4151-4014-a170-56a210eccfed\n client request ID: c5d784ea-0b44-447d-be61-01ac35e7274c","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-twhbflrbjdzn\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113075724997890048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6009,"pr_state":"open","pr_sha":"f0b1c5117f6606455c516a6143e8ed2f6c722be1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-22T01:11:26Z","started_at":"2026-10-22T01:11:26Z","completed_at":"2026-10-22T03:25:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T03:40:59.616521998Z","timing_metadata_checked_at":"2026-10-22T03:40:59.616521998Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T03:40:59.619860588Z","region_metadata_checked_at":"2026-10-22T03:40:59.619860588Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113071812228157440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7592,"pr_state":"closed","pr_sha":"c9a469c0a42acf6482344c2525dfc92f27189b30","final_merged_sha":"c9a469c0a42acf6482344c2525dfc92f27189b30","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T00:55:53Z","started_at":"2026-10-22T00:55:53Z","completed_at":"2026-10-22T03:14:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T03:30:13.514883462Z","timing_metadata_checked_at":"2026-10-22T03:30:13.514883462Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T03:30:13.518580354Z","region_metadata_checked_at":"2026-10-22T03:30:13.518580354Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3e970056600fd4f46d83b81ab8b7223733c7c5c87402dc0a9a1be95d80a348ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113071812228157440","occurred_at":"2026-10-22T00:55:53Z","failed_at":"e2e","signature_id":"4f129739bafddc5ac315cccd2d58e2f04f54ce8381b65dab5a26632d8e3d25a0","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x305427581240\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x305428017c50\u003e{\n s: \"dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x305427581240\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x305428017c50\u003e{\n s: \"dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"be0a16e09bfa0f5959b8c6c18e735d670abbd8db1c2e1fc90c731326c8bcaa62","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["be0a16e09bfa0f5959b8c6c18e735d670abbd8db1c2e1fc90c731326c8bcaa62"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7674/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113069370304368640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7674,"pr_state":"closed","pr_sha":"d57ce0fbf902f2a6d5ca99616dd6ba10c4d5b134","final_merged_sha":"d57ce0fbf902f2a6d5ca99616dd6ba10c4d5b134","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-22T00:46:11Z","started_at":"2026-10-22T00:46:11Z","completed_at":"2026-10-22T01:34:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T01:46:00.71048758Z","timing_metadata_checked_at":"2026-10-22T01:46:00.71048758Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T01:46:00.714792672Z","region_metadata_checked_at":"2026-10-22T01:46:00.714792672Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"df2754563464f0533daaedbfb27adf4de247f5ba2d4bb7985c476106a6f78dac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7674/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113069370304368640","occurred_at":"2026-10-22T00:46:11Z","failed_at":"provision","signature_id":"6b8d9984b55592851b0841c5e8c4a0f18f8f61520e7df4a51c7e6afc9a707725","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/kube-events","test_suite":"templatize-pipeline","failure_text":"time=2026-10-22T01:22:39.748Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events description=\"Step kube-events\\n Kind: Helm\\n\"\ntime=2026-10-22T01:22:40.100Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events values=\"map[enabled:true image:map[digest:sha256:f45f2e02544ba4005d39e435073a6c59ab5db372223dee379b08a36d3bc20871 registry:kubernetesshared.azurecr.io repository:shared/kube-events] resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]]]\"\ntime=2026-10-22T01:22:40.101Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-22T01:22:40.245Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events namespace=monitoring\ntime=2026-10-22T01:22:40.391Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events dryRun=false\ntime=2026-10-22T01:22:40.391Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-22T01:22:40.452Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-22T01:27:43.190Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err=\"resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-22T01:27:43.190Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a","failure_pattern":"failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}