{"meta":{"date":"2026-10-23","timezone":"UTC","generated_at":"2026-10-30T01:02:05Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":78,"failed_runs":53,"runs_with_occurrences":53,"runs_with_failure_pattern_matches":47,"runs_with_unmatched_occurrences":6,"runs_without_occurrence_rows":25,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7120/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113774695802212352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7120,"pr_state":"closed","pr_sha":"2e62dfb44ee3a1d443eb626604f2e79321982d6b","final_merged_sha":"2e62dfb44ee3a1d443eb626604f2e79321982d6b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T23:28:54Z","started_at":"2026-10-23T23:28:54Z","completed_at":"2026-10-24T00:42:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T00:48:12.361193065Z","timing_metadata_checked_at":"2026-10-24T00:48:12.361193065Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T00:48:12.36385286Z","region_metadata_checked_at":"2026-10-24T00:48:12.36385286Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b872704d614e310864295c79884573af68ac6db4551d3aaae23170fbfb010361","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7120/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113774695802212352","occurred_at":"2026-10-23T23:28:54Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-24T00:11:13.169Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-24T00:11:14.722Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-24T00:11:14.722Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-24T00:11:15.608Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-24T00:11:15.608Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:11:15.609Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j2212352-svc\ntime=2026-10-24T00:13:20.091Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-24T00:13:20.091Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=2m4s\ntime=2026-10-24T00:13:20.091Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.407Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-24T00:13:20.407Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.407Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.862Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-24T00:13:20.862Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.894Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-24T00:13:21.097Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.168Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j2212352-svc\ntime=2026-10-24T00:28:24.267Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230","failure_pattern":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113771192396550144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"1f5a50b2d3266a44d239ab6f3ee972a5e4c7385a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T23:14:58Z","started_at":"2026-10-23T23:14:58Z","completed_at":"2026-10-24T00:21:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T00:37:36.934889454Z","timing_metadata_checked_at":"2026-10-24T00:37:36.934889454Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T00:37:36.938360442Z","region_metadata_checked_at":"2026-10-24T00:37:36.938360442Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5456a38182c854b9240b182a16fa3551db571456f7a9d91c6e7a12aa1135c068","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113771192396550144","occurred_at":"2026-10-23T23:14:58Z","failed_at":"provision","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-24T00:10:57.890Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-24T00:10:58.200Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-24T00:10:58.200Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-24T00:10:58.456Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-24T00:10:58.679Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-24T00:10:58.679Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-24T00:10:58.736Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-24T00:16:02.844Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-24T00:16:02.844Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113768338508222464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7736,"pr_state":"closed","pr_sha":"ed7d121617ab665039646cdf53c36414616c413e","final_merged_sha":"ed7d121617ab665039646cdf53c36414616c413e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T23:03:38Z","started_at":"2026-10-23T23:03:38Z","completed_at":"2026-10-24T01:31:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T01:50:04.718046202Z","timing_metadata_checked_at":"2026-10-24T01:50:04.718046202Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T01:50:04.721574092Z","region_metadata_checked_at":"2026-10-24T01:50:04.721574092Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"83b160b5a8d941cdefd6d826d377074a58a9570e6232a253d55df32f363bc298","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113768338508222464","occurred_at":"2026-10-23T23:03:38Z","failed_at":"e2e","signature_id":"186c2f62dbba5354fdd50e3f198e2f5c7260e536efa09a9dba4dd236da4ad630","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x26e13640a140\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-jft4smjjlxxw\" to finish creating: GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\",\n \"name\": \"fb71e438-88d3-4b63-82a0-b820e717c220\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T00:13:46.41509851Z\",\n \"endTime\": \"2026-10-24T00:32:46.057566699Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\n correlation ID: 97b55a4e-bfeb-47d6-99bd-fc35a2cc6359\n client request ID: c7f67463-49ac-4db2-9fc4-28f026c30acf\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x26e13640a140\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-jft4smjjlxxw\" to finish creating: GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\",\n \"name\": \"fb71e438-88d3-4b63-82a0-b820e717c220\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T00:13:46.41509851Z\",\n \"endTime\": \"2026-10-24T00:32:46.057566699Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\n correlation ID: 97b55a4e-bfeb-47d6-99bd-fc35a2cc6359\n client request ID: c7f67463-49ac-4db2-9fc4-28f026c30acf","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f48bqq4pfgdf\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113759392745656320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7722,"pr_state":"closed","pr_sha":"df14005c558b392b82d592513411bdddc25a76dd","final_merged_sha":"df14005c558b392b82d592513411bdddc25a76dd","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T22:28:05Z","started_at":"2026-10-23T22:28:05Z","completed_at":"2026-10-24T00:59:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T01:19:18.217720373Z","timing_metadata_checked_at":"2026-10-24T01:19:18.217720373Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T01:19:18.228796843Z","region_metadata_checked_at":"2026-10-24T01:19:18.228796843Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7121/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113752610317012992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7121,"pr_state":"closed","pr_sha":"e4d28f07adf287586313aaefd41d8da31a598ee6","final_merged_sha":"e4d28f07adf287586313aaefd41d8da31a598ee6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T22:00:25Z","started_at":"2026-10-23T22:00:25Z","completed_at":"2026-10-24T00:22:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T00:37:35.739139211Z","timing_metadata_checked_at":"2026-10-24T00:37:35.739139211Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T00:37:35.742540299Z","region_metadata_checked_at":"2026-10-24T00:37:35.742540299Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7120/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113750398169780224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7120,"pr_state":"closed","pr_sha":"2e62dfb44ee3a1d443eb626604f2e79321982d6b","final_merged_sha":"2e62dfb44ee3a1d443eb626604f2e79321982d6b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T21:52:21Z","started_at":"2026-10-23T21:52:21Z","completed_at":"2026-10-23T22:44:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T23:03:27.644164546Z","timing_metadata_checked_at":"2026-10-23T23:03:27.644164546Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T23:03:27.648573733Z","region_metadata_checked_at":"2026-10-23T23:03:27.648573733Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"049c90a4e718f6f84058613ddb83fb73b00dc9d6592f999b1a713f3d6d3d6fcc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7120/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113750398169780224","occurred_at":"2026-10-23T21:52:21Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-23T22:20:23.920Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-23T22:20:25.642Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-23T22:20:25.642Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-23T22:20:26.982Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-23T22:20:26.982Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:20:26.982Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j9780224-svc\ntime=2026-10-23T22:23:03.498Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-23T22:23:03.499Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=2m37s\ntime=2026-10-23T22:23:03.499Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:04.054Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-23T22:23:04.054Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:04.054Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:04.811Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-23T22:23:04.811Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:04.874Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-23T22:23:05.250Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:05.375Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:06.263Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:06.263Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:06.263Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:06.263Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:06.263Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j9780224-svc\ntime=2026-10-23T22:38:09.485Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230","failure_pattern":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113746348372135936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7733,"pr_state":"closed","pr_sha":"60e48794e39b55e9d91fcf9341f73e41d6e52c64","final_merged_sha":"60e48794e39b55e9d91fcf9341f73e41d6e52c64","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T21:36:15Z","started_at":"2026-10-23T21:36:15Z","completed_at":"2026-10-23T23:51:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T00:05:23.532089038Z","timing_metadata_checked_at":"2026-10-24T00:05:23.532089038Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T00:05:23.535485128Z","region_metadata_checked_at":"2026-10-24T00:05:23.535485128Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"288b8bd106e54585f0471ad10b9f7cce02be3cde06bec85b2265d1be3e6d6e48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113746348372135936","occurred_at":"2026-10-23T21:36:15Z","failed_at":"e2e","signature_id":"0df12b245e7e51211310f93e12a460611030c5368b7e9d6d2e212e20fb97294f","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x35e0b0894420\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-9pzwhwthgd8h\" to finish creating: GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\",\n \"name\": \"4735d84e-eeda-4991-9a00-15bdfb0112df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T22:59:37.48096278Z\",\n \"endTime\": \"2026-10-23T23:18:39.961096304Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n correlation ID: e810e9b2-3fca-4c24-b421-0b45eb552e0e\n client request ID: 0af33e2e-2abb-488f-a355-27d28c97d1c5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x35e0b0894420\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-9pzwhwthgd8h\" to finish creating: GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\",\n \"name\": \"4735d84e-eeda-4991-9a00-15bdfb0112df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T22:59:37.48096278Z\",\n \"endTime\": \"2026-10-23T23:18:39.961096304Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n correlation ID: e810e9b2-3fca-4c24-b421-0b45eb552e0e\n client request ID: 0af33e2e-2abb-488f-a355-27d28c97d1c5\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"853c5a0981a4f4f757664ffdceb9d9daa8cd196e25af5d82fbc0ff87b37b75da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113746348372135936","occurred_at":"2026-10-23T21:36:15Z","failed_at":"e2e","signature_id":"ffad9075f8b073e455e1ac724969c29ae57f1247a7dc73dc4b06d2983015219a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n \u003c*errors.StatusError | 0x157c9dc7bc20\u003e: \n Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"\n {\n ErrStatus: {\n TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n Reason: \"InternalError\",\n Details: {\n Name: \"\",\n Group: \"\",\n Kind: \"\",\n UID: \"\",\n Causes: [\n {\n Type: \"\",\n Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n Field: \"\",\n },\n ],\n RetryAfterSeconds: 0,\n },\n Code: 500,\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n \u003c*errors.StatusError | 0x157c9dc7bc20\u003e: \n Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"\n {\n ErrStatus: {\n TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n Reason: \"InternalError\",\n Details: {\n Name: \"\",\n Group: \"\",\n Kind: \"\",\n UID: \"\",\n Causes: [\n {\n Type: \"\",\n Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n Field: \"\",\n },\n ],\n RetryAfterSeconds: 0,\n },\n Code: 500,\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1ae9bcc864e0b95b17cfb70f07fc859ef6b54ed4d13d4886751e4ae7fe6a17c0","failure_pattern":"Internal error occurred: failed calling webhook \"webhook","search_query":"Internal error occurred: failed calling webhook \"webhook"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1ae9bcc864e0b95b17cfb70f07fc859ef6b54ed4d13d4886751e4ae7fe6a17c0","1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113744756189171712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T21:29:55Z","started_at":"2026-10-23T21:29:55Z","completed_at":"2026-10-24T00:00:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T00:15:40.150518333Z","timing_metadata_checked_at":"2026-10-24T00:15:40.150518333Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T00:15:40.154122621Z","region_metadata_checked_at":"2026-10-24T00:15:40.154122621Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0794f9132643aaf1523e8634fbb0a4a67b8c8d814565c0f78c324df178c77864","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113744756189171712","occurred_at":"2026-10-23T21:29:55Z","failed_at":"alert","signature_id":"b6d85dbc1d4502551493043fa33721464a3b1282465557d679ae70789ff0b520","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T22:38:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9171712-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-cqcsl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"24c5a63c-ba67-481c-97bd-1b2284e4616c\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-cqcsl (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T22:40:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9171712-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-cqcsl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"24c5a63c-ba67-481c-97bd-1b2284e4616c\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-cqcsl (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-23T22:41:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9171712-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-l6h2t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"dc5bdfb6-fff6-4941-9c02-12766a972bae\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-l6h2t (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-23T22:43:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9171712-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-l6h2t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"dc5bdfb6-fff6-4941-9c02-12766a972bae\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-l6h2t (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"7db4e3c4cada51d3b0dd92c8070354388ee49875afe144096637693382c17828","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113744756189171712","occurred_at":"2026-10-23T21:29:55Z","failed_at":"e2e","signature_id":"0d5ace1ca3dc48db2d8bb7f3ca58f25eef51b628008732668244b790b4cfd770","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x348180f2e220\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*verifiers.pollObservationError | 0x348180f2e200\u003e{\n observation: \u003c*errors.errorString | 0x3481807fdb50\u003e{\n s: \"dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n interruption: \u003c*fmt.wrapError | 0x348180f2e1c0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x3481808801e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x348180f2e1a0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x3481828e4120\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x348180f2e220\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*verifiers.pollObservationError | 0x348180f2e200\u003e{\n observation: \u003c*errors.errorString | 0x3481807fdb50\u003e{\n s: \"dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n interruption: \u003c*fmt.wrapError | 0x348180f2e1c0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x3481808801e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x348180f2e1a0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x3481828e4120\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"af2d21e1e88696d05be0ca90beb20ce563a5bf796115e15a05448c96cb57eced","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","af2d21e1e88696d05be0ca90beb20ce563a5bf796115e15a05448c96cb57eced"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113738239658430464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7730,"pr_state":"open","pr_sha":"425946cefa2ca1aa9aa2ab804c3c6fa6a4e468eb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T21:04:02Z","started_at":"2026-10-23T21:04:02Z","completed_at":"2026-10-23T21:21:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T21:40:10.476699463Z","timing_metadata_checked_at":"2026-10-23T21:40:10.476699463Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-23T21:40:10.482339847Z","region_metadata_checked_at":"2026-10-23T22:42:44.61297521Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5a6dcd8b6b90520e49b2512ac789c04eea55cb7bca3f9443ee43f84e2fcf05af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113738239658430464","occurred_at":"2026-10-23T21:04:02Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113735254979121152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6770,"pr_state":"open","pr_sha":"b2f6a196346b1cee013e216591f54a34f1167689","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T20:52:10Z","started_at":"2026-10-23T20:52:10Z","completed_at":"2026-10-23T23:51:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T00:05:23.344365596Z","timing_metadata_checked_at":"2026-10-24T00:05:23.344365596Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T00:05:23.352013373Z","region_metadata_checked_at":"2026-10-24T00:05:23.352013373Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"8956f4d7fd2c3fdcf2f4eebf84d8a4a149e09d62b46e3d96ddd82048a4a9439c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113735254979121152","occurred_at":"2026-10-23T20:52:10Z","failed_at":"e2e","signature_id":"b819a089f33550631ed5b7c4847276496e869cbd6918d37a9176dd007ad100fd","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xac144fee4a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0xac1449cf1d0\u003e{\n s: \"dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xac144fee4a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0xac1449cf1d0\u003e{\n s: \"dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6b19f3be20271a1946ad6e5da34c15339012b8d43c1a7baf3efcab4e3e251f7","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}},{"row_id":"f4ec19dfdf50d3136e0590317938bd82e7479119066e6f55195fda21abe2cc63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113735254979121152","occurred_at":"2026-10-23T20:52:10Z","failed_at":"e2e","signature_id":"24816c79684a0f74c02483d2e3b9365bf73a0c2a88310b47fae43e1866fb1d1a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1355449513d0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1355449513d0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","f6b19f3be20271a1946ad6e5da34c15339012b8d43c1a7baf3efcab4e3e251f7"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113727107086422016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7724,"pr_state":"closed","pr_sha":"10c65bf8645ea92d95fe9f0900f66951e3173c17","final_merged_sha":"10c65bf8645ea92d95fe9f0900f66951e3173c17","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T20:19:48Z","started_at":"2026-10-23T20:19:48Z","completed_at":"2026-10-23T22:51:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T23:03:27.9794965Z","timing_metadata_checked_at":"2026-10-23T23:03:27.9794965Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T23:03:27.982371791Z","region_metadata_checked_at":"2026-10-23T23:03:27.982371791Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113725433458462720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7547,"pr_state":"closed","pr_sha":"28c20c5eff68a4950667d98fdfc5163f1798bdff","final_merged_sha":"cd3c21e53a196ba0ad2c38b6f73f33b744c9e3fa","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T20:13:09Z","started_at":"2026-10-23T20:13:09Z","completed_at":"2026-10-23T22:24:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T22:32:43.288898732Z","timing_metadata_checked_at":"2026-10-23T22:32:43.288898732Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T22:32:43.291659124Z","region_metadata_checked_at":"2026-10-23T22:32:43.291659124Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113725096341278720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7371,"pr_state":"closed","pr_sha":"5cd5b088b6de4c1d101879be129286fe2d305070","final_merged_sha":"42b177e82e37e27d732edd9dee43c759e521c0b1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T20:11:48Z","started_at":"2026-10-23T20:11:48Z","completed_at":"2026-10-23T22:41:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T22:52:53.390969044Z","timing_metadata_checked_at":"2026-10-23T22:52:53.390969044Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T22:52:53.394020333Z","region_metadata_checked_at":"2026-10-23T22:52:53.394020333Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"bc0cf0e25d0163b4a21461b666b6984511206da85039fedfa8426b527a822659","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113725096341278720","occurred_at":"2026-10-23T20:11:48Z","failed_at":"e2e","signature_id":"e4cd001bc94a7c7f7d880d8bd2124ba820cefb9f6079b7e26d5709adbf7dc519","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:328]: failed to create worker nodepool for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x204486f8560\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-wrxzvkddb94j\" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\",\n \"name\": \"13760d44-2d42-4c53-9fca-8d97381d23a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T21:19:36.261324147Z\",\n \"endTime\": \"2026-10-23T21:38:45.545540127Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\n correlation ID: 5d55a2cb-56aa-47f7-a5b9-cf7b9a27d226\n client request ID: f043a8e7-f2d1-473c-b2e3-11abda25efb3\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:328]: failed to create worker nodepool for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x204486f8560\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-wrxzvkddb94j\" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\",\n \"name\": \"13760d44-2d42-4c53-9fca-8d97381d23a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T21:19:36.261324147Z\",\n \"endTime\": \"2026-10-23T21:38:45.545540127Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\n correlation ID: 5d55a2cb-56aa-47f7-a5b9-cf7b9a27d226\n client request ID: f043a8e7-f2d1-473c-b2e3-11abda25efb3\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dfc814625c6feb3befd3ba6fee9aefad6e5bcb1f46bb84fd97660adb52ac90b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113725096341278720","occurred_at":"2026-10-23T20:11:48Z","failed_at":"e2e","signature_id":"96d11e5d24c2f96b51ede935923d2da2885cc5293b4b2067a83db979216fe98e","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3dd71ecc7860\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-gvwnv545c9xf\" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\",\n \"name\": \"b77ed110-707e-4c38-ab0a-33ad172ee68c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T21:19:37.486805843Z\",\n \"endTime\": \"2026-10-23T21:38:45.647369611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\n correlation ID: 6572f7b9-3e5a-422f-aa65-926c920bbad1\n client request ID: bb39b90a-4ada-4996-91b9-6bd38bf61535\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3dd71ecc7860\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-gvwnv545c9xf\" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\",\n \"name\": \"b77ed110-707e-4c38-ab0a-33ad172ee68c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T21:19:37.486805843Z\",\n \"endTime\": \"2026-10-23T21:38:45.647369611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\n correlation ID: 6572f7b9-3e5a-422f-aa65-926c920bbad1\n client request ID: bb39b90a-4ada-4996-91b9-6bd38bf61535\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113718483719032832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7177,"pr_state":"closed","pr_sha":"27fe8fe46cb154c4b4ff583dea5a4d9ad0f93dd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T19:45:32Z","started_at":"2026-10-23T19:45:32Z","completed_at":"2026-10-23T22:40:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T22:52:53.088550613Z","timing_metadata_checked_at":"2026-10-23T22:52:53.088550613Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T22:52:53.094372393Z","region_metadata_checked_at":"2026-10-23T22:52:53.094372393Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113714121043087360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7722,"pr_state":"closed","pr_sha":"df14005c558b392b82d592513411bdddc25a76dd","final_merged_sha":"df14005c558b392b82d592513411bdddc25a76dd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T19:28:05Z","started_at":"2026-10-23T19:28:05Z","completed_at":"2026-10-23T21:33:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T21:40:10.01702868Z","timing_metadata_checked_at":"2026-10-23T21:40:10.01702868Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T21:40:10.019583973Z","region_metadata_checked_at":"2026-10-23T21:40:10.019583973Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fd3d14190197148ecc19aadbd739ef4e7ab9789acb0a69503ea4d44824ccc53d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113714121043087360","occurred_at":"2026-10-23T19:28:05Z","failed_at":"e2e","signature_id":"bb2f64cac1081e7e6bb8f566f3bd4023215ffc8e942c129ba96fe59be1320761","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xa03dff15100\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-k98p95t7rwlk\" to finish creating: GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\",\n \"name\": \"1937a216-e9d1-46bb-b9f8-a55a34e91da8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T20:32:20.327838457Z\",\n \"endTime\": \"2026-10-23T20:51:29.045056606Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2p298-8vgqx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2p298-8vgqx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\n correlation ID: f3bdc38e-f649-43a0-824c-a60a987dceb4\n client request ID: f8924ec9-c563-43bf-b506-67458e3fbd1f\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xa03dff15100\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-k98p95t7rwlk\" to finish creating: GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\",\n \"name\": \"1937a216-e9d1-46bb-b9f8-a55a34e91da8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T20:32:20.327838457Z\",\n \"endTime\": \"2026-10-23T20:51:29.045056606Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2p298-8vgqx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2p298-8vgqx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\n correlation ID: f3bdc38e-f649-43a0-824c-a60a987dceb4\n client request ID: f8924ec9-c563-43bf-b506-67458e3fbd1f","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f48bqq4pfgdf\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7727/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113711841971539968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7727,"pr_state":"closed","pr_sha":"ca08ced119bb479f5b000757cde7279a437dd0f7","final_merged_sha":"a37be9689955179d7d10d530e2021e6edd24375a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T19:19:08Z","started_at":"2026-10-23T19:19:08Z","completed_at":"2026-10-23T21:29:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T21:40:10.223748588Z","timing_metadata_checked_at":"2026-10-23T21:40:10.223748588Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T21:40:10.22631778Z","region_metadata_checked_at":"2026-10-23T21:40:10.22631778Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"622a7d48b220b8232fc32161a4d99b39f35b183152f03f98eac07290738a9f1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7727/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113711841971539968","occurred_at":"2026-10-23T19:19:08Z","failed_at":"e2e","signature_id":"06405e4c71e72b70733326b19524a23a628ce5d6d98f1e230f7156ce01e4dc2e","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster \"cluster-hs-pre-7lnp9z\" should complete its version rollout\nUnexpected error:\n \u003c*fmt.wrapError | 0x1290b3e7a900\u003e: \n VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\n {\n msg: \"VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n err: \u003c*verifiers.pollObservationError | 0x1290b3e7a840\u003e{\n observation: \u003c*errors.errorString | 0x1290b436a5d0\u003e{\n s: \"clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n },\n interruption: \u003c*fmt.wrapError | 0x1290b3e7a820\u003e{\n msg: \"failed to get clusterversion \\\"version\\\": context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster \"cluster-hs-pre-7lnp9z\" should complete its version rollout\nUnexpected error:\n \u003c*fmt.wrapError | 0x1290b3e7a900\u003e: \n VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\n {\n msg: \"VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n err: \u003c*verifiers.pollObservationError | 0x1290b3e7a840\u003e{\n observation: \u003c*errors.errorString | 0x1290b436a5d0\u003e{\n s: \"clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n },\n interruption: \u003c*fmt.wrapError | 0x1290b3e7a820\u003e{\n msg: \"failed to get clusterversion \\\"version\\\": context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e9b156542f61a22d88e8e787cb5184aae92eaf130a724dfa3a85c4427efdde52","failure_pattern":"VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e9b156542f61a22d88e8e787cb5184aae92eaf130a724dfa3a85c4427efdde52"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113710890254602240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7573,"pr_state":"closed","pr_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","final_merged_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T19:15:21Z","started_at":"2026-10-23T19:15:21Z","completed_at":"2026-10-23T21:37:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T21:40:09.835051002Z","timing_metadata_checked_at":"2026-10-23T21:40:09.835051002Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T21:40:09.838110393Z","region_metadata_checked_at":"2026-10-23T21:40:09.838110393Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T18:38:10Z","started_at":"2026-10-23T18:38:10Z","completed_at":"2026-10-23T20:45:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T20:47:09.344921866Z","timing_metadata_checked_at":"2026-10-23T20:47:09.344921866Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T20:47:09.348670356Z","region_metadata_checked_at":"2026-10-23T20:47:09.348670356Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"29c64df2fccfa99f31c121203a61866ed9a88f910e15f64197537201b12cd64c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","failed_at":"e2e","signature_id":"97dc8f8152c396287fe01fe00d78417ce23034d6cc4cad15e28215178e3bc6eb","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2b8958bd95b0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2b8958bd95b0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24f84e8da288b18c5a6ce017a06862a7753a25291f7369a28e80acece5813bd2","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0"}},{"row_id":"31a329f696467577bdc278b3d28c654431cdaf84c67a78b857a96e31c72ca77d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","failed_at":"alert","signature_id":"bf583365df567c4528941bfba0cb382d9ef8ebcb3950c01907cf643a249725b2","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T19:51:45Z\n Ended: 2026-10-23T20:15:45Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T19:51:45Z\n Ended: 2026-10-23T20:15:45Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"6085bfb3e27b2d5a74d989c6e6f92efe4c567e1584c1930c5332a1747d219a6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","failed_at":"e2e","signature_id":"2871aba458073a19748dc486b5fa832486d5dacb815d2e076d0ba71820b99e15","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-22-hbhgzqp5zds5/cluster-candidate-4-22-9t7f9b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2655fb8fc880\u003e: \n failed to create HCP cluster cluster-candidate-4-22-9t7f9b: failed waiting for cluster=\"cluster-candidate-4-22-9t7f9b\" in resourcegroup=\"rg-candidate-4-22-hbhgzqp5zds5\" to finish creating: GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\",\n \"name\": \"9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T19:29:36.620172443Z\",\n \"endTime\": \"2026-10-23T19:58:43.098929016Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\n correlation ID: 91833dfb-2346-485c-9abf-90635cc143aa\n client request ID: 8cd6038e-c1e2-4db7-aaf4-36204667b8f5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-22-hbhgzqp5zds5/cluster-candidate-4-22-9t7f9b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2655fb8fc880\u003e: \n failed to create HCP cluster cluster-candidate-4-22-9t7f9b: failed waiting for cluster=\"cluster-candidate-4-22-9t7f9b\" in resourcegroup=\"rg-candidate-4-22-hbhgzqp5zds5\" to finish creating: GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\",\n \"name\": \"9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T19:29:36.620172443Z\",\n \"endTime\": \"2026-10-23T19:58:43.098929016Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\n correlation ID: 91833dfb-2346-485c-9abf-90635cc143aa\n client request ID: 8cd6038e-c1e2-4db7-aaf4-36204667b8f5\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8519de070e400b2e1895c8587740c3cefd463defc67166ec08a70a549082e3af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","failed_at":"alert","signature_id":"5f39e806b7aca5a0dc9a2e9d8787423dbc4725456fa3be8c785dbeffe0c743c5","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T19:53:47Z\n Ended: 2026-10-23T20:19:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ad470ce0-381a-4372-b725-f995a508a24f\"\n Description: Pod ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-23T19:53:47Z\n Ended: 2026-10-23T20:19:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ad470ce0-381a-4372-b725-f995a508a24f\"\n Description: Pod ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"89615c1bd3079cfa0d3f0607c6d034df640b2dc1bf4cd7d5e121fbdb5cf3dabf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","failed_at":"alert","signature_id":"2286281296e2588a0e23ff286161d911dea899212cf719d1a69d41fc8aea0670","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T20:05:44Z\n Ended: 2026-10-23T20:17:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T20:05:45Z\n Ended: 2026-10-23T20:17:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"be11a21e0d1e018b2fd313fd4bc51fbc7dbd8d897e46b32887f39fb0af2fd3e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","failed_at":"alert","signature_id":"2ec5e8a05ec938e5d3691a4dec2f741db485c232fb0b69893230211b7263938c","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T19:45:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-wpx95\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f99f4808-64ca-4f14-b866-30cd9d5a44aa\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-wpx95 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T19:48:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-b4nrw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b83bb111-3a9c-4b96-b2ca-76053771e1a5\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-b4nrw (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-23T19:48:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-b4nrw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b83bb111-3a9c-4b96-b2ca-76053771e1a5\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-b4nrw (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-23T19:51:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-wpx95\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f99f4808-64ca-4f14-b866-30cd9d5a44aa\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-wpx95 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","24f84e8da288b18c5a6ce017a06862a7753a25291f7369a28e80acece5813bd2","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113683564749918208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7371,"pr_state":"closed","pr_sha":"5cd5b088b6de4c1d101879be129286fe2d305070","final_merged_sha":"42b177e82e37e27d732edd9dee43c759e521c0b1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T17:26:46Z","started_at":"2026-10-23T17:26:46Z","completed_at":"2026-10-23T19:46:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T19:54:41.739610157Z","timing_metadata_checked_at":"2026-10-23T19:54:41.739610157Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T19:54:41.742453047Z","region_metadata_checked_at":"2026-10-23T19:54:41.742453047Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d7be0bc6a494d651e80e9a1196c02a5dea4d1dc4fcc3aa5dd62d65631cf9c562","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113683564749918208","occurred_at":"2026-10-23T17:26:46Z","failed_at":"e2e","signature_id":"aa04e18b7fbd233f6ca4b1e47d82cf0953e640b1a2ab6e85b1475ab4208f342f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-dzr6vp\"\nUnexpected error:\n \u003c*errors.joinError | 0x3fde2a0147b0\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x3fde2a4d4080\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x3fde2a4d4040\u003e{\n msg: \"failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x3fde2a4d4020\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x3fde2aa98140\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-dzr6vp\"\nUnexpected error:\n \u003c*errors.joinError | 0x3fde2a0147b0\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x3fde2a4d4080\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x3fde2a4d4040\u003e{\n msg: \"failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x3fde2a4d4020\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x3fde2aa98140\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113683432016973824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T17:26:03Z","started_at":"2026-10-23T17:26:03Z","completed_at":"2026-10-23T18:33:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T18:41:13.329803442Z","timing_metadata_checked_at":"2026-10-23T18:41:13.329803442Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T18:41:13.333463932Z","region_metadata_checked_at":"2026-10-23T18:41:13.333463932Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ce417f15220b7f068cac40ec17ec318e6f4b08dad19fd2db5c0664ccdd4c7b73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113683432016973824","occurred_at":"2026-10-23T17:26:03Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-23T18:08:55.483Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-23T18:08:56.785Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-23T18:08:56.785Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-23T18:08:57.580Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-23T18:08:57.580Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:08:57.580Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j6973824-svc\ntime=2026-10-23T18:13:02.175Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-23T18:13:02.175Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=4m5s\ntime=2026-10-23T18:13:02.175Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:02.795Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-23T18:13:02.795Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:02.795Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:03.759Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-23T18:13:03.759Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:03.939Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-23T18:13:04.167Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.333Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.935Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.935Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.935Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.935Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.935Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j6973824-svc\ntime=2026-10-23T18:28:07.753Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230","failure_pattern":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113683046036148224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7704,"pr_state":"closed","pr_sha":"e7c23babe29805031def86c975833c6d3badf2cb","final_merged_sha":"e7c23babe29805031def86c975833c6d3badf2cb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T17:24:43Z","started_at":"2026-10-23T17:24:43Z","completed_at":"2026-10-23T19:28:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T19:33:46.41613331Z","timing_metadata_checked_at":"2026-10-23T19:33:46.41613331Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T19:33:46.422394993Z","region_metadata_checked_at":"2026-10-23T19:33:46.422394993Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680800309317632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7657,"pr_state":"open","pr_sha":"1de2577a7c0ba79238d6d8e854e960afc56e2cb8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T17:15:47Z","started_at":"2026-10-23T17:15:47Z","completed_at":"2026-10-23T19:39:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T19:44:21.598378042Z","timing_metadata_checked_at":"2026-10-23T19:44:21.598378042Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T19:44:21.601435431Z","region_metadata_checked_at":"2026-10-23T19:44:21.601435431Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"3ee878a1e406a0200cc1773d09e3dd2845faf0209161da5b1cdffaa9340a261e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680800309317632","occurred_at":"2026-10-23T17:15:47Z","failed_at":"e2e","signature_id":"30e7ab70c0561d3688101981f498a87ca5fd5787330e0b440c1bbb06abf887df","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-sn4xdl\" for cluster \"cluster-candidate-4-21-sn4xdl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x60436518ee0\u003e: \n failed to create NodePool np-sn4xdl: failed waiting for nodepool=\"np-sn4xdl\" for cluster \"cluster-candidate-4-21-sn4xdl\" in resourcegroup=\"rg-candidate-4-21-c4tsdrwbnnfb\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\",\n \"name\": \"d83a0f27-909b-4319-9bb2-43e65d178f6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:34:43.074031046Z\",\n \"endTime\": \"2026-10-23T18:53:43.987521771Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\n correlation ID: 327dae19-27b0-4a9b-8e18-5ab3841e0ec7\n client request ID: 2803ecfb-10a6-4276-a6a2-935c10a7ca77\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-sn4xdl\" for cluster \"cluster-candidate-4-21-sn4xdl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x60436518ee0\u003e: \n failed to create NodePool np-sn4xdl: failed waiting for nodepool=\"np-sn4xdl\" for cluster \"cluster-candidate-4-21-sn4xdl\" in resourcegroup=\"rg-candidate-4-21-c4tsdrwbnnfb\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\",\n \"name\": \"d83a0f27-909b-4319-9bb2-43e65d178f6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:34:43.074031046Z\",\n \"endTime\": \"2026-10-23T18:53:43.987521771Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\n correlation ID: 327dae19-27b0-4a9b-8e18-5ab3841e0ec7\n client request ID: 2803ecfb-10a6-4276-a6a2-935c10a7ca77\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a7eaf4ff554d0aaa765aba5c6bb7e6b8d74e9e544fb89e1c56e50a8a43830b5e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680800309317632","occurred_at":"2026-10-23T17:15:47Z","failed_at":"e2e","signature_id":"edc4a2a86804f16bb2ca5ef5fec66418b27ca460d596142f26648f4d317f99da","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x21c5dc2ca7e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f88rfw7wtj4v\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\",\n \"name\": \"b02b5a9b-3f7f-467b-bc65-f56966e21cc5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:32:08.161782793Z\",\n \"endTime\": \"2026-10-23T18:51:14.083013713Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-f88rfw7wtj4v--managed/ingress-cert-le-np-1-b4g5h-9bn6x-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-f88rfw7wtj4v--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-b4g5h-9bn6x-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\n correlation ID: 053e3b77-21ef-4381-b997-ce492e16aa00\n client request ID: d87da6eb-f6e0-450c-b35a-d8d6ce034e64\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x21c5dc2ca7e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f88rfw7wtj4v\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\",\n \"name\": \"b02b5a9b-3f7f-467b-bc65-f56966e21cc5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:32:08.161782793Z\",\n \"endTime\": \"2026-10-23T18:51:14.083013713Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-f88rfw7wtj4v--managed/ingress-cert-le-np-1-b4g5h-9bn6x-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-f88rfw7wtj4v--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-b4g5h-9bn6x-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\n correlation ID: 053e3b77-21ef-4381-b997-ce492e16aa00\n client request ID: d87da6eb-f6e0-450c-b35a-d8d6ce034e64","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f48bqq4pfgdf\" to finish creating: GET https://rp"}},{"row_id":"c7c207b37dbd1d396783438e6cd497301d6c655c23a79ef9f0fba082f8276154","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680800309317632","occurred_at":"2026-10-23T17:15:47Z","failed_at":"e2e","signature_id":"e1da39dbfa2686ab72adc9980862457714384b0d27434b03e987e157712cc1b9","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x10298a136780\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-thv2btpwv2x8\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\",\n \"name\": \"505bd364-3947-46d9-aabe-9bb9ef0e8d13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:20:41.038643697Z\",\n \"endTime\": \"2026-10-23T18:49:43.826799473Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 9m9s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\n correlation ID: fb84c414-e895-47e3-bdef-c718ca7da71b\n client request ID: e0f67746-2722-4f93-b4a1-3c122c4d2e69\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x10298a136780\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-thv2btpwv2x8\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\",\n \"name\": \"505bd364-3947-46d9-aabe-9bb9ef0e8d13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:20:41.038643697Z\",\n \"endTime\": \"2026-10-23T18:49:43.826799473Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 9m9s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\n correlation ID: fb84c414-e895-47e3-bdef-c718ca7da71b\n client request ID: e0f67746-2722-4f93-b4a1-3c122c4d2e69\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7722,"pr_state":"closed","pr_sha":"df14005c558b392b82d592513411bdddc25a76dd","final_merged_sha":"df14005c558b392b82d592513411bdddc25a76dd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T17:12:53Z","started_at":"2026-10-23T17:12:53Z","completed_at":"2026-10-23T19:11:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T19:23:43.671843707Z","timing_metadata_checked_at":"2026-10-23T19:23:43.671843707Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T19:23:43.675155198Z","region_metadata_checked_at":"2026-10-23T19:23:43.675155198Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2f63dce4cc5a8dab55a8fc4b58f57d263af998dbb41c5a5307aa17b689998c33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","failed_at":"e2e","signature_id":"7e722ad5a4ed9ab1a5b378b6eaaa935525e7b9867c93ed401690a68dd2ba6792","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x29d3447c6480\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-gbjmnzzdfntv\" to finish creating: GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\",\n \"name\": \"644d90e9-61e3-42ef-a2dd-083bda41c8ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T17:53:58.718650875Z\",\n \"endTime\": \"2026-10-23T18:23:02.815422763Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\n correlation ID: 6d68ce4a-22a8-4eea-a977-b246a66b35ea\n client request ID: 9a628c19-f8fd-4bf6-859e-bb2f0567065e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x29d3447c6480\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-gbjmnzzdfntv\" to finish creating: GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\",\n \"name\": \"644d90e9-61e3-42ef-a2dd-083bda41c8ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T17:53:58.718650875Z\",\n \"endTime\": \"2026-10-23T18:23:02.815422763Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\n correlation ID: 6d68ce4a-22a8-4eea-a977-b246a66b35ea\n client request ID: 9a628c19-f8fd-4bf6-859e-bb2f0567065e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5533139caf247a6ca8444d533d47b407147e4fb0e7212689be79c976866021bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","failed_at":"alert","signature_id":"b9051e9b8ccf6adbe45aedab22f7090fbb17021fbbdd41caae75f1e8b7eda5f2","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T18:28:28Z\n Ended: 2026-10-23T18:40:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T18:28:28Z\n Ended: 2026-10-23T18:40:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"6ec8e687df7eb940d20aeeab0a421aa36ed2a14dbe62740029fd3fdd62a8e8dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","failed_at":"alert","signature_id":"ed02ebe0267da497eb0e3be41998c6fcb596d452363ba5bd7181d595e7764413","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T18:17:28Z\n Ended: 2026-10-23T18:44:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9b5be082-3743-4901-bdab-8d87701f28f1\"\n Description: Pod ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-23T18:17:28Z\n Ended: 2026-10-23T18:44:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9b5be082-3743-4901-bdab-8d87701f28f1\"\n Description: Pod ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"84a3897a89851e60faa7f6b79254a2524c66a1353eb25f721b66cf0b5abf7a97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","failed_at":"alert","signature_id":"4b46ae0c9f8b5ffa392131441a033fd3d6fba4c8a02a7cf64403d6e16934d074","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T18:14:34Z\n Ended: 2026-10-23T18:39:29Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T18:14:35Z\n Ended: 2026-10-23T18:39:29Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7547,"pr_state":"closed","pr_sha":"28c20c5eff68a4950667d98fdfc5163f1798bdff","final_merged_sha":"cd3c21e53a196ba0ad2c38b6f73f33b744c9e3fa","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T17:04:04Z","started_at":"2026-10-23T17:04:04Z","completed_at":"2026-10-23T19:09:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T19:13:32.732584213Z","timing_metadata_checked_at":"2026-10-23T19:13:32.732584213Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T19:13:32.738088796Z","region_metadata_checked_at":"2026-10-23T19:13:32.738088796Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"322a128d5046f8d0239d5a5955e9ee3461fb8e4d8469c7fc3b71e60924b48eb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","failed_at":"e2e","signature_id":"86725f75fec081c22e7a197ddb0df251563b2a69bc88d75ade0edf6269655c72","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc108f3a63c0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-9rjzw7\" in resourcegroup=\"private-ingress-zqfzzkb8lcwr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20261023T175235Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9bc1794b-fa34-4d7e-af9a-8ae43e2a8056', request correlation id '12c52ee3-c818-4a7a-b25e-ae843a2763d9'.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n correlation ID: f4e0dcee-95e8-42c0-8753-7683f8dd661f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc108f3a63c0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-9rjzw7\" in resourcegroup=\"private-ingress-zqfzzkb8lcwr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20261023T175235Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9bc1794b-fa34-4d7e-af9a-8ae43e2a8056', request correlation id '12c52ee3-c818-4a7a-b25e-ae843a2763d9'.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n correlation ID: f4e0dcee-95e8-42c0-8753-7683f8dd661f\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0003cc8ca3a9f7347553bcc0638ed02c32204aa3c9fd097c1f10d2bf5392fbd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8a00ccf9f6e4f7f79b357ab5a1ebcdd84808c7dfef72506a6c3e8038833ebd47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","failed_at":"e2e","signature_id":"3cd718d00c36786db9c07977d8d9bab6da7e3aa088465ca79459dcdaa748a25b","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xa64ef1027e0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-h54k9n8pjkjm\" to finish creating: GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\",\n \"name\": \"1e0c8c9f-614e-479d-8628-81206456e882\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T17:54:15.481914664Z\",\n \"endTime\": \"2026-10-23T18:23:16.346423782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n correlation ID: 11ba9467-ece2-482a-942e-01e77f78ac77\n client request ID: d489ddc3-32d0-4662-9c28-60d44153b44d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xa64ef1027e0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-h54k9n8pjkjm\" to finish creating: GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\",\n \"name\": \"1e0c8c9f-614e-479d-8628-81206456e882\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T17:54:15.481914664Z\",\n \"endTime\": \"2026-10-23T18:23:16.346423782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n correlation ID: 11ba9467-ece2-482a-942e-01e77f78ac77\n client request ID: d489ddc3-32d0-4662-9c28-60d44153b44d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8587d6ef8cd7dac5fe4005b92f7045c941c29895bdc330fc36d748cf48c16a17","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e2172c4bb32284869f1de057047fa8ea9d3572329fd8dfe120c64117605db5d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","failed_at":"alert","signature_id":"56a88ed15c81d66a2d6b20e1497719151538162e99edd29f52fd5232a43bb042","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T18:30:05Z\n Ended: 2026-10-23T18:41:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9103104-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.24:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-t567t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T18:30:05Z\n Ended: 2026-10-23T18:42:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9103104-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.24:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-t567t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-23T18:31:05Z\n Ended: 2026-10-23T18:40:13Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9103104-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.24:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-t567t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-23T18:31:05Z\n Ended: 2026-10-23T18:40:13Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9103104-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.24:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-t567t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"fbba215f7deb6756269fedf74811651fd4f058c6537e50fc7f98e522d43abf51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","failed_at":"e2e","signature_id":"dfebbf0731f81307daf766e898cda81fae9d9e47bbb68807b280ca90f03dd147","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x794496e4000\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x79449964570\u003e{\n s: \"dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x794496e4000\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x79449964570\u003e{\n s: \"dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"197f9640ca1cff10bbe506b04538db682332c0dd1edad7858df9b62446a0bf4e","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","197f9640ca1cff10bbe506b04538db682332c0dd1edad7858df9b62446a0bf4e","8587d6ef8cd7dac5fe4005b92f7045c941c29895bdc330fc36d748cf48c16a17","b0003cc8ca3a9f7347553bcc0638ed02c32204aa3c9fd097c1f10d2bf5392fbd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113671635931238400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"closed","pr_sha":"e159b4bb81965488c7898a51e008061eee591672","final_merged_sha":"afd3417980c4b5052b54fdf377e648bf6df383fc","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T16:39:22Z","started_at":"2026-10-23T16:39:22Z","completed_at":"2026-10-23T19:03:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T19:13:32.499159117Z","timing_metadata_checked_at":"2026-10-23T19:13:32.499159117Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T19:13:32.502996905Z","region_metadata_checked_at":"2026-10-23T19:13:32.502996905Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113660833035718656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7073,"pr_state":"closed","pr_sha":"9c4815df62c125480cb9a6a711a2a7ca92f94b47","final_merged_sha":"9c4815df62c125480cb9a6a711a2a7ca92f94b47","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T15:56:27Z","started_at":"2026-10-23T15:56:27Z","completed_at":"2026-10-23T18:08:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T18:09:09.445518693Z","timing_metadata_checked_at":"2026-10-23T18:09:09.445518693Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T18:09:09.449240483Z","region_metadata_checked_at":"2026-10-23T18:09:09.449240483Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"bb725c5e57612c011c4fb7966cf1b4c067c0fe9dd834cfe54f1da4be4be259cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113660833035718656","occurred_at":"2026-10-23T15:56:27Z","failed_at":"e2e","signature_id":"d9f385324d27a77fd6938089006a621965c3093828acb8f05519d964d111d9ed","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x319d4a6beb60\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f48bqq4pfgdf\" to finish creating: GET https://rp.j5718656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468\",\n \"name\": \"96166412-c6d9-4ece-a45c-cf56e7487468\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T16:53:56.577784124Z\",\n \"endTime\": \"2026-10-23T17:13:00.532320401Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-t2t6b-kptmf: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-f48bqq4pfgdf--managed/ingress-cert-le-np-1-t2t6b-kptmf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-f48bqq4pfgdf--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-t2t6b-kptmf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '3' seconds after additional tokens are available. Refer t... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5718656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468\n correlation ID: 1798d4dc-6289-4a4b-aaed-4eef33b20f02\n client request ID: 11e72bdf-c049-47a8-91d7-be1bc1c7965f\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x319d4a6beb60\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f48bqq4pfgdf\" to finish creating: GET https://rp.j5718656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468\",\n \"name\": \"96166412-c6d9-4ece-a45c-cf56e7487468\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T16:53:56.577784124Z\",\n \"endTime\": \"2026-10-23T17:13:00.532320401Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-t2t6b-kptmf: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-f48bqq4pfgdf--managed/ingress-cert-le-np-1-t2t6b-kptmf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-f48bqq4pfgdf--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-t2t6b-kptmf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '3' seconds after additional tokens are available. Refer t... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5718656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468\n correlation ID: 1798d4dc-6289-4a4b-aaed-4eef33b20f02\n client request ID: 11e72bdf-c049-47a8-91d7-be1bc1c7965f","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f48bqq4pfgdf\" to finish creating: GET https://rp"}},{"row_id":"f0453f76922fd91e30566295217f002af85f2a0615d98578c66fc775810e72a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113660833035718656","occurred_at":"2026-10-23T15:56:27Z","failed_at":"e2e","signature_id":"4b06ceefd21c3c876e41349b56654277c2324b6708e3809cfe51f45fabf75683","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:182]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0x1025b98de6a8\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0x1025b95ed360\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0x1025b9b81ec0\u003e{\n errs: [\n \u003c*errors.errorString | 0x1025b92c2ec0\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:182]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0x1025b98de6a8\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0x1025b95ed360\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0x1025b9b81ec0\u003e{\n errs: [\n \u003c*errors.errorString | 0x1025b92c2ec0\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7656/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113649065022459904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7656,"pr_state":"open","pr_sha":"9e1e4d11640c388e142df4ccb0c67b1fd9fa1864","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T15:09:40Z","started_at":"2026-10-23T15:09:40Z","completed_at":"2026-10-23T17:08:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T17:26:26.12609843Z","timing_metadata_checked_at":"2026-10-23T17:26:26.12609843Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T17:26:26.130310813Z","region_metadata_checked_at":"2026-10-23T17:26:26.130310813Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bd8eed2ab5c8f17c9aa2ed897c08bd2589954f0b79de2130e8bc89c4a1ce456f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7656/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113649065022459904","occurred_at":"2026-10-23T15:09:40Z","failed_at":"e2e","signature_id":"1ddc8754dde4d08e908579fef94b5559366fe2b19d0cf98ab8ba070611484821","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x19a5a6aee900\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-662w726xqd2k\" to finish creating: GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\",\n \"name\": \"91414204-0657-405d-a101-7608067a7216\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T15:54:03.184233444Z\",\n \"endTime\": \"2026-10-23T16:13:03.632601143Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Compute/virtualMachines/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-662w726xqd2k--managed/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Network/networkInterfaces/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '1' seconds after additional tokens are available. Refer t... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n correlation ID: 47f6ba53-01c6-4790-943f-535cc7879659\n client request ID: 259bc435-7233-4d58-9c0f-3d51379cd131\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x19a5a6aee900\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-662w726xqd2k\" to finish creating: GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\",\n \"name\": \"91414204-0657-405d-a101-7608067a7216\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T15:54:03.184233444Z\",\n \"endTime\": \"2026-10-23T16:13:03.632601143Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Compute/virtualMachines/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-662w726xqd2k--managed/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Network/networkInterfaces/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '1' seconds after additional tokens are available. Refer t... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n correlation ID: 47f6ba53-01c6-4790-943f-535cc7879659\n client request ID: 259bc435-7233-4d58-9c0f-3d51379cd131\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a41a55832324284d10413d9a99da8343f3b8c1a9dc53d5ecf869bc08ac34982","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer t... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6a41a55832324284d10413d9a99da8343f3b8c1a9dc53d5ecf869bc08ac34982"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113646623203528704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"d4cf70253dad68d7965425675df966a0248a19da","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T14:59:59Z","started_at":"2026-10-23T14:59:59Z","completed_at":"2026-10-23T15:44:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T16:02:08.637430426Z","timing_metadata_checked_at":"2026-10-23T16:02:08.637430426Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T16:02:08.643375506Z","region_metadata_checked_at":"2026-10-23T16:02:08.643375506Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0e7953380fb5b3da4f5ad17a3eba7e96934a993a8f2e2eaadfa67f12913bb2e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113646623203528704","occurred_at":"2026-10-23T14:59:59Z","failed_at":"provision","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-23T15:33:31.685Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-23T15:33:32.582Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-23T15:33:32.582Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T15:33:33.039Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 namespace=secret-sync-controller\ntime=2026-10-23T15:33:33.396Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 dryRun=false\ntime=2026-10-23T15:33:33.396Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T15:33:33.530Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T15:38:39.422Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-23T15:38:39.423Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7689/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113645366145454080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7689,"pr_state":"open","pr_sha":"c4af17f6afb5ffd805ec4e7ca4fe9f04d35bc830","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T14:54:59Z","started_at":"2026-10-23T14:54:59Z","completed_at":"2026-10-23T17:06:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T17:26:26.259163494Z","timing_metadata_checked_at":"2026-10-23T17:26:26.259163494Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T17:26:26.262287082Z","region_metadata_checked_at":"2026-10-23T17:26:26.262287082Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113638233085579264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7073,"pr_state":"closed","pr_sha":"9c4815df62c125480cb9a6a711a2a7ca92f94b47","final_merged_sha":"9c4815df62c125480cb9a6a711a2a7ca92f94b47","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T14:26:38Z","started_at":"2026-10-23T14:26:38Z","completed_at":"2026-10-23T15:17:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T15:29:51.341568019Z","timing_metadata_checked_at":"2026-10-23T15:29:51.341568019Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T15:29:51.344969308Z","region_metadata_checked_at":"2026-10-23T15:29:51.344969308Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d3605408e6b20da4db5abe542a3adea105da6b7f6f52093aa83735571bf96ed3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113638233085579264","occurred_at":"2026-10-23T14:26:38Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113633427663097856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7718,"pr_state":"closed","pr_sha":"674af40e29af62bde83301a2b777a34cd4bc2e25","final_merged_sha":"674af40e29af62bde83301a2b777a34cd4bc2e25","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T14:07:33Z","started_at":"2026-10-23T14:07:33Z","completed_at":"2026-10-23T16:10:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T16:23:23.365331257Z","timing_metadata_checked_at":"2026-10-23T16:23:23.365331257Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T16:23:23.368608747Z","region_metadata_checked_at":"2026-10-23T16:23:23.368608747Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7717/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113628897605062656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7717,"pr_state":"open","pr_sha":"9b0accb13bb234cb67bd068b4bb6026742de1f24","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T13:49:33Z","started_at":"2026-10-23T13:49:33Z","completed_at":"2026-10-23T16:09:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T16:23:22.941881711Z","timing_metadata_checked_at":"2026-10-23T16:23:22.941881711Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T16:23:22.947575794Z","region_metadata_checked_at":"2026-10-23T16:23:22.947575794Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"80e2117167845d59df016a91077a81a2186589c291e3c5d76b72c41b954ad94c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7717/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113628897605062656","occurred_at":"2026-10-23T13:49:33Z","failed_at":"alert","signature_id":"762381bb4f426bcce14d2dcedf2af2dc5db6afa5d64699c2dabcc9907a483ca0","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T15:27:17Z\n Ended: 2026-10-23T15:33:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5062656-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tkibdc5gs5nul5ksrupnsegrco0ubpn-w6p5u2c0q5q4w4r\", pod=\"ignition-server-proxy-bf7d94666-n9xlf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tkibdc5gs5nul5ksrupnsegrco0ubpn-w6p5u2c0q5q4w4r/ignition-server-proxy-bf7d94666-n9xlf has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113627585765183488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7547,"pr_state":"closed","pr_sha":"28c20c5eff68a4950667d98fdfc5163f1798bdff","final_merged_sha":"cd3c21e53a196ba0ad2c38b6f73f33b744c9e3fa","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T13:44:20Z","started_at":"2026-10-23T13:44:20Z","completed_at":"2026-10-23T14:51:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T15:08:51.801086415Z","timing_metadata_checked_at":"2026-10-23T15:08:51.801086415Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T15:08:51.804291405Z","region_metadata_checked_at":"2026-10-23T15:08:51.804291405Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"5a40ef5dbcf7bbd41896414488146247909f2866263295a708c4bcd31e28668f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113627585765183488","occurred_at":"2026-10-23T13:44:20Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-23T14:27:54.359Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-23T14:27:55.884Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-23T14:27:55.885Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-23T14:27:56.758Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-23T14:27:56.759Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T14:27:56.759Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j5183488-svc\ntime=2026-10-23T14:30:02.581Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-23T14:30:02.581Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=2m6s\ntime=2026-10-23T14:30:02.581Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T14:30:03.058Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-23T14:30:03.058Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T14:30:03.059Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T14:30:03.941Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-23T14:30:03.941Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T14:30:03.972Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-23T14:30:04.189Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T14:30:04.255Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T14:30:04.808Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T14:30:04.808Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T14:30:04.808Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T14:30:04.808Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T14:30:04.808Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j5183488-svc\ntime=2026-10-23T14:45:07.671Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230","failure_pattern":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded"}},{"row_id":"f7101adc0e1e4701aae6b0ba1e10fd2c837dd8145987e22f936342b56c97d819","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113627585765183488","occurred_at":"2026-10-23T13:44:20Z","failed_at":"alert","signature_id":"28d90db273f3dc28418a9638298084d32843c4ff21548aa85701ba2afb5f93dc","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 62 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-q7xh2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-q7xh2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-ndlbl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-ndlbl has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-7xjfm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-7xjfm has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-5dvf5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-5dvf5 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-mb68l\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-mb68l has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-ssk4m\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-ssk4m has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-9ljfg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-9ljfg has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-hlrms\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-hlrms has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-7zxmv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-7zxmv has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-995kh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-995kh has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-jj844\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-jj844 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-hk2pc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-hk2pc has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-v22fj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-v22fj has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-x55nd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-x55nd has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-dbxr7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-dbxr7 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-gmslj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-gmslj has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-tfmkl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-tfmkl has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-wcfrc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-wcfrc has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-vz2cv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-vz2cv has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-9mcmc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-9mcmc has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-d6nhd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-d6nhd has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-9txxf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-9txxf has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-6b8wc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-6b8wc has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-nrgxf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-nrgxf has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-sgtgx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-sgtgx has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-rqj2v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-rqj2v has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-8vhqg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-8vhqg has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-rsscm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-rsscm has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-t4fln\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-t4fln has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-tjxjb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-tjxjb has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-m5d5c\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-m5d5c has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-bhfw8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-bhfw8 has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-8kc57\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-8kc57 has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-nmlxd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-nmlxd has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-7nz9z\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-7nz9z has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-rr5tb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-rr5tb has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-zgzmg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-zgzmg has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-v2xpg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-v2xpg has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-v96bl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-v96bl has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-j9jl5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-j9jl5 has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-b87xs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-b87xs has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-pphfm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-pphfm has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-hgkjq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-hgkjq has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-xd72x\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-xd72x has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-8jcjf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-8jcjf has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-rvdns\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-rvdns has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-x4kzj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-x4kzj has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-b8qd2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-b8qd2 has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-qt827\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-qt827 has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-q8mvg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-q8mvg has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-kfjrt\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-kfjrt has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-hwkjz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-hwkjz has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-9td26\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-9td26 has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-cr96z\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-cr96z has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-9dqr2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-9dqr2 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-d56c6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-d56c6 has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-s89qq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-s89qq has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-k2xxx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-k2xxx has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-jb7dj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-jb7dj has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-l6zq2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-l6zq2 has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-hzddl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-hzddl has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-k2wfc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-k2wfc has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113627582963388416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7548,"pr_state":"open","pr_sha":"47f81f80646559a4433eb3e0cdad367eaebf7c0e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T13:44:19Z","started_at":"2026-10-23T13:44:19Z","completed_at":"2026-10-23T16:01:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T16:12:24.103916358Z","timing_metadata_checked_at":"2026-10-23T16:12:24.103916358Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T16:12:24.108144346Z","region_metadata_checked_at":"2026-10-23T16:12:24.108144346Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113621295194181632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7679,"pr_state":"closed","pr_sha":"aef55fb48491e559a7a2689dd16c32b57b3baced","final_merged_sha":"aef55fb48491e559a7a2689dd16c32b57b3baced","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T13:19:20Z","started_at":"2026-10-23T13:19:20Z","completed_at":"2026-10-23T15:24:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T15:40:29.796145823Z","timing_metadata_checked_at":"2026-10-23T15:40:29.796145823Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T15:40:29.799498515Z","region_metadata_checked_at":"2026-10-23T15:40:29.799498515Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7711/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113617584753479680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7711,"pr_state":"closed","pr_sha":"594817273d6d32829fbd4bd4d593d044d4d06f63","final_merged_sha":"594817273d6d32829fbd4bd4d593d044d4d06f63","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T13:04:35Z","started_at":"2026-10-23T13:04:35Z","completed_at":"2026-10-23T15:14:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T15:29:51.733692719Z","timing_metadata_checked_at":"2026-10-23T15:29:51.733692719Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T15:29:51.73636031Z","region_metadata_checked_at":"2026-10-23T15:29:51.73636031Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113612033818628096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7544,"pr_state":"open","pr_sha":"ed7741c7e007b22fcf2a10919d0cf5a727c16488","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T12:42:32Z","started_at":"2026-10-23T12:42:32Z","completed_at":"2026-10-23T15:14:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T15:29:51.607870136Z","timing_metadata_checked_at":"2026-10-23T15:29:51.607870136Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T15:29:51.611526024Z","region_metadata_checked_at":"2026-10-23T15:29:51.611526024Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"f5b3faebf525c5bdc1bd5ed455a2d8219b9c553eb26482955c54ab80ee636d72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113612033818628096","occurred_at":"2026-10-23T12:42:32Z","failed_at":"e2e","signature_id":"2105a054cb380d7c595411f3110d328751601f1b100d33affce72e45f10a0150","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3ce657fb6660\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-f9dtl496nd4k\" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\",\n \"name\": \"5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:52:15.708697202Z\",\n \"endTime\": \"2026-10-23T14:11:23.696759282Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-f9dtl496nd4k--managed/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-f9dtl496nd4k--managed/providers/Microsoft.Network/networkInterfaces/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to http... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\n correlation ID: 2e16a443-f7c3-4087-8290-e874ac42ea22\n client request ID: d8c04265-7efa-4df2-aadc-80112544e6fc\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3ce657fb6660\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-f9dtl496nd4k\" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\",\n \"name\": \"5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:52:15.708697202Z\",\n \"endTime\": \"2026-10-23T14:11:23.696759282Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-f9dtl496nd4k--managed/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-f9dtl496nd4k--managed/providers/Microsoft.Network/networkInterfaces/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to http... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\n correlation ID: 2e16a443-f7c3-4087-8290-e874ac42ea22\n client request ID: d8c04265-7efa-4df2-aadc-80112544e6fc\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f9e3ef3ac9130312b92e134e4395de00ce9e3a9ec324a6919141f76a3e9fa9ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113612033818628096","occurred_at":"2026-10-23T12:42:32Z","failed_at":"e2e","signature_id":"d9cde6d3be6b80b67637a1674fb67e65e7c5c386976a816549974fb5fd2ebc69","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x159fda57b1e0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-972dcv\" in resourcegroup=\"rg-np-version-upgrade-972dcv-p4hg8tz7gs76\" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\",\n \"name\": \"d066a71c-277e-4671-a4dd-16329cfd2682\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T14:16:35.253267721Z\",\n \"endTime\": \"2026-10-23T14:35:40.055586863Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n correlation ID: df6d449d-b4ac-498f-8d50-5d981e7f434b\n client request ID: 09867594-8f78-4da8-bf75-c401ddee2f7d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x159fda57b1e0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-972dcv\" in resourcegroup=\"rg-np-version-upgrade-972dcv-p4hg8tz7gs76\" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\",\n \"name\": \"d066a71c-277e-4671-a4dd-16329cfd2682\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T14:16:35.253267721Z\",\n \"endTime\": \"2026-10-23T14:35:40.055586863Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n correlation ID: df6d449d-b4ac-498f-8d50-5d981e7f434b\n client request ID: 09867594-8f78-4da8-bf75-c401ddee2f7d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113603186051780608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7688,"pr_state":"open","pr_sha":"42fa8ee11e9c7733166eb3dee4c7c97e972e2a16","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T12:07:11Z","started_at":"2026-10-23T12:07:11Z","completed_at":"2026-10-23T14:11:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T14:26:30.155429374Z","timing_metadata_checked_at":"2026-10-23T14:26:30.155429374Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T14:26:30.158815066Z","region_metadata_checked_at":"2026-10-23T14:26:30.158815066Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"108c13a432fa7e78ab5d3b85caacf9edd9688968d46262ffb877a5c4e03d4b85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113603186051780608","occurred_at":"2026-10-23T12:07:11Z","failed_at":"e2e","signature_id":"4cf06d6c9bb7b85aaa08e454a9e64270b9208afcbca137d23650a0ab18180e76","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-l9g7mz\"\nUnexpected error:\n \u003c*errors.joinError | 0x2d180ff06060\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\": nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2d180f0a8800\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2d180f0a87a0\u003e{\n msg: \"failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2d180f0a8780\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x2d180ff5f680\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-l9g7mz\"\nUnexpected error:\n \u003c*errors.joinError | 0x2d180ff06060\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\": nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2d180f0a8800\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2d180f0a87a0\u003e{\n msg: \"failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2d180f0a8780\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x2d180ff5f680\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113601669802496000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7611,"pr_state":"open","pr_sha":"7849571c59127d0f7b0e8c2238d533f1bce89dd8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T12:01:21Z","started_at":"2026-10-23T12:01:21Z","completed_at":"2026-10-23T14:46:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T14:58:12.088703374Z","timing_metadata_checked_at":"2026-10-23T14:58:12.088703374Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T14:58:12.091725865Z","region_metadata_checked_at":"2026-10-23T14:58:12.091725865Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"95830e10f1cbb78af49f216374c3e0578842a84c3ce2dd5f08332bc11ca3dbdb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113601669802496000","occurred_at":"2026-10-23T12:01:21Z","failed_at":"e2e","signature_id":"f653863f476c685d13e3a7415f0b0b11a8654a0f134f21f96f43a33cb5375be8","test_name":"Image Registry Policy should deny pods with images from disallowed registries","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:113]: Denial should come from the image registry policy, got: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nExpected\n \u003cstring\u003e: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nto contain substring\n \u003cstring\u003e: not from an allowed registry\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:113]: Denial should come from the image registry policy, got: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nExpected\n \u003cstring\u003e: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nto contain substring\n \u003cstring\u003e: not from an allowed registry","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5e13e8ebe7676d236f83bd844b8c7a828a419da28309fcb2c22c74d629e1c635","failure_pattern":"Denial should come from the image registry policy, got: pod \"\u003cpod\u003e\" is forbidden: error looking up service account \u003cnamespace\u003e/default: serviceaccount \"default\" not found","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:113]: Denial should come from the image registry policy, got: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["5e13e8ebe7676d236f83bd844b8c7a828a419da28309fcb2c22c74d629e1c635"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113599326222553088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7665,"pr_state":"closed","pr_sha":"225d2042326f69bb80ffd6c1717691fa155634a4","final_merged_sha":"b40bc5cbb80a171bde8ca5e40c43b439603f5493","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T11:52:02Z","started_at":"2026-10-23T11:52:02Z","completed_at":"2026-10-23T14:03:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T14:16:02.06654715Z","timing_metadata_checked_at":"2026-10-23T14:16:02.06654715Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T14:16:02.070572038Z","region_metadata_checked_at":"2026-10-23T14:16:02.070572038Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"846db787994a7fb8692e8f383f26e454e06855969c866100230110ed8595b07c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113599326222553088","occurred_at":"2026-10-23T11:52:02Z","failed_at":"e2e","signature_id":"b2e5ac734561cd50d6d2bb64f76bb0716f87849cddae08ac51680aa3c5151ea8","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x99f918edcc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vknb2mpvr5xz\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\",\n \"name\": \"76509bdd-86ae-4734-aa51-d853651ea0fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:59:15.455605547Z\",\n \"endTime\": \"2026-10-23T13:18:15.598008978Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-hc6wj-5qqzr: Failed: availabilitysets failed to create or update. err: failed to get existing resource v20260901preview-vknb2mpvr5xz--managed/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-vknb2mpvr5xz--managed/providers/Microsoft.Compute/availabilitySets/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after addi... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n correlation ID: 82980038-7d5f-4ddb-bc41-a8d67ca04b71\n client request ID: 6668afc8-6cc9-4e23-9e4f-4e194ede10a5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x99f918edcc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vknb2mpvr5xz\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\",\n \"name\": \"76509bdd-86ae-4734-aa51-d853651ea0fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:59:15.455605547Z\",\n \"endTime\": \"2026-10-23T13:18:15.598008978Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-hc6wj-5qqzr: Failed: availabilitysets failed to create or update. err: failed to get existing resource v20260901preview-vknb2mpvr5xz--managed/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-vknb2mpvr5xz--managed/providers/Microsoft.Compute/availabilitySets/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after addi... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n correlation ID: 82980038-7d5f-4ddb-bc41-a8d67ca04b71\n client request ID: 6668afc8-6cc9-4e23-9e4f-4e194ede10a5\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"802121af546da5e12fc3cac43502a22a348ae1f3e72897092323375f66a04dbc","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after addi... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f3272ac2047854c4bb6dfad0710270e49254abb7c680e24b82e676547e859f60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113599326222553088","occurred_at":"2026-10-23T11:52:02Z","failed_at":"e2e","signature_id":"4b66edeccfb70c1f5cfd0f1d6ba0073696d1cd6c7003d445193ae337daf93d50","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cda231b9b40\u003e: \n failed to create NodePool np-d4pgmj: failed waiting for nodepool=\"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\" in resourcegroup=\"rg-candidate-4-22-jrp5tdgcl9kv\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n \"name\": \"f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:58:53.825112936Z\",\n \"endTime\": \"2026-10-23T13:17:55.550011725Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7q7k8t3p4d7j9m-np-d4pgmj-4fc64-lwc4j: Failed: availabilitysets failed to create or update. err: failed to get existing resource rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/providers/Microsoft.Compute/availabilitySets/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-b... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n correlation ID: bb68fb60-f553-4611-a472-1dc9f8329c89\n client request ID: 638d8201-2360-4c79-a5aa-4979c9f07d33\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cda231b9b40\u003e: \n failed to create NodePool np-d4pgmj: failed waiting for nodepool=\"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\" in resourcegroup=\"rg-candidate-4-22-jrp5tdgcl9kv\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n \"name\": \"f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:58:53.825112936Z\",\n \"endTime\": \"2026-10-23T13:17:55.550011725Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7q7k8t3p4d7j9m-np-d4pgmj-4fc64-lwc4j: Failed: availabilitysets failed to create or update. err: failed to get existing resource rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/providers/Microsoft.Compute/availabilitySets/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-b... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n correlation ID: bb68fb60-f553-4611-a472-1dc9f8329c89\n client request ID: 638d8201-2360-4c79-a5aa-4979c9f07d33\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"11f1e4870cb02ff7036064941db2a3263ef1df1d64949078bceae777a2c26ab6","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-b... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["11f1e4870cb02ff7036064941db2a3263ef1df1d64949078bceae777a2c26ab6","802121af546da5e12fc3cac43502a22a348ae1f3e72897092323375f66a04dbc"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113597591521333248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"0d965839a628d29699c887cc0aaeee8761a5136f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T11:45:03Z","started_at":"2026-10-23T11:45:03Z","completed_at":"2026-10-23T12:45:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T12:52:43.319432949Z","timing_metadata_checked_at":"2026-10-23T12:52:43.319432949Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T12:52:43.322357328Z","region_metadata_checked_at":"2026-10-23T12:52:43.322357328Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3df5e7d43b55cadb37fc39d02d9d59c6feed457c2a0682b2671defb6c6d6ff0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113597591521333248","occurred_at":"2026-10-23T11:45:03Z","failed_at":"provision","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-23T12:34:43.631Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-23T12:34:44.030Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-23T12:34:44.030Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-23T12:34:44.168Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-23T12:34:44.296Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-23T12:34:44.296Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-23T12:34:44.405Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-23T12:39:47.576Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-23T12:39:47.576Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113594282169339904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6718,"pr_state":"closed","pr_sha":"37e25e27f18c451621015c1d957095596de22c13","final_merged_sha":"de22b69e0ee31f17c2b9e52b1813cdd58df0d48f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T11:32:00Z","started_at":"2026-10-23T11:32:00Z","completed_at":"2026-10-23T14:42:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T14:58:11.929248138Z","timing_metadata_checked_at":"2026-10-23T14:58:11.929248138Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T14:58:11.932501629Z","region_metadata_checked_at":"2026-10-23T14:58:11.932501629Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6d23ed7386314a696e6966ea40e4d1370a4cc28bbefd088d3e42f5b052b9aca9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113594282169339904","occurred_at":"2026-10-23T11:32:00Z","failed_at":"e2e","signature_id":"c11a2050e197d4a44a7c09a12f94ca83ff56f060fe76b2742e4b18d0997beadb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1909134f0c20\u003e: \n failed to create NodePool np-qptx5c: failed waiting for nodepool=\"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\" in resourcegroup=\"rg-candidate-5-1-6mzh9r6brjfl\" to finish creating: GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n \"name\": \"1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:34:04.539419115Z\",\n \"endTime\": \"2026-10-23T13:53:12.193747284Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Compute/virtualMachines/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Network/networkInterfaces/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' se... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n correlation ID: 65b99ffd-0c17-417b-bd92-7368727e5721\n client request ID: 5a9e3ea8-7ac7-474e-85b1-9320f2c4be29\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1909134f0c20\u003e: \n failed to create NodePool np-qptx5c: failed waiting for nodepool=\"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\" in resourcegroup=\"rg-candidate-5-1-6mzh9r6brjfl\" to finish creating: GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n \"name\": \"1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:34:04.539419115Z\",\n \"endTime\": \"2026-10-23T13:53:12.193747284Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Compute/virtualMachines/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Network/networkInterfaces/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' se... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n correlation ID: 65b99ffd-0c17-417b-bd92-7368727e5721\n client request ID: 5a9e3ea8-7ac7-474e-85b1-9320f2c4be29\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d25cf9b217ca7a1560a0ac8f986a95cd4726bd1277cdf1dc11174901dd19f0f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '10' se... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7d25cf9b217ca7a1560a0ac8f986a95cd4726bd1277cdf1dc11174901dd19f0f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113593113002905600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7709,"pr_state":"closed","pr_sha":"6d5d6acdf16d24f21224286cd936ea39a6baa4c5","final_merged_sha":"6d5d6acdf16d24f21224286cd936ea39a6baa4c5","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T11:27:12Z","started_at":"2026-10-23T11:27:12Z","completed_at":"2026-10-23T13:49:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T14:05:34.367142534Z","timing_metadata_checked_at":"2026-10-23T14:05:34.367142534Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T14:05:34.371771318Z","region_metadata_checked_at":"2026-10-23T14:05:34.371771318Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T11:26:04Z","started_at":"2026-10-23T11:26:04Z","completed_at":"2026-10-23T13:50:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T14:05:34.507948949Z","timing_metadata_checked_at":"2026-10-23T14:05:34.507948949Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T14:05:34.511089538Z","region_metadata_checked_at":"2026-10-23T14:05:34.511089538Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"535650aa6350e3e71f15d0cdd9c7d2318d2bd52099935763007fe522861d769d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","failed_at":"e2e","signature_id":"7bb7aa38d25f69b831b53681016019396c025b2d18da7932ffb9e298e6579b79","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1df6789587c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*verifiers.pollObservationError | 0x1df6789587a0\u003e{\n observation: \u003c*errors.errorString | 0x1df677e0e5e0\u003e{\n s: \"dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n interruption: \u003c*fmt.wrapError | 0x1df678958780\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1df6776d4500\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1df678958760\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1df679343380\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1df6789587c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*verifiers.pollObservationError | 0x1df6789587a0\u003e{\n observation: \u003c*errors.errorString | 0x1df677e0e5e0\u003e{\n s: \"dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n interruption: \u003c*fmt.wrapError | 0x1df678958780\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1df6776d4500\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1df678958760\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1df679343380\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"50aafdd0364e960cb215062171251b48e16daefd02feb64915b33b81ee7aa50a","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred"}},{"row_id":"5f3f7ca09dbacfe70d0f66dedf5945dc16a57e44c31ea096e12a13e100d082e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","failed_at":"alert","signature_id":"fd738f0b860459440540292342939e3ce110921d918faf32d78e2b8f0cbc73ec","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T12:35:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-558mc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a45ad8a2-0299-48d6-b5cc-feba766709c3\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-558mc (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T12:38:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0830592-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.214:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-9dtps\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a3569b9f-561d-4f48-9ca7-6bcdc31d1728\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-9dtps (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-23T12:38:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0830592-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.214:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-9dtps\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a3569b9f-561d-4f48-9ca7-6bcdc31d1728\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-9dtps (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-23T12:40:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-558mc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a45ad8a2-0299-48d6-b5cc-feba766709c3\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-558mc (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"a1093e1cdcbaa320746a5d80413431e6248d97d7cc9e33cea5e5d9f176d08174","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","failed_at":"alert","signature_id":"7467c8b82e1de4741264fd7a83a7e3d54e11cebdb1e11c074699979a6d8ec2ff","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T12:56:11Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T12:56:11Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"b0b3ff979c0ffa690a65193a5684b0114559266d0f9f4a6e814f88313f3cc100","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","failed_at":"alert","signature_id":"91e41adf8a4cbf20601080ac6e87fc658cc44668486f116ba1401163a81cca7b","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T12:46:11Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T12:46:11Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","50aafdd0364e960cb215062171251b48e16daefd02feb64915b33b81ee7aa50a","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7635,"pr_state":"closed","pr_sha":"6845cd8bcb534cf4c453c27e6340e2159aa649c7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T11:17:38Z","started_at":"2026-10-23T11:17:38Z","completed_at":"2026-10-23T13:42:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T13:55:04.787124021Z","timing_metadata_checked_at":"2026-10-23T13:55:04.787124021Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T13:55:04.790406712Z","region_metadata_checked_at":"2026-10-23T13:55:04.790406712Z"},"failed_at":["alert","e2e"],"failed_test_count":19,"occurrences":[{"row_id":"25cffe24767929d9ee8ce9a025b003c7735ee56b05c1bee236e85764888ade95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"712adee26418d9b97d2614e8f26ca63599b26688a67ad28c5cce4a2daa4c4e84","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-23-lrb762\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2b9ddee00df0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-23-lrb762\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2b9ddee00df0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"318d0792b95dfd2b4f2e2b02e4714ee97afea4b2ef325bd8c6c78b1fdac2ed6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"6a69757f3cbee730cf9d60087714cca23dc702792cdaa339a7014d9b67fd1143","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-20-qkcvbq\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x49ccf31c560\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-20-qkcvbq\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x49ccf31c560\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"3ac55bb0b00ea8abb99435fe51bb6799979d28dd105549716d07127d6b2432d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"f75c1d43c7a4c5c09304ce65eeb484f3c9012f635c820f63f75d2054e572fc61","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-5-1-4pwngq\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x293284ca6570\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-5-1-4pwngq\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x293284ca6570\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"4640ded552e9ae71e0ef71b4273e807820bc9641271797014c78f23f7b220e0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"8d01f3d5bb1dffe043107fd710c91d72482c52b3696c1ab17cb3c16a039f0369","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"7f52 bbf59cb1 ...nR.......R...\n00007ffd4f7fae70: 00007f52 6e7fc640 00007ffd 4f7fb140 @..nR...@..O....\n00007ffd4f7fae80: 00007ffd 4f7fafae 00007ffd 4f7fafaf ...O.......O....\n00007ffd4f7fae90: 00007f52 6e7fc640 00007f52 bbed47f5 @..nR....G..R...\n00007ffd4f7faea0: 00007f52 bc045c80 00000000 00000120 .\\..R... .......\n00007ffd4f7faeb0: 00000000 003d0f00 b406abf0 07e46d00 ..=......m......\n00007ffd4f7faec0: 00007f52 bbe48740 00000000 00000006 @...R...........\n00007ffd4f7faed0: 00000000 1a0a9f00 00007ffd 4f7fb200 ...........O....\n00007ffd4f7faee0: 00001a1d 095421e0 00007f52 bbe89646 .!T.....F...R...\n00007ffd4f7faef0: 00007f52 bc046e90 00007f52 bbe737f3 .n..R....7..R...\n00007ffd4f7faf00: 00000000 00000020 b406abf0 07e46d00 ........m......\n00007ffd4f7faf10: 00007f52 6dffc000 00007f52 bbf48ac4 ...mR.......R...\n00007ffd4f7faf20: 00000000 00000000 00000000 0000000d ................\n\ngoroutine 1 gp=0x1a1d095421e0 m=0 mp=0x17528da0 [running, locked to thread]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x1a1d096486a8 sp=0x1a1d09648698 pc=0x56668c8\nruntime.newm1(0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:2922 +0x85 fp=0x1a1d096486e8 sp=0x1a1d096486a8 pc=0x562b625\nruntime.newm(0x5635301?, 0x0, 0xd19a3c0?)\n\t/usr/lib/golang/src/runtime/proc.go:2902 +0x125 fp=0x1a1d09648718 sp=0x1a1d096486e8 pc=0x562b525\nruntime.startTemplateThread()\n\t/usr/lib/golang/src/runtime/proc.go:2947 +0x85 fp=0x1a1d09648748 sp=0x1a1d09648718 pc=0x562b725\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:245 +0x213 fp=0x1a1d096487e0 sp=0x1a1d09648748 pc=0x56269d3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096487e8 sp=0x1a1d096487e0 pc=0x5668501\n\ngoroutine 2 gp=0x1a1d09542d20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09648fa8 sp=0x1a1d09648f88 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x1a1d09648fe0 sp=0x1a1d09648fa8 pc=0x5626db3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09648fe8 sp=0x1a1d09648fe0 pc=0x5668501\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 18 gp=0x1a1d096843c0 m=nil [GC sweep wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644788 sp=0x1a1d09644768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:279 +0x94 fp=0x1a1d096447c8 sp=0x1a1d09644788 pc=0x560c694\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x1a1d096447e0 sp=0x1a1d096447c8 pc=0x55fd617\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096447e8 sp=0x1a1d096447e0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 19 gp=0x1a1d096845a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x1a1d09692000?, 0x1045f740?, 0x1?, 0x0?, 0x1a1d096845a0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644f78 sp=0x1a1d09644f58 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x17525960)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x1a1d09644fa8 sp=0x1a1d09644f78 pc=0x560a029\nruntime.bgscavenge(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:653 +0x3c fp=0x1a1d09644fc8 sp=0x1a1d09644fa8 pc=0x560a5bc\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x1a1d09644fe0 sp=0x1a1d09644fc8 pc=0x55fd5d7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09644fe8 sp=0x1a1d09644fe0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 3 gp=0x1a1d09543a40 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09649788 sp=0x1a1d09649768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x1a1d096497e0 sp=0x1a1d09649788 pc=0x5635487\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096497e8 sp=0x1a1d096497e0 pc=0x5668501\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\nrax 0x0\nrbx 0x7f52bbe48740\nrcx 0x7f52bbed694c\nrdx 0x6\nrdi 0xa70b\nrsi 0xa70b\nrbp 0xa70b\nrsp 0x7ffd4f7fae30\nr8 0x7ffd4f7faf00\nr9 0x7f52bbffc4e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7ffd4f7fb200\nr14 0x1a1d095421e0\nr15 0x10\nrip 0x7f52bbed694c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n7f52 bbf59cb1 ...nR.......R...\n00007ffd4f7fae70: 00007f52 6e7fc640 00007ffd 4f7fb140 @..nR...@..O....\n00007ffd4f7fae80: 00007ffd 4f7fafae 00007ffd 4f7fafaf ...O.......O....\n00007ffd4f7fae90: 00007f52 6e7fc640 00007f52 bbed47f5 @..nR....G..R...\n00007ffd4f7faea0: 00007f52 bc045c80 00000000 00000120 .\\..R... .......\n00007ffd4f7faeb0: 00000000 003d0f00 b406abf0 07e46d00 ..=......m......\n00007ffd4f7faec0: 00007f52 bbe48740 00000000 00000006 @...R...........\n00007ffd4f7faed0: 00000000 1a0a9f00 00007ffd 4f7fb200 ...........O....\n00007ffd4f7faee0: 00001a1d 095421e0 00007f52 bbe89646 .!T.....F...R...\n00007ffd4f7faef0: 00007f52 bc046e90 00007f52 bbe737f3 .n..R....7..R...\n00007ffd4f7faf00: 00000000 00000020 b406abf0 07e46d00 ........m......\n00007ffd4f7faf10: 00007f52 6dffc000 00007f52 bbf48ac4 ...mR.......R...\n00007ffd4f7faf20: 00000000 00000000 00000000 0000000d ................\n\ngoroutine 1 gp=0x1a1d095421e0 m=0 mp=0x17528da0 [running, locked to thread]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x1a1d096486a8 sp=0x1a1d09648698 pc=0x56668c8\nruntime.newm1(0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:2922 +0x85 fp=0x1a1d096486e8 sp=0x1a1d096486a8 pc=0x562b625\nruntime.newm(0x5635301?, 0x0, 0xd19a3c0?)\n\t/usr/lib/golang/src/runtime/proc.go:2902 +0x125 fp=0x1a1d09648718 sp=0x1a1d096486e8 pc=0x562b525\nruntime.startTemplateThread()\n\t/usr/lib/golang/src/runtime/proc.go:2947 +0x85 fp=0x1a1d09648748 sp=0x1a1d09648718 pc=0x562b725\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:245 +0x213 fp=0x1a1d096487e0 sp=0x1a1d09648748 pc=0x56269d3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096487e8 sp=0x1a1d096487e0 pc=0x5668501\n\ngoroutine 2 gp=0x1a1d09542d20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09648fa8 sp=0x1a1d09648f88 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x1a1d09648fe0 sp=0x1a1d09648fa8 pc=0x5626db3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09648fe8 sp=0x1a1d09648fe0 pc=0x5668501\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 18 gp=0x1a1d096843c0 m=nil [GC sweep wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644788 sp=0x1a1d09644768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:279 +0x94 fp=0x1a1d096447c8 sp=0x1a1d09644788 pc=0x560c694\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x1a1d096447e0 sp=0x1a1d096447c8 pc=0x55fd617\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096447e8 sp=0x1a1d096447e0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 19 gp=0x1a1d096845a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x1a1d09692000?, 0x1045f740?, 0x1?, 0x0?, 0x1a1d096845a0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644f78 sp=0x1a1d09644f58 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x17525960)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x1a1d09644fa8 sp=0x1a1d09644f78 pc=0x560a029\nruntime.bgscavenge(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:653 +0x3c fp=0x1a1d09644fc8 sp=0x1a1d09644fa8 pc=0x560a5bc\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x1a1d09644fe0 sp=0x1a1d09644fc8 pc=0x55fd5d7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09644fe8 sp=0x1a1d09644fe0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 3 gp=0x1a1d09543a40 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09649788 sp=0x1a1d09649768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x1a1d096497e0 sp=0x1a1d09649788 pc=0x5635487\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096497e8 sp=0x1a1d096497e0 pc=0x5668501\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\nrax 0x0\nrbx 0x7f52bbe48740\nrcx 0x7f52bbed694c\nrdx 0x6\nrdi 0xa70b\nrsi 0xa70b\nrbp 0xa70b\nrsp 0x7ffd4f7fae30\nr8 0x7ffd4f7faf00\nr9 0x7f52bbffc4e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7ffd4f7fb200\nr14 0x1a1d095421e0\nr15 0x10\nrip 0x7f52bbed694c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}},{"row_id":"4a1ab976087b2051580fbdceda37e0edc112272ddde7073223e37b4618daa633","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"2623387444ac60aad1c81d411318d579ab144dfc53c580add9b2105a4d1cf697","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x3a796e6c5e8\u003e: \n resource group \"v20260901preview-7822pl2jslgp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-7822pl2jslgp\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\",\n \"name\": \"aa1c3345-b774-4a31-837e-23361b1b3df1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:19:48.083110134Z\",\n \"endTime\": \"2026-10-23T12:43:54.717460671Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkfr4jhp6k8h2l5ht0m1oco8hjk189n still exists (deletion dispatched at 2026-10-23T12:19:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n correlation ID: d70e853f-388e-4661-a9cc-9552b311d878\n client request ID: 94a8bbf5-a064-4508-a581-559607551023\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x3a796e6c5e8\u003e: \n resource group \"v20260901preview-7822pl2jslgp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-7822pl2jslgp\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\",\n \"name\": \"aa1c3345-b774-4a31-837e-23361b1b3df1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:19:48.083110134Z\",\n \"endTime\": \"2026-10-23T12:43:54.717460671Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkfr4jhp6k8h2l5ht0m1oco8hjk189n still exists (deletion dispatched at 2026-10-23T12:19:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n correlation ID: d70e853f-388e-4661-a9cc-9552b311d878\n client request ID: 94a8bbf5-a064-4508-a581-559607551023\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5046700d935fc03a7505b806200b97251d76fc54516fdd722362e107c47414f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"d010fa9f43f9777168167e02e99bc932a1d4357cede95310e395869051417e5a","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x236cf2ed0920\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-46ntd4r5fjxr\" to finish creating: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\",\n \"name\": \"993068cc-46a0-4e4b-9083-bf129b4ea866\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:05:04.420745783Z\",\n \"endTime\": \"2026-10-23T12:24:12.746383509Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-cm2dh-2kqqm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-cm2dh-2kqqm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine private-ingress-np-1-cm2dh-2kqqm: Failed: availabilitysets failed to create or update. err: failed to get existing resource private-ingress-46ntd4r5fjxr--managed/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/availabilitySets/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '1' sec... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n correlation ID: d70fefde-ba5d-4298-ad39-0d96bb22c447\n client request ID: 9ac09fab-3463-40c0-9c76-35eeb3b20d7a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x236cf2ed0920\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-46ntd4r5fjxr\" to finish creating: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\",\n \"name\": \"993068cc-46a0-4e4b-9083-bf129b4ea866\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:05:04.420745783Z\",\n \"endTime\": \"2026-10-23T12:24:12.746383509Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-cm2dh-2kqqm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-cm2dh-2kqqm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine private-ingress-np-1-cm2dh-2kqqm: Failed: availabilitysets failed to create or update. err: failed to get existing resource private-ingress-46ntd4r5fjxr--managed/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/availabilitySets/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '1' sec... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n correlation ID: d70fefde-ba5d-4298-ad39-0d96bb22c447\n client request ID: 9ac09fab-3463-40c0-9c76-35eeb3b20d7a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bf2604f0a57cc83f42180ea2d978fdb012e4de2c6d11e90a5b15c16b2e73342d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '1' sec... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"626db38d418d448ace33bd35e6868f428e8e5d497db76f9fe4f34d9a43565c29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"041b58f88c07e6b6df5a78fe1b06fddf2a98976eca66b8f788c458fffb00b1ac","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"src/runtime/mgc.go:1695 +0x17 fp=0xccc45f767e0 sp=0xccc45f767c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f767e8 sp=0xccc45f767e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0xccc46384000 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638a740 sp=0xccc4638a720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638a7c8 sp=0xccc4638a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638a7e0 sp=0xccc4638a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638a7e8 sp=0xccc4638a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0xccc461ac3c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f7d740 sp=0xccc45f7d720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f7d7c8 sp=0xccc45f7d740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7d7e0 sp=0xccc45f7d7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7d7e8 sp=0xccc45f7d7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0xccc460045a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f76f40 sp=0xccc45f76f20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f76fc8 sp=0xccc45f76f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f76fe0 sp=0xccc45f76fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f76fe8 sp=0xccc45f76fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0xccc463841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638af40 sp=0xccc4638af20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638afc8 sp=0xccc4638af40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638afe0 sp=0xccc4638afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638afe8 sp=0xccc4638afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0xccc461ac5a0 m=5 mp=0xccc46000008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0xccc45f7dea0 sp=0xccc45f7de90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0xccc45f7df40 sp=0xccc45f7dea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0xccc45f7dfc8 sp=0xccc45f7df40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7dfe0 sp=0xccc45f7dfc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7dfe8 sp=0xccc45f7dfe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 36 gp=0xccc463843c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207e7eb6?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638b740 sp=0xccc4638b720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638b7c8 sp=0xccc4638b740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638b7e0 sp=0xccc4638b7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638b7e8 sp=0xccc4638b7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax 0x0\nrbx 0x7fc7134fd640\nrcx 0x7fc75a9bf94c\nrdx 0x6\nrdi 0xa70a\nrsi 0xa711\nrbp 0xa711\nrsp 0x7fc7134fc790\nr8 0x7fc7134fc860\nr9 0x7fc75aae54e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7fc7134fcb60\nr14 0xccc45e750e0\nr15 0xffffffffffffffff\nrip 0x7fc75a9bf94c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\nsrc/runtime/mgc.go:1695 +0x17 fp=0xccc45f767e0 sp=0xccc45f767c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f767e8 sp=0xccc45f767e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0xccc46384000 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638a740 sp=0xccc4638a720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638a7c8 sp=0xccc4638a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638a7e0 sp=0xccc4638a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638a7e8 sp=0xccc4638a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0xccc461ac3c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f7d740 sp=0xccc45f7d720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f7d7c8 sp=0xccc45f7d740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7d7e0 sp=0xccc45f7d7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7d7e8 sp=0xccc45f7d7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0xccc460045a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f76f40 sp=0xccc45f76f20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f76fc8 sp=0xccc45f76f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f76fe0 sp=0xccc45f76fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f76fe8 sp=0xccc45f76fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0xccc463841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638af40 sp=0xccc4638af20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638afc8 sp=0xccc4638af40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638afe0 sp=0xccc4638afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638afe8 sp=0xccc4638afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0xccc461ac5a0 m=5 mp=0xccc46000008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0xccc45f7dea0 sp=0xccc45f7de90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0xccc45f7df40 sp=0xccc45f7dea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0xccc45f7dfc8 sp=0xccc45f7df40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7dfe0 sp=0xccc45f7dfc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7dfe8 sp=0xccc45f7dfe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 36 gp=0xccc463843c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207e7eb6?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638b740 sp=0xccc4638b720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638b7c8 sp=0xccc4638b740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638b7e0 sp=0xccc4638b7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638b7e8 sp=0xccc4638b7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax 0x0\nrbx 0x7fc7134fd640\nrcx 0x7fc75a9bf94c\nrdx 0x6\nrdi 0xa70a\nrsi 0xa711\nrbp 0xa711\nrsp 0x7fc7134fc790\nr8 0x7fc7134fc860\nr9 0x7fc75aae54e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7fc7134fcb60\nr14 0xccc45e750e0\nr15 0xffffffffffffffff\nrip 0x7fc75a9bf94c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}},{"row_id":"64f61ee9d531d5f8f1f76d76654d391b769f0775295f0f3593cad62e5c46f583","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"b79149b9fc941d47e42641fbc00f500cd063ac6efa35986bbc24fa7b5d878613","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x335f1b87d830\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x335f1b87d830\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"666464137fea0f6b78005eb7849b64c818ce0a82c023aa57e1e2b0e6b4e7b932","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"df0ed8cb7549a63439671f3dc385d910a4106ef5cb0378e7a7e6d06be9fbae28","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595527e8 sp=0x1077595527e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0x1077596043c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x10775961a740 sp=0x10775961a720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x10775961a7c8 sp=0x10775961a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961a7e0 sp=0x10775961a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961a7e8 sp=0x10775961a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0x1077597883c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559740 sp=0x107759559720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077595597c8 sp=0x107759559740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077595597e0 sp=0x1077595597c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595597e8 sp=0x1077595597e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0x1077597885a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559f40 sp=0x107759559f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759559fc8 sp=0x107759559f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759559fe0 sp=0x107759559fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759559fe8 sp=0x107759559fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0x1077599841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759552f40 sp=0x107759552f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759552fc8 sp=0x107759552f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759552fe0 sp=0x107759552fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759552fe8 sp=0x107759552fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0x1077596045a0 m=4 mp=0x107759582008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x10775961aea0 sp=0x10775961ae90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x10775961af40 sp=0x10775961aea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x10775961afc8 sp=0x10775961af40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961afe0 sp=0x10775961afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961afe8 sp=0x10775961afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 10 gp=0x107759788780 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207ccb99?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759616740 sp=0x107759616720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077596167c8 sp=0x107759616740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077596167e0 sp=0x1077596167c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077596167e8 sp=0x1077596167e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax 0x0\nrbx 0x7fd81f5da640\nrcx 0x7fd866a9c94c\nrdx 0x6\nrdi 0xa70c\nrsi 0xa715\nrbp 0xa715\nrsp 0x7fd81f5d9790\nr8 0x7fd81f5d9860\nr9 0x7fd866bc24e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7fd81f5d9b60\nr14 0x1077594510e0\nr15 0xffffffffffffffff\nrip 0x7fd866a9c94c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595527e8 sp=0x1077595527e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0x1077596043c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x10775961a740 sp=0x10775961a720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x10775961a7c8 sp=0x10775961a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961a7e0 sp=0x10775961a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961a7e8 sp=0x10775961a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0x1077597883c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559740 sp=0x107759559720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077595597c8 sp=0x107759559740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077595597e0 sp=0x1077595597c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595597e8 sp=0x1077595597e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0x1077597885a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559f40 sp=0x107759559f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759559fc8 sp=0x107759559f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759559fe0 sp=0x107759559fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759559fe8 sp=0x107759559fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0x1077599841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759552f40 sp=0x107759552f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759552fc8 sp=0x107759552f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759552fe0 sp=0x107759552fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759552fe8 sp=0x107759552fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0x1077596045a0 m=4 mp=0x107759582008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x10775961aea0 sp=0x10775961ae90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x10775961af40 sp=0x10775961aea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x10775961afc8 sp=0x10775961af40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961afe0 sp=0x10775961afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961afe8 sp=0x10775961afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 10 gp=0x107759788780 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207ccb99?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759616740 sp=0x107759616720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077596167c8 sp=0x107759616740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077596167e0 sp=0x1077596167c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077596167e8 sp=0x1077596167e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax 0x0\nrbx 0x7fd81f5da640\nrcx 0x7fd866a9c94c\nrdx 0x6\nrdi 0xa70c\nrsi 0xa715\nrbp 0xa715\nrsp 0x7fd81f5d9790\nr8 0x7fd81f5d9860\nr9 0x7fd866bc24e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7fd81f5d9b60\nr14 0x1077594510e0\nr15 0xffffffffffffffff\nrip 0x7fd866a9c94c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}},{"row_id":"69e4b0fe75f9ced8b37c8f67fb051d047e4efe88e6f3d1b274e1ff14e65db359","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"150ce84edc5920033c79f1d71e7933469c3d65ffd64a306a05fe2dabbc654d50","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x327f918b1650\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x327f918b1650\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"6e006293ec8d08e090ef03eefd19448d31b9ee46c3ebe7907fff99076cbb1028","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"bb412fd2d485854280b1c9d07b101b3e663f2d9f084fff735671240db5800f4f","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1319a6308af0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1319a6308af0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"817d54e9806b87cc64b891dd20efaf7a325768152c79179dc5b290db79cb0158","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"alert","signature_id":"6fbf7b7865ca254c8d602bde28984e80b575972637dde5ddb43c09fae006779f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 11 time(s) (3 unknown, 8 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T13:16:41Z\n Ended: 2026-10-23T13:26:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6471040-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s\", pod=\"cluster-network-operator-6857fbf8d7-5cgmz\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s/cluster-network-operator-6857fbf8d7-5cgmz has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T13:16:41Z\n Ended: 2026-10-23T13:26:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6471040-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s\", pod=\"ingress-operator-787469b994-cfnzc\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s/ingress-operator-787469b994-cfnzc has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-23T13:16:41Z\n Ended: 2026-10-23T13:26:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6471040-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s\", pod=\"dns-operator-6d4899cf84-7qfzv\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s/dns-operator-6d4899cf84-7qfzv has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"87198aa68b978aa4fa23f281997335ee5d094a66a90fe9525872b6c64671c26c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"4e8926f9374f6904cc9826c0dee209a2e2cebf4787bc66fa48d1d8df4d4e2067","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x36fd45eaa060\u003e: \n resource group \"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-jdxf2b\" in resourcegroup=\"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\",\n \"name\": \"9861ff20-e277-4831-93f1-1162afbd108a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:01:17.842344425Z\",\n \"endTime\": \"2026-10-23T13:25:21.427910502Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkg8fusv1drnq89nr4jr4kc118h003a still exists (deletion dispatched at 2026-10-23T13:01:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n correlation ID: ef1977d2-31af-4849-9940-4bbd6bbe201a\n client request ID: 363124c1-5480-4435-98da-729155a01144\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x36fd45eaa060\u003e: \n resource group \"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-jdxf2b\" in resourcegroup=\"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\",\n \"name\": \"9861ff20-e277-4831-93f1-1162afbd108a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:01:17.842344425Z\",\n \"endTime\": \"2026-10-23T13:25:21.427910502Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkg8fusv1drnq89nr4jr4kc118h003a still exists (deletion dispatched at 2026-10-23T13:01:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n correlation ID: ef1977d2-31af-4849-9940-4bbd6bbe201a\n client request ID: 363124c1-5480-4435-98da-729155a01144\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a15585f27b15a0beb7b694961587ff9aa26e6273cd5d5eacb9eb2dc19e48f0cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"06077bac2bdfbefc430aa21e7f1c63585e4b0facfbed8535434aa7846c098a94","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x33f93d769890\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x33f93d769890\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"a64f7937108fc8e5b7a4d94c75a9a0a6b25620c2f627af6b3606a2e31444fb06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"70670de7f9371c033d82071ca8d86a5ee534cd8acfcecfb1faac595d80a44997","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-21-6jd898\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1690edf5cbb0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-21-6jd898\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1690edf5cbb0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"b7247d6416224591356eb1d1103849fbb2b0f69c17a6d017a939bb7aeeea8876","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"905453ac6e335acb4f35cd240fa3b7536659d0d95ac9186a5592efab0f975eea","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-5-0-ct2hxc\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1c7daddd6340\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-5-0-ct2hxc\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1c7daddd6340\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"e2a4a12303c6624876cb14958c8a132102c4de1224a91bf843990f6f594711e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"d879674894167409266496657d54be06c8aa7f724945913ba06a1ba076ac2290","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-22-f9qq4x\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3292fdd6e850\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-22-f9qq4x\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3292fdd6e850\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"e94770ca09d2f44d33b7185c06a8bd56a44209a99bb11c1b7851d17bbf47aea8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"99fcfb8a0f2194c211d91e98d4874fd0b7609259139ff0a43afeb62267c42148","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x247eb91d9c60\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x247eb91d9c60\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"edb63f2dd199cd7474d4e0e124e4fcd740993efdf306f06a40abf037eade9b44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","failed_at":"e2e","signature_id":"5d631566a7a8e50a23a29e16a0e18527e365de9f11bacc81791b4a3fa0dd48df","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2fa2959cc2d0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2fa2959cc2d0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}}],"failure_pattern_summary":{"signature_count":19,"failure_pattern_ids":["392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","bf2604f0a57cc83f42180ea2d978fdb012e4de2c6d11e90a5b15c16b2e73342d"],"matched_occurrences":19,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7237/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113584077738086400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7237,"pr_state":"open","pr_sha":"3c589ea378f756ab2139d90d5e1ebde66e5f6212","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T10:51:18Z","started_at":"2026-10-23T10:51:18Z","completed_at":"2026-10-23T13:10:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T13:13:11.460854451Z","timing_metadata_checked_at":"2026-10-23T13:13:11.460854451Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T13:13:11.463817541Z","region_metadata_checked_at":"2026-10-23T13:13:11.463817541Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113579905894584320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T10:34:52Z","started_at":"2026-10-23T10:34:52Z","completed_at":"2026-10-23T10:35:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:38:31.719713986Z","timing_metadata_checked_at":"2026-10-23T10:38:31.719713986Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-23T10:38:31.722652376Z","region_metadata_checked_at":"2026-10-23T11:40:00.698769121Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"18d561d0630d7bc635e78e8c827375e9f362a8d9b1f06431b42f588010502087","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113579905894584320","occurred_at":"2026-10-23T10:34:52Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113579232666849280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"closed","pr_sha":"08ce1d2ebc586eef03841c656a3e31593041645f","final_merged_sha":"6405f41e9d626eac01c2304a4b0fb1eefd6c1e81","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T10:32:12Z","started_at":"2026-10-23T10:32:12Z","completed_at":"2026-10-23T12:36:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T12:42:38.05894022Z","timing_metadata_checked_at":"2026-10-23T12:42:38.05894022Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T12:42:38.06241261Z","region_metadata_checked_at":"2026-10-23T12:42:38.06241261Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"68c2b31ca8077012cc08591db3fa83efa6fa6fa790ca3189e6af8bdb38cd3c35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113579232666849280","occurred_at":"2026-10-23T10:32:12Z","failed_at":"e2e","signature_id":"9813e169df513a9dded1ccb5b4bf7b1a057e15d6c3c68da4e0c1eb71f143dc1c","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x3fb2ef80a5e8\u003e: \n VerifyNodesReady failed: there are not ready nodes: cilium-cl-cilium-np-vlnxf-8twbs (status=False, reason=KubeletNotReady, message=container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, since=2026-10-23T11:44:16Z)\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-mj26w2hgsz8z\" to finish creating: GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\",\n \"name\": \"e205c3df-0cc8-46bb-84fb-519cf03956a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T11:25:56.40229713Z\",\n \"endTime\": \"2026-10-23T11:45:02.834813538Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-vlnxf-8twbs: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-vlnxf-8twbs: Failed: interfaces failed to create or update. err: failed to get existing resource cni-cilium-mj26w2hgsz8z--managed/cilium-cl-cilium-np-vlnxf-8twbs-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-mj26w2hgsz8z--managed/providers/Microsoft.Network/networkInterfaces/cilium-cl-cilium-np-vlnxf-8twbs-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://ak... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\n correlation ID: 98f92748-9012-4fff-87e0-a43e42d009b7\n client request ID: adb6cc59-3412-4269-9828-a7c6939df4dc\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x3fb2ef80a5e8\u003e: \n VerifyNodesReady failed: there are not ready nodes: cilium-cl-cilium-np-vlnxf-8twbs (status=False, reason=KubeletNotReady, message=container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, since=2026-10-23T11:44:16Z)\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-mj26w2hgsz8z\" to finish creating: GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\",\n \"name\": \"e205c3df-0cc8-46bb-84fb-519cf03956a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T11:25:56.40229713Z\",\n \"endTime\": \"2026-10-23T11:45:02.834813538Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-vlnxf-8twbs: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-vlnxf-8twbs: Failed: interfaces failed to create or update. err: failed to get existing resource cni-cilium-mj26w2hgsz8z--managed/cilium-cl-cilium-np-vlnxf-8twbs-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-mj26w2hgsz8z--managed/providers/Microsoft.Network/networkInterfaces/cilium-cl-cilium-np-vlnxf-8twbs-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://ak... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\n correlation ID: 98f92748-9012-4fff-87e0-a43e42d009b7\n client request ID: adb6cc59-3412-4269-9828-a7c6939df4dc\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113578553021829120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6823,"pr_state":"closed","pr_sha":"7675cd70bbd032c6518e23950b0738453ba0352f","final_merged_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T10:29:30Z","started_at":"2026-10-23T10:29:30Z","completed_at":"2026-10-23T12:39:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T12:42:38.245922885Z","timing_metadata_checked_at":"2026-10-23T12:42:38.245922885Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T12:42:38.248884376Z","region_metadata_checked_at":"2026-10-23T12:42:38.248884376Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113577901101158400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T10:26:54Z","started_at":"2026-10-23T10:26:54Z","completed_at":"2026-10-23T10:27:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:38:31.409529893Z","timing_metadata_checked_at":"2026-10-23T10:38:31.409529893Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-23T10:38:31.412849282Z","region_metadata_checked_at":"2026-10-23T11:40:00.601080984Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"51c036079b04b94e4807a88907f0b5e1de09f5e46bb5b65794380e3dec951897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113577901101158400","occurred_at":"2026-10-23T10:26:54Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113572065633112064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T10:03:43Z","started_at":"2026-10-23T10:03:43Z","completed_at":"2026-10-23T10:25:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:38:31.240068943Z","timing_metadata_checked_at":"2026-10-23T10:38:31.240068943Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-23T10:38:31.246276623Z","region_metadata_checked_at":"2026-10-23T11:40:00.501509655Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"61a404be9dcad36f99193137e12ba0afa465d970f12903d14097167b1bb733e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113572065633112064","occurred_at":"2026-10-23T10:03:43Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113567629972082688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"0d965839a628d29699c887cc0aaeee8761a5136f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T09:46:05Z","started_at":"2026-10-23T09:46:05Z","completed_at":"2026-10-23T10:02:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:06:48.058083794Z","timing_metadata_checked_at":"2026-10-23T10:06:48.058083794Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-23T10:06:48.063886575Z","region_metadata_checked_at":"2026-10-23T11:10:40.921377599Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fbbb70946c987b0c83fc68b1e2cfc2d0b569a4f3c2566d1bd6ed4c3a2730ec67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113567629972082688","occurred_at":"2026-10-23T09:46:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7711/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113559441314942976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7711,"pr_state":"closed","pr_sha":"594817273d6d32829fbd4bd4d593d044d4d06f63","final_merged_sha":"594817273d6d32829fbd4bd4d593d044d4d06f63","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T09:13:33Z","started_at":"2026-10-23T09:13:33Z","completed_at":"2026-10-23T10:27:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:38:30.957541661Z","timing_metadata_checked_at":"2026-10-23T10:38:30.957541661Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T10:38:30.963667441Z","region_metadata_checked_at":"2026-10-23T10:38:30.963667441Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"33d6eb0a363f4c1f5cd9d1b4d71cd0c57ff910dd5da900a5c71bb04d7eef51e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7711/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113559441314942976","occurred_at":"2026-10-23T09:13:33Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-23T10:04:28.865Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-23T10:04:30.857Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-23T10:04:30.857Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-23T10:04:32.118Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-23T10:04:32.118Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T10:04:32.118Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j4942976-svc\ntime=2026-10-23T10:19:36.095Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113552356485369856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7653,"pr_state":"closed","pr_sha":"1c5f99177052e49fb875d9503c65dd8f8ccd901f","final_merged_sha":"1c5f99177052e49fb875d9503c65dd8f8ccd901f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T08:45:24Z","started_at":"2026-10-23T08:45:24Z","completed_at":"2026-10-23T10:51:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T11:00:23.731114941Z","timing_metadata_checked_at":"2026-10-23T11:00:23.731114941Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T11:00:23.734491931Z","region_metadata_checked_at":"2026-10-23T11:00:23.734491931Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7708/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113549009753739264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7708,"pr_state":"open","pr_sha":"74a52de593438c730c6d1f3d637a258a1433f7d3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T08:32:06Z","started_at":"2026-10-23T08:32:06Z","completed_at":"2026-10-23T10:43:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:49:27.084747333Z","timing_metadata_checked_at":"2026-10-23T10:49:27.084747333Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T10:49:27.090355018Z","region_metadata_checked_at":"2026-10-23T10:49:27.090355018Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b32fcd7e379870126f4fbc2f6fc18fd208848a5a0d5e94ba13c70037583b9c6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7708/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113549009753739264","occurred_at":"2026-10-23T08:32:06Z","failed_at":"e2e","signature_id":"504e0ef2def37e129dc9af51f3cea8c5643dc0df0434eba4232abc7a90d05a61","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a2ce0502aa0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2q9b4whtt4bx\" to finish creating: GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n \"name\": \"ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:46:54.928693784Z\",\n \"endTime\": \"2026-10-23T10:05:56.71052837Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: Creating: virtualmachine creating or updating\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n correlation ID: b9eeb7ac-2571-4ebc-8877-daaa0fdd2739\n client request ID: 12c440df-92bf-4a8e-95cd-5978adfb18ec\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a2ce0502aa0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2q9b4whtt4bx\" to finish creating: GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n \"name\": \"ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:46:54.928693784Z\",\n \"endTime\": \"2026-10-23T10:05:56.71052837Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: Creating: virtualmachine creating or updating\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n correlation ID: b9eeb7ac-2571-4ebc-8877-daaa0fdd2739\n client request ID: 12c440df-92bf-4a8e-95cd-5978adfb18ec\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113548717339447296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7665,"pr_state":"closed","pr_sha":"d275b2b37224325c68e49510de4444cf2bca1e2f","final_merged_sha":"b40bc5cbb80a171bde8ca5e40c43b439603f5493","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T08:30:48Z","started_at":"2026-10-23T08:30:48Z","completed_at":"2026-10-23T10:53:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T11:00:23.980298518Z","timing_metadata_checked_at":"2026-10-23T11:00:23.980298518Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T11:00:23.984036007Z","region_metadata_checked_at":"2026-10-23T11:00:23.984036007Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"db6c10e70af402a12d7cb4be9641d14cafdd4ccc3cbed5bde6c3e6710077073f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113548717339447296","occurred_at":"2026-10-23T08:30:48Z","failed_at":"e2e","signature_id":"eac87251b067df59431cf544f3a76dfd222fded34dfba939a62a417723a28785","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x174cbf30c80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-nwv4f2dxcv4b\" to finish creating: GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\",\n \"name\": \"7be20667-78b5-4033-a3f1-3a092d6628f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:40:10.449361114Z\",\n \"endTime\": \"2026-10-23T09:59:13.321219203Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-nwv4f2dxcv4b--managed/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-nwv4f2dxcv4b--managed/providers/Microsoft.Network/networkInterfaces/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available. Refer... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n correlation ID: 6164f95f-1b20-49d8-a767-33fc06ba3b2f\n client request ID: 11eb729f-0d89-4edd-a41e-8cb113738ef8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x174cbf30c80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-nwv4f2dxcv4b\" to finish creating: GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\",\n \"name\": \"7be20667-78b5-4033-a3f1-3a092d6628f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:40:10.449361114Z\",\n \"endTime\": \"2026-10-23T09:59:13.321219203Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-nwv4f2dxcv4b--managed/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-nwv4f2dxcv4b--managed/providers/Microsoft.Network/networkInterfaces/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available. Refer... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n correlation ID: 6164f95f-1b20-49d8-a767-33fc06ba3b2f\n client request ID: 11eb729f-0d89-4edd-a41e-8cb113738ef8\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5bd4868e5b799ebb3599c1c4a1a9e5adf369cd18ab421a17e5fc658ab1e93070","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["5bd4868e5b799ebb3599c1c4a1a9e5adf369cd18ab421a17e5fc658ab1e93070"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113546001531801600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7709,"pr_state":"closed","pr_sha":"6d5d6acdf16d24f21224286cd936ea39a6baa4c5","final_merged_sha":"6d5d6acdf16d24f21224286cd936ea39a6baa4c5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T08:20:08Z","started_at":"2026-10-23T08:20:08Z","completed_at":"2026-10-23T10:42:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:49:27.235091728Z","timing_metadata_checked_at":"2026-10-23T10:49:27.235091728Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T10:49:27.243963604Z","region_metadata_checked_at":"2026-10-23T10:49:27.243963604Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"47266996af4087ad3f2941e3b3ec390b9b8d7800737f2bc79e54bb19a38c783c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113546001531801600","occurred_at":"2026-10-23T08:20:08Z","failed_at":"e2e","signature_id":"51444fe40b9218630af21f6e92d01bfeda9c0f66e83382df75ec9d39e17d9e40","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x14521ece600\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n err: \u003c*verifiers.pollObservationError | 0x14521ece580\u003e{\n observation: \u003c*errors.errorString | 0x14522933200\u003e{\n s: \"dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n },\n interruption: \u003c*fmt.wrapError | 0x14521ece560\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1452193c140\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x14521ece540\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x14522d0f260\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x14521ece600\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n err: \u003c*verifiers.pollObservationError | 0x14521ece580\u003e{\n observation: \u003c*errors.errorString | 0x14522933200\u003e{\n s: \"dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n },\n interruption: \u003c*fmt.wrapError | 0x14521ece560\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1452193c140\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x14521ece540\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x14522d0f260\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"30002276c0d676e1121ab63d27800e924a0d329b41c2b24beb8a2391f90d92df","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}},{"row_id":"bed607c8e923e263217cdd4a02c8de17c27e370a9101ca3b74c2e89fd77557ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113546001531801600","occurred_at":"2026-10-23T08:20:08Z","failed_at":"e2e","signature_id":"40c4710e56e4b7283917fa74062c14639587a07ba8e4e6a92442bf16cdee182b","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x37701b1c12c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-snrv785b6qc8\" to finish creating: GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n \"name\": \"dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:27:43.519923521Z\",\n \"endTime\": \"2026-10-23T09:46:52.566133312Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-snrv785b6qc8--managed/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-snrv785b6qc8--managed/providers/Microsoft.Network/networkInterfaces/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refe... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n correlation ID: f689e4cc-97e8-4962-aa14-27d872557ae0\n client request ID: d83db591-fd3e-4280-8e24-4c7c60560e28\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x37701b1c12c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-snrv785b6qc8\" to finish creating: GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n \"name\": \"dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:27:43.519923521Z\",\n \"endTime\": \"2026-10-23T09:46:52.566133312Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-snrv785b6qc8--managed/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-snrv785b6qc8--managed/providers/Microsoft.Network/networkInterfaces/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refe... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n correlation ID: f689e4cc-97e8-4962-aa14-27d872557ae0\n client request ID: d83db591-fd3e-4280-8e24-4c7c60560e28\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e9ecd96fd76a659f151ac5df30d476a4d8045c25ee424b28225788c72e8971d2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refe... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["30002276c0d676e1121ab63d27800e924a0d329b41c2b24beb8a2391f90d92df","e9ecd96fd76a659f151ac5df30d476a4d8045c25ee424b28225788c72e8971d2"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113545371434094592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7687,"pr_state":"open","pr_sha":"29047bc05a8801dfd0dc8a18882cc86b63eaed7c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T08:17:38Z","started_at":"2026-10-23T08:17:38Z","completed_at":"2026-10-23T10:26:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:38:30.788553309Z","timing_metadata_checked_at":"2026-10-23T10:38:30.788553309Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T10:38:30.791687799Z","region_metadata_checked_at":"2026-10-23T10:38:30.791687799Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"67e27aaafd9d9cea696ca17ad1d19159b914b858c72f67b24374107130e4b438","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113545371434094592","occurred_at":"2026-10-23T08:17:38Z","failed_at":"e2e","signature_id":"ff19091ea6b6fd108dd7780cdd577078ceaa836eeffb4b21c27d5ef11633d976","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d7f216af3c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-sw22pzbqp5qg\" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\",\n \"name\": \"57d3456e-a011-478a-861a-14e4fe67f873\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:12:07.672503322Z\",\n \"endTime\": \"2026-10-23T09:31:10.853698527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\n correlation ID: 62c61aed-7617-418e-886e-be6374e7f22a\n client request ID: eb9543dc-eb4b-4b68-a493-4efa1c23921e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d7f216af3c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-sw22pzbqp5qg\" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\",\n \"name\": \"57d3456e-a011-478a-861a-14e4fe67f873\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:12:07.672503322Z\",\n \"endTime\": \"2026-10-23T09:31:10.853698527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\n correlation ID: 62c61aed-7617-418e-886e-be6374e7f22a\n client request ID: eb9543dc-eb4b-4b68-a493-4efa1c23921e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ee218b7b8098067e4df18424c6d4ed94c402f4234957a450ab56e360cb1310e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113545371434094592","occurred_at":"2026-10-23T08:17:38Z","failed_at":"e2e","signature_id":"3582f51ef43ab6cebf49738a8faa79e10138b0aa9aed91bb247ce640202ba0e7","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.004s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x231711c56ee0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.004s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x231711c56ee0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893-\u003e172.30.0.10:53: read: connection refused\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}},{"row_id":"ff88578ee6dd312c5d410bfd7386a0980f80cb795507bc7faf524244d333c927","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113545371434094592","occurred_at":"2026-10-23T08:17:38Z","failed_at":"e2e","signature_id":"f9315d1a908c869487b8528009d6e3be71e485ce7d3c349af0d4312ee843c244","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xfba86cf63e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-gkpx2dhvd6bm\" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\",\n \"name\": \"7f0ee636-256c-422d-a7d4-36addd512bcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:14:45.906565692Z\",\n \"endTime\": \"2026-10-23T09:33:49.808866171Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-m7jn2-ljktd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-m7jn2-ljktd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n correlation ID: 02fd815c-7a91-4e34-a2c7-a3013c24d175\n client request ID: f1495231-4c88-4b44-9a23-53972ede44a2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xfba86cf63e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-gkpx2dhvd6bm\" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\",\n \"name\": \"7f0ee636-256c-422d-a7d4-36addd512bcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:14:45.906565692Z\",\n \"endTime\": \"2026-10-23T09:33:49.808866171Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-m7jn2-ljktd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-m7jn2-ljktd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n correlation ID: 02fd815c-7a91-4e34-a2c7-a3013c24d175\n client request ID: f1495231-4c88-4b44-9a23-53972ede44a2\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2913b68eb188eee155973da8f2278b815824c08e26b8568b029df43566c7bce9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2913b68eb188eee155973da8f2278b815824c08e26b8568b029df43566c7bce9","bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113544934832214016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"0d965839a628d29699c887cc0aaeee8761a5136f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T08:15:49Z","started_at":"2026-10-23T08:15:49Z","completed_at":"2026-10-23T08:59:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T09:03:46.036336836Z","timing_metadata_checked_at":"2026-10-23T09:03:46.036336836Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T09:03:46.039376427Z","region_metadata_checked_at":"2026-10-23T09:03:46.039376427Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3930c740e9db08084b3cc9b82631dcb1cc6d7b0a924d589630db711552e298e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113544934832214016","occurred_at":"2026-10-23T08:15:49Z","failed_at":"provision","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-23T08:49:29.045Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-23T08:49:29.470Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-23T08:49:29.470Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-23T08:49:29.655Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-23T08:49:29.801Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-23T08:49:29.801Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-23T08:49:29.867Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-23T08:54:32.503Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-23T08:54:32.504Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113542145401950208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7658,"pr_state":"open","pr_sha":"7c9b0b8c33d516c487fc648fef24aa2e517e9395","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T08:02:16Z","started_at":"2026-10-23T08:02:16Z","completed_at":"2026-10-23T10:25:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:38:30.527880256Z","timing_metadata_checked_at":"2026-10-23T10:38:30.527880256Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T10:38:30.531145245Z","region_metadata_checked_at":"2026-10-23T10:38:30.531145245Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"30cd3d22dbbf3a4a00aa5ba354e6dc0ef8e04cee6541304cdd162c8033d23be4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113542145401950208","occurred_at":"2026-10-23T08:02:16Z","failed_at":"e2e","signature_id":"d04d5f67613fcded3e78320b3bd7ce7a8186accddad999333f00e221da713c03","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3068a4b9eb40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-z7kvkzn6t5zb\" to finish creating: GET https://rp.j1950208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896\",\n \"name\": \"90af8b99-1ac0-4bf4-84da-3d4413591896\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:06:48.532373154Z\",\n \"endTime\": \"2026-10-23T09:25:48.512267136Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1950208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896\n correlation ID: f7aa4bb1-0464-427f-ba40-60c91211299f\n client request ID: 1f07b0d4-9ffc-4cd6-8bff-a623d13c31fb\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3068a4b9eb40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-z7kvkzn6t5zb\" to finish creating: GET https://rp.j1950208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896\",\n \"name\": \"90af8b99-1ac0-4bf4-84da-3d4413591896\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:06:48.532373154Z\",\n \"endTime\": \"2026-10-23T09:25:48.512267136Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1950208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896\n correlation ID: f7aa4bb1-0464-427f-ba40-60c91211299f\n client request ID: 1f07b0d4-9ffc-4cd6-8bff-a623d13c31fb","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f48bqq4pfgdf\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113540180034981888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7682,"pr_state":"closed","pr_sha":"f1986ec811a1588158df61aabf80ee48537cf900","final_merged_sha":"f1986ec811a1588158df61aabf80ee48537cf900","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T07:57:01Z","started_at":"2026-10-23T07:57:01Z","completed_at":"2026-10-23T10:13:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:17:39.527030513Z","timing_metadata_checked_at":"2026-10-23T10:17:39.527030513Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T10:17:39.530111205Z","region_metadata_checked_at":"2026-10-23T10:17:39.530111205Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113539016845430784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7463,"pr_state":"closed","pr_sha":"962fb789f66c3a28fca02c0fc2977023613c2549","final_merged_sha":"962fb789f66c3a28fca02c0fc2977023613c2549","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T07:52:23Z","started_at":"2026-10-23T07:52:23Z","completed_at":"2026-10-23T10:05:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:17:39.184735027Z","timing_metadata_checked_at":"2026-10-23T10:17:39.184735027Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T10:17:39.187946119Z","region_metadata_checked_at":"2026-10-23T10:17:39.187946119Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113536694941650944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7073,"pr_state":"closed","pr_sha":"04e6d776df28ac94ae9fef2631abe616677867d2","final_merged_sha":"9c4815df62c125480cb9a6a711a2a7ca92f94b47","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T07:43:10Z","started_at":"2026-10-23T07:43:10Z","completed_at":"2026-10-23T09:47:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T09:55:54.306898384Z","timing_metadata_checked_at":"2026-10-23T09:55:54.306898384Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T09:55:54.310506474Z","region_metadata_checked_at":"2026-10-23T09:55:54.310506474Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"be01687a9e550b3b3000da2bd62c34a87493af5137a1ad1e4cebc82443e3a029","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113536694941650944","occurred_at":"2026-10-23T07:43:10Z","failed_at":"alert","signature_id":"aeb0e7cf0f7fb6668a74127ede7587f34beff19ae9034d243e6f7865af3d860f","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T08:40:51Z\n Ended: 2026-10-23T08:58:52Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1650944-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T08:40:51Z\n Ended: 2026-10-23T08:57:52Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1650944-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7710/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113536268276076544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7710,"pr_state":"closed","pr_sha":"0ae5853ac62d4d40e984be7216cc557c2debaef2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T07:41:28Z","started_at":"2026-10-23T07:41:28Z","completed_at":"2026-10-23T09:59:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:06:48.444660716Z","timing_metadata_checked_at":"2026-10-23T10:06:48.444660716Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T10:06:48.448085605Z","region_metadata_checked_at":"2026-10-23T10:06:48.448085605Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113514603643146240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7709,"pr_state":"closed","pr_sha":"6d5d6acdf16d24f21224286cd936ea39a6baa4c5","final_merged_sha":"6d5d6acdf16d24f21224286cd936ea39a6baa4c5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T06:15:23Z","started_at":"2026-10-23T06:15:23Z","completed_at":"2026-10-23T08:19:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T08:22:11.705142922Z","timing_metadata_checked_at":"2026-10-23T08:22:11.705142922Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T08:22:11.707771113Z","region_metadata_checked_at":"2026-10-23T08:22:11.707771113Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2c9e6c81427d83a425d92d12ac7beed71e89a727faa1d6cd1136aff7792a6a8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113514603643146240","occurred_at":"2026-10-23T06:15:23Z","failed_at":"e2e","signature_id":"cbafe1b07c5b593f659af5b88993e3571384544466e1095e0be4c9ecc5336c65","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b310ced2400\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-d2zgjcg49b99\" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\",\n \"name\": \"8b884d06-af05-4a79-8c59-961aac29861b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T07:12:27.383685402Z\",\n \"endTime\": \"2026-10-23T07:31:36.955210198Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-vxg9v: Failed: availabilitysets failed to create or update. err: failed to get existing resource clusternp128-d2zgjcg49b99--managed/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/availabilitySets/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Ple... message truncated\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n correlation ID: 07968fa2-a7d8-4604-9db4-9bbf3995867c\n client request ID: f057ae85-7143-412c-b3d7-6a49973e5447\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b310ced2400\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-d2zgjcg49b99\" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\",\n \"name\": \"8b884d06-af05-4a79-8c59-961aac29861b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T07:12:27.383685402Z\",\n \"endTime\": \"2026-10-23T07:31:36.955210198Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-vxg9v: Failed: availabilitysets failed to create or update. err: failed to get existing resource clusternp128-d2zgjcg49b99--managed/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/availabilitySets/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Ple... message truncated\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n correlation ID: 07968fa2-a7d8-4604-9db4-9bbf3995867c\n client request ID: f057ae85-7143-412c-b3d7-6a49973e5447\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"146ba2f03b8e73d8b9aa488d4c220ca3b04255c2372536bbb59e6aa3a97d9e10","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Ple... message truncated Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a91666bc9db11f873b3403e1752d9207b86be8b69307f4d30fb2f92b880b9cb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113514603643146240","occurred_at":"2026-10-23T06:15:23Z","failed_at":"e2e","signature_id":"72e291739086d9b63e2a0faa5afedbb7022d6a44908bff0d2b6ef6a7ca8a9ab5","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-4m47cf\" for cluster \"cluster-hs-pre-4m47cf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x25f14c702960\u003e: \n failed to create NodePool np-4m47cf: failed waiting for nodepool=\"np-4m47cf\" for cluster \"cluster-hs-pre-4m47cf\" in resourcegroup=\"rg-hs-pre--phztwwdrkw7h\" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\",\n \"name\": \"d4f6c18f-891a-4c33-a2ba-8972525d31ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T07:15:22.43923977Z\",\n \"endTime\": \"2026-10-23T07:34:26.438466293Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\n correlation ID: dcef897a-e7d0-4e15-8934-33f203c26727\n client request ID: f4ae8e10-89f7-445a-9181-9c41d68ea128\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-4m47cf\" for cluster \"cluster-hs-pre-4m47cf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x25f14c702960\u003e: \n failed to create NodePool np-4m47cf: failed waiting for nodepool=\"np-4m47cf\" for cluster \"cluster-hs-pre-4m47cf\" in resourcegroup=\"rg-hs-pre--phztwwdrkw7h\" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\",\n \"name\": \"d4f6c18f-891a-4c33-a2ba-8972525d31ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T07:15:22.43923977Z\",\n \"endTime\": \"2026-10-23T07:34:26.438466293Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\n correlation ID: dcef897a-e7d0-4e15-8934-33f203c26727\n client request ID: f4ae8e10-89f7-445a-9181-9c41d68ea128\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c1d4f438c1a9c84e9dc24a19caf6f5f7dc48a7de5cc21873c1f48389432c4652","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113514603643146240","occurred_at":"2026-10-23T06:15:23Z","failed_at":"e2e","signature_id":"3e24e82e343cd7508ed60a7ec8730ea060e11e44c4fe22d5afc11ad0663e5eaa","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1f9142952260\u003e: \n DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1f9142952240\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1f9142952260\u003e: \n DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1f9142952240\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7c740bf3534ca5945b9fa94518bc4b7a520a009eef4fbd7a1fc2602d0f2b958","failure_pattern":"DNS for route host agnhost-e2e-sample-app-\u003cid\u003e","search_query":"DNS for route host agnhost-e2e-sample-app-s57f2"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["146ba2f03b8e73d8b9aa488d4c220ca3b04255c2372536bbb59e6aa3a97d9e10","d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","f7c740bf3534ca5945b9fa94518bc4b7a520a009eef4fbd7a1fc2602d0f2b958"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113511646260367360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7598,"pr_state":"closed","pr_sha":"e872b0b1784fd8f726ea3c9eef95a6da48012c10","final_merged_sha":"be965db2bf14d58e471ed21952ec3e210e01ecba","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T06:03:38Z","started_at":"2026-10-23T06:03:38Z","completed_at":"2026-10-23T08:13:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T08:22:10.908002295Z","timing_metadata_checked_at":"2026-10-23T08:22:10.908002295Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T08:22:10.911220084Z","region_metadata_checked_at":"2026-10-23T08:22:10.911220084Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113511186858250240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7653,"pr_state":"closed","pr_sha":"1c5f99177052e49fb875d9503c65dd8f8ccd901f","final_merged_sha":"1c5f99177052e49fb875d9503c65dd8f8ccd901f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T06:01:48Z","started_at":"2026-10-23T06:01:48Z","completed_at":"2026-10-23T08:11:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T08:22:11.320122113Z","timing_metadata_checked_at":"2026-10-23T08:22:11.320122113Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T08:22:11.323463402Z","region_metadata_checked_at":"2026-10-23T08:22:11.323463402Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b138fad304bf89895ad207b8b3c17aa9927d1f26f72d38d86256bae13a852d94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113511186858250240","occurred_at":"2026-10-23T06:01:48Z","failed_at":"e2e","signature_id":"8bc9007fc25b5640ebae9f4863d7359c865853bd666601aeb7d286abdb3e9538","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x23df1c5ad4a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x23df1caefde0\u003e{\n s: \"dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x23df1c5ad4a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x23df1caefde0\u003e{\n s: \"dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a83a5cad2332d517fe6d82b654adc4ab2b95557c26b786eb1ec2e48ab0c65371","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a83a5cad2332d517fe6d82b654adc4ab2b95557c26b786eb1ec2e48ab0c65371"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7463,"pr_state":"closed","pr_sha":"962fb789f66c3a28fca02c0fc2977023613c2549","final_merged_sha":"962fb789f66c3a28fca02c0fc2977023613c2549","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T05:21:32Z","started_at":"2026-10-23T05:21:32Z","completed_at":"2026-10-23T07:32:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T07:49:57.812861965Z","timing_metadata_checked_at":"2026-10-23T07:49:57.812861965Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T07:49:57.815963654Z","region_metadata_checked_at":"2026-10-23T07:49:57.815963654Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"01c502601d4119bc60e5862bd7cef49b99cf2bc1cad09c447e15f66e1f5020bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","failed_at":"e2e","signature_id":"5efae9732adc28512b06dafb9969fee7b2e6984fa695d99435471c1675c6ced2","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x325a69290ac0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-vq4tbh9zl5hx\" to finish creating: GET https://rp.j3270400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7\",\n \"name\": \"608a5015-c575-47f4-96b6-93f98f6b51f7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T06:24:56.751408267Z\",\n \"endTime\": \"2026-10-23T06:43:58.642073257Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3270400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7\n correlation ID: aa734835-f56e-4cf5-867a-994867172e04\n client request ID: 977d7eeb-364d-4bbf-931a-b35202d4aeda\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x325a69290ac0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-vq4tbh9zl5hx\" to finish creating: GET https://rp.j3270400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7\",\n \"name\": \"608a5015-c575-47f4-96b6-93f98f6b51f7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T06:24:56.751408267Z\",\n \"endTime\": \"2026-10-23T06:43:58.642073257Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3270400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7\n correlation ID: aa734835-f56e-4cf5-867a-994867172e04\n client request ID: 977d7eeb-364d-4bbf-931a-b35202d4aeda","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f48bqq4pfgdf\" to finish creating: GET https://rp"}},{"row_id":"3efcac708b33e844d5b8f0a931ddc38d3498c6a5fe540dbecd44e2fabd4e6988","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","failed_at":"e2e","signature_id":"6541df12b028842ffaa548f5c83c0ed141122c488181c09a33e556c71ac2d3e8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-xhrpwn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1788046d3680\u003e: \n DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x178804888840\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-xhrpwn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1788046d3680\u003e: \n DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x178804888840\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"7c99358a485ffa7af524e3dd63693216f0539fa896b2849a5751c68ad504f4d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","failed_at":"e2e","signature_id":"8211b3315a6019168c62a8ef8d3269cdc9b89dc80385b17b9be39252d09088e9","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade"}},{"row_id":"7f9e02d188bb492cf10df7c5369215db9d5221bb5bc356392c6a68cfcde68dda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","failed_at":"e2e","signature_id":"7dc8c3eafac5404affbb8c6a5aceaf23d8664d06a783b6429405a361c74eae60","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x9a09bb1e000\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n err: \u003c*errors.errorString | 0x9a09aa7e480\u003e{\n s: \"router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x9a09bb1e000\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n err: \u003c*errors.errorString | 0x9a09aa7e480\u003e{\n s: \"router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a346f2ee38025c72bdbab745d0e046d927c6651ff3fe7f95b5953a94c870fd22","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=\u003cuuid\u003e pass=\u003cuuid\u003e: expected healthy complete coverage","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}"}},{"row_id":"99f20e36d23700be0b2cb9532591f69f7e0be709df6b87decd4b9394d1d485ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","failed_at":"alert","signature_id":"19ad7954ab7f29b25a54f19b9d0544d2410dbddc79a38cc95515845245055d71","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T06:49:50Z\n Ended: 2026-10-23T06:56:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3270400-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.213:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3270400/providers/microsoft.monitor/accounts/hcps-j3270400\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le\", pod=\"capi-provider-7d64b7b878-2mwb4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"40ed8520-738a-4471-a9a5-0fd0a1d23770\"\n Description: Pod ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le/capi-provider-7d64b7b878-2mwb4 (manager) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-23T06:49:50Z\n Ended: 2026-10-23T06:56:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3270400-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.213:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3270400/providers/microsoft.monitor/accounts/hcps-j3270400\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le\", pod=\"capi-provider-7d64b7b878-2mwb4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"40ed8520-738a-4471-a9a5-0fd0a1d23770\"\n Description: Pod ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le/capi-provider-7d64b7b878-2mwb4 (manager) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"bb8f8e7b0f2f40efe3cd71693207dc166cfcaed8cd655c4adfa2ca83707eaa19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","failed_at":"e2e","signature_id":"8211b3315a6019168c62a8ef8d3269cdc9b89dc80385b17b9be39252d09088e9","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade"}},{"row_id":"d7685c6acb01823be5924d9e6c2531c5edd32e23c2c565cf1b09530dd89a0cd4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","failed_at":"e2e","signature_id":"8c2d12315219ca1579f718de9c20b9b687edb43eb1f14b0f2771853e17c295f1","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0x3221f83f4600\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n {\n msg: \"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n err: \u003c*errors.errorString | 0x3221fa87e2b0\u003e{\n s: \"DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0x3221f83f4600\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n {\n msg: \"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n err: \u003c*errors.errorString | 0x3221fa87e2b0\u003e{\n s: \"DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2f3aceed2bd993d066f5fc126ddd12907122d755568207fafbd6490e3839a54a","failure_pattern":"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet \u003cnamespace\u003e/global-pull-secret-syncer has no desired pods scheduled","search_query":"occurred"}},{"row_id":"f9bd79d8dfc9532ecff16e8f6f33d02cbebf4a7d96e7b3b41c5e8f29045eb4e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","failed_at":"e2e","signature_id":"93484e969eefe4c20a3e175e375caad69b1f31f45ea3188e5adb186dca3ed1fc","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.004s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: 0\nto be \u003e=\n \u003cint\u003e: 2\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.004s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: 0\nto be \u003e=\n \u003cint\u003e: 2","failure_pattern_match":{"status":"clustered","failure_pattern_id":"990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have at least 2 nodes (autoscaling min), found 0","search_query":"node pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["2f3aceed2bd993d066f5fc126ddd12907122d755568207fafbd6490e3839a54a","97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","a346f2ee38025c72bdbab745d0e046d927c6651ff3fe7f95b5953a94c870fd22","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113483940428779520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6770,"pr_state":"open","pr_sha":"07f510b66f9aa2c139caf8a69028d8ecae305451","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T04:13:32Z","started_at":"2026-10-23T04:13:32Z","completed_at":"2026-10-23T05:04:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T05:17:17.915377527Z","timing_metadata_checked_at":"2026-10-23T05:17:17.915377527Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T05:17:17.918871217Z","region_metadata_checked_at":"2026-10-23T05:17:17.918871217Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5f6a1789ec7c9fb5272473030109fcc7e4f0aa2eb1d8f4d310a38a331936d1e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113483940428779520","occurred_at":"2026-10-23T04:13:32Z","failed_at":"provision","signature_id":"5351510afe6cf56e3a8b6c40bacbd2bff16093b6e23691ab45c119900b535d4b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"templatize-pipeline","failure_text":"time=2026-10-23T04:51:07.246Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-23T04:52:17.695Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry registry.stage.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc to arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc.\\nThe image will still be available under it's original digest sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc in the target registry.\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed after 5 attempts: oras cp registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc --from-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json --to-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e081efea8bc058d3864749e2214faab4cad1dba916314a282090b76fc0651463","failure_pattern":"Command failed after 5 attempts: oras cp registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config","search_query":"Command failed after 5 attempts: oras cp registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc --from-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json --to-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e081efea8bc058d3864749e2214faab4cad1dba916314a282090b76fc0651463"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113459428752625664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7687,"pr_state":"open","pr_sha":"29047bc05a8801dfd0dc8a18882cc86b63eaed7c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T02:36:08Z","started_at":"2026-10-23T02:36:08Z","completed_at":"2026-10-23T04:39:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T04:56:33.240788604Z","timing_metadata_checked_at":"2026-10-23T04:56:33.240788604Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T04:56:33.246341281Z","region_metadata_checked_at":"2026-10-23T04:56:33.246341281Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1fe47f74f90c1d69a015bdc285ecb57226f5f55b696a9272c68453ca8f5e9362","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113459428752625664","occurred_at":"2026-10-23T02:36:08Z","failed_at":"e2e","signature_id":"361d3eae8393f6eef155ba76cade65d4102abb09f4a193a37d5f4161c2bf94bf","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x12108ffe4840\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gmjhx5ggwwt9\" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\",\n \"name\": \"74fee9f7-b402-4d5d-b683-6e4318e06904\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T03:34:18.354098626Z\",\n \"endTime\": \"2026-10-23T03:53:18.691661116Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\n correlation ID: b1f52151-27ba-447c-87a0-0859e9b312e7\n client request ID: f918cf71-16c2-4813-9b6f-6f13fae950d6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x12108ffe4840\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gmjhx5ggwwt9\" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\",\n \"name\": \"74fee9f7-b402-4d5d-b683-6e4318e06904\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T03:34:18.354098626Z\",\n \"endTime\": \"2026-10-23T03:53:18.691661116Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\n correlation ID: b1f52151-27ba-447c-87a0-0859e9b312e7\n client request ID: f918cf71-16c2-4813-9b6f-6f13fae950d6\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2b9bdd4041435fbb1ebcc8822849b13727525515ea0e45ab8a72b5dcb1c5ddfd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113459428752625664","occurred_at":"2026-10-23T02:36:08Z","failed_at":"e2e","signature_id":"a0d813b5252e54924e130bf68e119ec22802307c2b8775c3330243b5fac10ac1","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x2987b030ca40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rzw7qwnn5qjc\" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n \"name\": \"76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T03:22:03.46163902Z\",\n \"endTime\": \"2026-10-23T03:51:08.692840996Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 23m19s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n correlation ID: bb01406a-f12b-4a31-a989-90ce9d573bdf\n client request ID: f0973675-7d1b-4ddd-b467-290c8a664a39\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x2987b030ca40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rzw7qwnn5qjc\" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n \"name\": \"76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T03:22:03.46163902Z\",\n \"endTime\": \"2026-10-23T03:51:08.692840996Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 23m19s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n correlation ID: bb01406a-f12b-4a31-a989-90ce9d573bdf\n client request ID: f0973675-7d1b-4ddd-b467-290c8a664a39\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d00e36f774c9473d3e3006be646c433331ff2d9f4bffa58ae584755e17c97f3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113459428752625664","occurred_at":"2026-10-23T02:36:08Z","failed_at":"alert","signature_id":"b85e109c2f4dc5d74b9ee622dcf47c821f416845163d08b78b0f76a24ed1a9fa","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T03:34:48Z\n Ended: 2026-10-23T03:58:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2625664-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tk89mvlqfc5cf20qag1guihinhj7c0v-z6o9i3i9h0a8n5p\", pod=\"router-5cfd955d57-tpd7s\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tk89mvlqfc5cf20qag1guihinhj7c0v-z6o9i3i9h0a8n5p/router-5cfd955d57-tpd7s has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113458941882011648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7704,"pr_state":"closed","pr_sha":"59e60f95beccad9eb3025fd45308990ee9370b0d","final_merged_sha":"e7c23babe29805031def86c975833c6d3badf2cb","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T02:34:12Z","started_at":"2026-10-23T02:34:12Z","completed_at":"2026-10-23T04:49:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T05:07:05.031874276Z","timing_metadata_checked_at":"2026-10-23T05:07:05.031874276Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T05:07:05.035804567Z","region_metadata_checked_at":"2026-10-23T05:07:05.035804567Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7706/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113457466313281536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7706,"pr_state":"closed","pr_sha":"a2f066d3e7d19fa7f34bfa5927a455748bf793d7","final_merged_sha":"a2f066d3e7d19fa7f34bfa5927a455748bf793d7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T02:28:20Z","started_at":"2026-10-23T02:28:20Z","completed_at":"2026-10-23T04:41:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T04:56:33.661100014Z","timing_metadata_checked_at":"2026-10-23T04:56:33.661100014Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T04:56:33.664233401Z","region_metadata_checked_at":"2026-10-23T04:56:33.664233401Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113456944491532288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7688,"pr_state":"open","pr_sha":"5a4ce05f156772a7722d8a47aa6773bcc6d2458a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T02:26:16Z","started_at":"2026-10-23T02:26:16Z","completed_at":"2026-10-23T04:44:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T04:56:33.396362442Z","timing_metadata_checked_at":"2026-10-23T04:56:33.396362442Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T04:56:33.401092422Z","region_metadata_checked_at":"2026-10-23T04:56:33.401092422Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113446671063977984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"43a9a1c9a1caa8d8332e5913fbd7c37518e0d4b0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T01:45:26Z","started_at":"2026-10-23T01:45:26Z","completed_at":"2026-10-23T02:58:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T03:10:48.042868462Z","timing_metadata_checked_at":"2026-10-23T03:10:48.042868462Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T03:10:48.046289154Z","region_metadata_checked_at":"2026-10-23T03:10:48.046289154Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"9f33361483e77f54cc4c0b3ad35cb671f58b27f89d5425bb4713bd277efb9c69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113446671063977984","occurred_at":"2026-10-23T01:45:26Z","failed_at":"provision","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-23T02:42:44.129Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-23T02:42:44.420Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-23T02:42:44.420Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T02:42:44.639Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 namespace=secret-sync-controller\ntime=2026-10-23T02:42:44.852Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 dryRun=false\ntime=2026-10-23T02:42:44.852Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T02:42:44.905Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T02:47:48.850Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-23T02:47:48.850Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}},{"row_id":"a83aefef5b2b23f59eacd88dca7b0529211def9189372478ca38e84458f2a828","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113446671063977984","occurred_at":"2026-10-23T01:45:26Z","failed_at":"alert","signature_id":"7d7d5f484ae9cf25602860243ff6931ef9c6211bb6bd55469a3f901426372f60","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T02:40:08Z\n Ended: 2026-10-23T02:49:10Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j3977984-mgmt-2\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113434930414358528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7705,"pr_state":"closed","pr_sha":"78193d44c70069ac60e402c5ee2f646eadb201d4","final_merged_sha":"e2aa104a4154e9875791cd051472491971fa6720","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T00:58:47Z","started_at":"2026-10-23T00:58:47Z","completed_at":"2026-10-23T01:58:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T02:07:42.524413131Z","timing_metadata_checked_at":"2026-10-23T02:07:42.524413131Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T02:07:42.529933515Z","region_metadata_checked_at":"2026-10-23T02:07:42.529933515Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"616d201d15b298590d934fb1f69b1818d7a2eb1d4e7ef62a750641646a94b684","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113434930414358528","occurred_at":"2026-10-23T00:58:47Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-23T01:37:22.600Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-23T01:37:23.985Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-23T01:37:23.985Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-23T01:37:24.555Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-23T01:37:24.555Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T01:37:24.555Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j4358528-svc\ntime=2026-10-23T01:52:27.816Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113432151310471168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"43a9a1c9a1caa8d8332e5913fbd7c37518e0d4b0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T00:47:45Z","started_at":"2026-10-23T00:47:45Z","completed_at":"2026-10-23T01:31:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T01:46:07.587634413Z","timing_metadata_checked_at":"2026-10-23T01:46:07.587634413Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T01:46:07.590962604Z","region_metadata_checked_at":"2026-10-23T01:46:07.590962604Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"86d93ec2f702efd04dda8e470b8492ea9d53629be0150159c47211778105cc57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113432151310471168","occurred_at":"2026-10-23T00:47:45Z","failed_at":"provision","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-23T01:14:47.048Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-23T01:14:47.425Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-23T01:14:47.425Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T01:14:47.606Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 namespace=secret-sync-controller\ntime=2026-10-23T01:14:47.801Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 dryRun=false\ntime=2026-10-23T01:14:47.801Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T01:14:47.849Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T01:19:51.482Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-23T01:19:51.483Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113430486683488256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7495,"pr_state":"closed","pr_sha":"44f14d33feaaf1469b88ef579c0c604155ebe587","final_merged_sha":"c8ebae4103c841cfb1590b88cc329be1a9829823","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-23T00:41:08Z","started_at":"2026-10-23T00:41:08Z","completed_at":"2026-10-23T03:06:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T03:21:36.921370821Z","timing_metadata_checked_at":"2026-10-23T03:21:36.921370821Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T03:21:36.92508461Z","region_metadata_checked_at":"2026-10-23T03:21:36.92508461Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113420718787006464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7688,"pr_state":"open","pr_sha":"5a4ce05f156772a7722d8a47aa6773bcc6d2458a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-23T00:01:14Z","started_at":"2026-10-23T00:01:14Z","completed_at":"2026-10-23T02:12:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T02:29:16.758736063Z","timing_metadata_checked_at":"2026-10-23T02:29:16.758736063Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T02:29:16.768386538Z","region_metadata_checked_at":"2026-10-23T02:29:16.768386538Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2cb98aeae601922e7c271d6f488c00f9ab93e68e461b437f35065b35de63c450","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113420718787006464","occurred_at":"2026-10-23T00:01:14Z","failed_at":"e2e","signature_id":"0d9828b0bb1f27d9cc3639f58f494971e86d707f6ec7be8d51e1bfc562947fba","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x24b9ff37f340\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xqmhq7fznl4t\" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n \"name\": \"90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T00:52:55.985747163Z\",\n \"endTime\": \"2026-10-23T01:11:56.482302149Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-brczf to exist\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-z45q7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-z45q7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xqmhq7fznl4t--managed/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Network/networkInterfaces/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n correlation ID: 980d0943-0968-489f-acea-10a235919d44\n client request ID: 9fc719e7-58e9-4f80-97ec-7d7310001822\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x24b9ff37f340\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xqmhq7fznl4t\" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n \"name\": \"90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T00:52:55.985747163Z\",\n \"endTime\": \"2026-10-23T01:11:56.482302149Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-brczf to exist\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-z45q7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-z45q7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xqmhq7fznl4t--managed/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Network/networkInterfaces/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n correlation ID: 980d0943-0968-489f-acea-10a235919d44\n client request ID: 9fc719e7-58e9-4f80-97ec-7d7310001822\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0183c80982ac91e40ea21123d1f1a20353ca5fecf0db33e487b670cf6d4d4996","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"45a87dfdbefc9aa9c37c178c7ed34c68c20cba466b67acdf2ed87a7e40961f8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113420718787006464","occurred_at":"2026-10-23T00:01:14Z","failed_at":"e2e","signature_id":"4fa3b47449004f479ac51d54be8b694777e1df03e185828cb11995c7ea0071e0","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x5006b07f360\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-lzb9ghdnlbff\" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4\",\n \"name\": \"26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T00:53:35.812103666Z\",\n \"endTime\": \"2026-10-23T01:12:35.662761223Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4\n correlation ID: c531ae4a-649c-4e3c-a628-8707dc54dc8b\n client request ID: e95dcc96-488a-4fef-96e9-1e87cdadcd34\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x5006b07f360\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-lzb9ghdnlbff\" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4\",\n \"name\": \"26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T00:53:35.812103666Z\",\n \"endTime\": \"2026-10-23T01:12:35.662761223Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4\n correlation ID: c531ae4a-649c-4e3c-a628-8707dc54dc8b\n client request ID: e95dcc96-488a-4fef-96e9-1e87cdadcd34\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ad42859e07209f1114861895b936e12ef4ab1ccc24f62d90852515dca3b6318a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113420718787006464","occurred_at":"2026-10-23T00:01:14Z","failed_at":"e2e","signature_id":"6579f8d0a151b49461fd52fcea9ac6cb7fc7ffd497fe8e411acf99fb8274cfa3","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xd5c47f9e8e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-4ffqgf6wvkdd\" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005\",\n \"name\": \"de16b959-58ce-4c0d-a6d8-56ec91482005\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T00:56:40.180542426Z\",\n \"endTime\": \"2026-10-23T01:15:42.00975543Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005\n correlation ID: 5bb64d81-21d2-4966-9cfc-0f0641febce3\n client request ID: 70b4240c-c009-4928-97c5-bfd8b74a6560\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xd5c47f9e8e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-4ffqgf6wvkdd\" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005\",\n \"name\": \"de16b959-58ce-4c0d-a6d8-56ec91482005\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T00:56:40.180542426Z\",\n \"endTime\": \"2026-10-23T01:15:42.00975543Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005\n correlation ID: 5bb64d81-21d2-4966-9cfc-0f0641febce3\n client request ID: 70b4240c-c009-4928-97c5-bfd8b74a6560\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0183c80982ac91e40ea21123d1f1a20353ca5fecf0db33e487b670cf6d4d4996","f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]}]}]}