{"meta":{"date":"2026-04-02","timezone":"UTC","generated_at":"2026-10-31T22:13:44Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":69,"failed_runs":30,"runs_with_occurrences":30,"runs_with_failure_pattern_matches":25,"runs_with_unmatched_occurrences":5,"runs_without_occurrence_rows":39,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039833971700797440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"aa58964cb680b5b6ade57a062f99fb0bcb91efe1","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T22:34:51Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f62713fb36f79c5efde6398151f42d45ab2608a08069c73b2a0dfbb936a4b128","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039833971700797440","occurred_at":"2026-04-02T22:34:51Z","failed_at":"e2e","signature_id":"68c0a3c505cc04ac399ca6bd1c538cf324df259b78f922af5304567f2cd89b8c","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a32e40\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-bpwllj, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: failed waiting for cluster=\"np-version-upgrade-cluster-bpwllj\" in resourcegroup=\"rg-np-version-upgrade-bpwllj-plkrh6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-bpwllj, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: failed waiting for cluster=\\\"np-version-upgrade-cluster-bpwllj\\\" in resourcegroup=\\\"rg-np-version-upgrade-bpwllj-plkrh6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009c19a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6\",\n },\n \u003c*fmt.wrapErrors | 0xc000a328a0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-bpwllj\\\" in resourcegroup=\\\"rg-np-version-upgrade-bpwllj-plkrh6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009c19a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a32e40\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-bpwllj, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: failed waiting for cluster=\"np-version-upgrade-cluster-bpwllj\" in resourcegroup=\"rg-np-version-upgrade-bpwllj-plkrh6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-bpwllj, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: failed waiting for cluster=\\\"np-version-upgrade-cluster-bpwllj\\\" in resourcegroup=\\\"rg-np-version-upgrade-bpwllj-plkrh6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009c19a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6\",\n },\n \u003c*fmt.wrapErrors | 0xc000a328a0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-bpwllj\\\" in resourcegroup=\\\"rg-np-version-upgrade-bpwllj-plkrh6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009c19a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: failed waiting for cluster=\"np-version-upgrade-cluster-bpwllj\" in resourcegroup=\"rg-np-version-upgrade-bpwllj-plkrh6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bpwllj in resource group rg-np-version-upgrade-bpwllj-plkrh6, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4745/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039825828665102336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4745,"pr_state":"closed","pr_sha":"99e96f4559428dbe5230d03746fc0ac0dfcad360","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T22:02:30Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9367e37be615cb3b552da74fab8ad08431a2dd8d44f0403b0cb561b1761861ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4745/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039825828665102336","occurred_at":"2026-04-02T22:02:30Z","failed_at":"provision","signature_id":"f332a18ecda630e2a184fe05289bdad353416b3d510dcea88d09198c9377f182","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-04-02T23:03:56.735Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-04-02T23:03:57.419Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[armPermissionsManagerIdentity:map[certName:armHelperCert2 clientId:3331e670-0804-48e8-a086-6241671ddc93 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj5102336.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj5102336.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clusterScopedIdentitiesConfig:map[roleSetName:dev] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:06e749f2-4dc1-4350-b38e-e21264d3f4fa databaseName:arohcpprow-rp-j5102336 databaseUrl:https://arohcpprow-rp-j5102336.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-9bvn0l27/pipeline@sha256:65f03c580a3645b3375976c0bfefc797ab3ec100b36d5bd2362c81e57a961f14 replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:100m memory:500Mi]] zoneCount:4] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-9bvn0l27/pipeline:pull workloadIdentityClientId:5a9df00b-febb-4f42-8e0d-b40467b66a3c workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:06e749f2-4dc1-4350-b38e-e21264d3f4fa workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-04-02T23:03:57.419Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-04-02T23:03:57.479Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-04-02T23:03:57.594Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-04-02T23:03:57.594Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-04-02T23:03:57.706Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-04-02T23:09:02.481Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-04-02T23:09:02.481Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-04-02T23:09:02.481Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-04-02T23:09:05.429Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[armPermissionsManagerIdentity:map[certName:armHelperCert2 clientId:3331e670-0804-48e8-a086-6241671ddc93 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj5102336.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj5102336.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clusterScopedIdentitiesConfig:map[roleSetName:dev] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:06e749f2-4dc1-4350-b38e-e21264d3f4fa databaseName:arohcpprow-rp-j5102336 databaseUrl:https://arohcpprow-rp-j5102336.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-9bvn0l27/pipeline@sha256:65f03c580a3645b3375976c0bfefc797ab3ec100b36d5bd2362c81e57a961f14 replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:100m memory:500Mi]] zoneCount:4] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-9bvn0l27/pipeline:pull workloadIdentityClientId:5a9df00b-febb-4f42-8e0d-b40467b66a3c workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:06e749f2-4dc1-4350-b38e-e21264d3f4fa workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-04-02T23:09:05.430Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4xTTW8aMRC9768Y7QmkdYRIP1SqHGjaU5UKtT2l6sFrv4KD17Mae0FE/PjKEMoqWlSub96bmTf2KzwSCSJ3YhDpjqxOOunaY8T1E0z66oKdxSQuLCs6Qt90g38QNggHJLbanODxr4KIqJx3acXinnVyHBbsndmVFZXae96qBkmciarWZo1gDwVhtTJtWR3lP2AEaSG8cRZy73WMmfUiUBGycQZqjZ3a6M6nwRYHztwY7kLqqQe439njkwvWhWV/jIe2EAUPk21cHtKrnEydTA6IPqP1vGsQ0oBugL8AJN8TITmjX21yxbyXJR84uMRy3cx7Dn/c8kG3/Xvo506gpAvJNVDmQLlWe5Gdb5/h/x97bmTReT/wUG3nvarP+ElY/P5YrLsaEpAQv+QPG4s9bVcQUDYRk25aqpG2QKCR1QkZHpXTyfSdmrxRk+nP6e1scjt7+/6xHNPNDV3kfJhNpo/leFzs6YldoLUL9s6FAOnFjAOd09WP1es8FXtqhXP1vGhFLdsj+bomFXleVtQgRr3MLVkshOpdz7xFNH8DAAD//x/fd0cMBAAA\"\ntime=2026-04-02T23:09:05.432Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-04-02T23:09:05.432Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4695/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039802952352272384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4695,"pr_state":"closed","pr_sha":"6daf8c114ebd038152190aa5494897dc30b1119f","final_merged_sha":"3a0b8ba847a3b9f4913be118e29015fc3ad907bf","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T20:31:36Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6e8f4660db706d0caef193dd9fc1519ce4bf229074ce92618ba4aad16c9f7814","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4695/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039802952352272384","occurred_at":"2026-04-02T20:31:36Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039798611746754560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4133,"pr_state":"closed","pr_sha":"4f10ef55a408ea39a4f644c08d4ab2d01cf39c2a","final_merged_sha":"4f10ef55a408ea39a4f644c08d4ab2d01cf39c2a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T20:14:21Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039797287487541248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4633,"pr_state":"closed","pr_sha":"ba3f04026ea078c237d6bbfdb9a8efe05fa74961","final_merged_sha":"ba3f04026ea078c237d6bbfdb9a8efe05fa74961","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T20:09:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039795675343884288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"closed","pr_sha":"3fbc2b02fed0b6ac62c94496c0ae43094a0a41b5","final_merged_sha":"4fe7725f48a2f3f1c0ac843b4434f502807b38e5","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-02T20:02:17Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039785313722699776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"closed","pr_sha":"f7592c077483ca1a038e6f738e7625dab685f759","final_merged_sha":"4fe7725f48a2f3f1c0ac843b4434f502807b38e5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T19:21:30Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1e17161f8e1a09859e0e08699269fcc8602b72c11ee7779ba86508a01dde92c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039785313722699776","occurred_at":"2026-04-02T19:21:30Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-04-02T19:48:00.107Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-04-02T19:48:02.240Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-04-02T19:48:03.698Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=0c4911c1685a20609549f75db7625cacf2320f36305028898d025569bae2fa4c\ntime=2026-04-02T19:49:34.390Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j2699776/providers/Microsoft.Resources/deployments/0c4911c1685a20609549f75db7625cacf2320f36305028898d025569bae2fa4c/operationStatuses/08584264476026849074\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2699776/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2699776/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j2699776\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report c42af2c3-7d12-4171-9bd0-571348450d45:4/2/2026 7:49:23 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039774912159158272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4610,"pr_state":"open","pr_sha":"3d07b56b14da43b50647b97d828f5d1da565f69e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-02T18:40:10Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039765179733905408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4257,"pr_state":"closed","pr_sha":"f9b0dd45c8c2499c52d47ec79f8297cdae4dfe1f","final_merged_sha":"f9b0dd45c8c2499c52d47ec79f8297cdae4dfe1f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T18:01:17Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4737/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039758986357510144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4737,"pr_state":"closed","pr_sha":"746ba3b55d1f9fd1376d1584ed8fdb8331877f1c","final_merged_sha":"746ba3b55d1f9fd1376d1584ed8fdb8331877f1c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T17:36:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039756884201705472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"ff5b88c96712835bead8c845c79f61dfb754f685","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T17:28:32Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"24bc00d89aa721a3faa998f406a8da064e29763ab1be0a4b1f5af4c30c2d2ff8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039756884201705472","occurred_at":"2026-04-02T17:28:32Z","failed_at":"e2e","signature_id":"1641631fc25a83c856bf3a44c13dcdcc31609b9cab9b771b411ae29422fa4984","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:210]: Timed out after 600.004s.\nboth ImageDigestMirrorSet entries should exist on the hosted cluster\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000192db0\u003e: \n IDMS verification failed:\n source \"fake-source2.example.com/fake\" not found in any IDMS\n {\n s: \"IDMS verification failed:\\nsource \\\"fake-source2.example.com/fake\\\" not found in any IDMS\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:210]: Timed out after 600.004s.\nboth ImageDigestMirrorSet entries should exist on the hosted cluster\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000192db0\u003e: \n IDMS verification failed:\n source \"fake-source2.example.com/fake\" not found in any IDMS\n {\n s: \"IDMS verification failed:\\nsource \\\"fake-source2.example.com/fake\\\" not found in any IDMS\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7dbb3245ba2adfe36d40679785a616da1431c27d6cab66a4f5200c0ae786eac3","failure_pattern":"source \"fake-source2.example.com/fake\" not found in any IDMS","search_query":"source \"fake-source2.example.com/fake\" not found in any IDMS"}},{"row_id":"49ce90fde6ca8a04cbd2dc577124669dd58b60ba2be7862606cc7c61c410e535","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039756884201705472","occurred_at":"2026-04-02T17:28:32Z","failed_at":"e2e","signature_id":"54c243422f5df0c30e7732ac540ccdd1e1573e0977388b9b82998ccb9c682885","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"HCP/test/util/framework/hcp_helper.go:201 +0x218 fp=0xc0000dbef0 sp=0xc0000dbde8 pc=0x525c458\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:349 +0x94 fp=0xc0000dbf78 sp=0xc0000dbef0 pc=0x525ea54\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc0000dbfe0 sp=0xc0000dbf78 pc=0x3af0ab0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000dbfe8 sp=0xc0000dbfe0 pc=0x2dbecc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 850\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 861 gp=0xc000168000 m=nil [IO wait]:\nruntime.gopark(0xc0009da6c0?, 0x3100000000?, 0x50?, 0x7?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000c70720 sp=0xc000c70700 pc=0x2db650e\nruntime.netpollblock(0x2e20d18?, 0x2d42066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000c70758 sp=0xc000c70720 pc=0x2d75fb7\ninternal/poll.runtime_pollWait(0x7f75e94bf600, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000c70778 sp=0xc000c70758 pc=0x2db4da5\ninternal/poll.(*pollDesc).wait(0xc00050e500?, 0xc00147b000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000c707a0 sp=0xc000c70778 pc=0x2e43a47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc00050e500, {0xc00147b000, 0x3000, 0x3000})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000c70838 sp=0xc000c707a0 pc=0x2e44e79\nnet.(*netFD).Read(0xc00050e500, {0xc00147b000?, 0xf?, 0xc000cd8750?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000c70880 sp=0xc000c70838 pc=0x300a4e5\nnet.(*conn).Read(0xc00019cb68, {0xc00147b000?, 0x7f75e8391040?, 0x7f76318eaf30?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000c708c8 sp=0xc000c70880 pc=0x301c585\ncrypto/tls.(*atLeastReader).Read(0xc0010805a0, {0xc00147b000?, 0xc001440b01?, 0xf?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000c70910 sp=0xc000c708c8 pc=0x308801b\nbytes.(*Buffer).ReadFrom(0xc00075c9a8, {0x8c14100, 0xc0010805a0})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000c70968 sp=0xc000c70910 pc=0x2e81c78\ncrypto/tls.(*Conn).readFromUntil(0xc00075c708, {0x8c14ee0, 0xc00019cb68}, 0xc000c709d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000c709a0 sp=0xc000c70968 pc=0x30881fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc00075c708, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000c70c18 sp=0xc000c709a0 pc=0x308527b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc00075c708, {0xc000e62000, 0x1000, 0x34ca980?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000c70c80 sp=0xc000c70c18 pc=0x308bb65\nbufio.(*Reader).Read(0xc000fe4600, {0xc000948584, 0x9, 0x34d8a2e?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000c70cb8 sp=0xc000c70c80 pc=0x2f1e1d7\nio.ReadAtLeast({0x8c13700, 0xc000fe4600}, {0xc000948584, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000c70d00 sp=0xc000c70cb8 pc=0x2e10e8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000948584, 0x9, 0xc00000007b?}, {0x8c13700?, 0xc000fe4600?})\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:250 +0x65 fp=0xc000c70d50 sp=0xc000c70d00 pc=0x34c1745\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000948540)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:513 +0x6b fp=0xc000c70dd8 sp=0xc000c70d50 pc=0x34c230b\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000948540)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:572 +0x18 fp=0xc000c70e18 sp=0xc000c70dd8 pc=0x34c2a78\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000c70fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2254 +0xca fp=0xc000c70f60 sp=0xc000c70e18 pc=0x34d596a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc00097ee00)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2123 +0x52 fp=0xc000c70fc8 sp=0xc000c70f60 pc=0x34d4cf2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0x25 fp=0xc000c70fe0 sp=0xc000c70fc8 pc=0x34ce485\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000c70fe8 sp=0xc000c70fe0 pc=0x2dbecc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 860\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0xda5\n\nrax 0xca\nrbx 0x0\nrcx 0x2dc0ae3\nrdx 0x0\nrdi 0xc1bd5b0\nrsi 0x80\nrbp 0x7ffdf970e738\nrsp 0x7ffdf970e6f0\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x7ffdf970e718\nr13 0xc0000cf490\nr14 0xc1bb4c0\nr15 0xffffffffffffffff\nrip 0x2dc0ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nHCP/test/util/framework/hcp_helper.go:201 +0x218 fp=0xc0000dbef0 sp=0xc0000dbde8 pc=0x525c458\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:349 +0x94 fp=0xc0000dbf78 sp=0xc0000dbef0 pc=0x525ea54\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc0000dbfe0 sp=0xc0000dbf78 pc=0x3af0ab0\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000dbfe8 sp=0xc0000dbfe0 pc=0x2dbecc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 850\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 861 gp=0xc000168000 m=nil [IO wait]:\nruntime.gopark(0xc0009da6c0?, 0x3100000000?, 0x50?, 0x7?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000c70720 sp=0xc000c70700 pc=0x2db650e\nruntime.netpollblock(0x2e20d18?, 0x2d42066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000c70758 sp=0xc000c70720 pc=0x2d75fb7\ninternal/poll.runtime_pollWait(0x7f75e94bf600, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000c70778 sp=0xc000c70758 pc=0x2db4da5\ninternal/poll.(*pollDesc).wait(0xc00050e500?, 0xc00147b000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000c707a0 sp=0xc000c70778 pc=0x2e43a47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc00050e500, {0xc00147b000, 0x3000, 0x3000})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000c70838 sp=0xc000c707a0 pc=0x2e44e79\nnet.(*netFD).Read(0xc00050e500, {0xc00147b000?, 0xf?, 0xc000cd8750?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000c70880 sp=0xc000c70838 pc=0x300a4e5\nnet.(*conn).Read(0xc00019cb68, {0xc00147b000?, 0x7f75e8391040?, 0x7f76318eaf30?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000c708c8 sp=0xc000c70880 pc=0x301c585\ncrypto/tls.(*atLeastReader).Read(0xc0010805a0, {0xc00147b000?, 0xc001440b01?, 0xf?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000c70910 sp=0xc000c708c8 pc=0x308801b\nbytes.(*Buffer).ReadFrom(0xc00075c9a8, {0x8c14100, 0xc0010805a0})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000c70968 sp=0xc000c70910 pc=0x2e81c78\ncrypto/tls.(*Conn).readFromUntil(0xc00075c708, {0x8c14ee0, 0xc00019cb68}, 0xc000c709d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000c709a0 sp=0xc000c70968 pc=0x30881fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc00075c708, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000c70c18 sp=0xc000c709a0 pc=0x308527b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc00075c708, {0xc000e62000, 0x1000, 0x34ca980?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000c70c80 sp=0xc000c70c18 pc=0x308bb65\nbufio.(*Reader).Read(0xc000fe4600, {0xc000948584, 0x9, 0x34d8a2e?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000c70cb8 sp=0xc000c70c80 pc=0x2f1e1d7\nio.ReadAtLeast({0x8c13700, 0xc000fe4600}, {0xc000948584, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000c70d00 sp=0xc000c70cb8 pc=0x2e10e8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000948584, 0x9, 0xc00000007b?}, {0x8c13700?, 0xc000fe4600?})\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:250 +0x65 fp=0xc000c70d50 sp=0xc000c70d00 pc=0x34c1745\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000948540)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:513 +0x6b fp=0xc000c70dd8 sp=0xc000c70d50 pc=0x34c230b\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000948540)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:572 +0x18 fp=0xc000c70e18 sp=0xc000c70dd8 pc=0x34c2a78\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000c70fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2254 +0xca fp=0xc000c70f60 sp=0xc000c70e18 pc=0x34d596a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc00097ee00)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2123 +0x52 fp=0xc000c70fc8 sp=0xc000c70f60 pc=0x34d4cf2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0x25 fp=0xc000c70fe0 sp=0xc000c70fc8 pc=0x34ce485\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000c70fe8 sp=0xc000c70fe0 pc=0x2dbecc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 860\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0xda5\n\nrax 0xca\nrbx 0x0\nrcx 0x2dc0ae3\nrdx 0x0\nrdi 0xc1bd5b0\nrsi 0x80\nrbp 0x7ffdf970e738\nrsp 0x7ffdf970e6f0\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x7ffdf970e718\nr13 0xc0000cf490\nr14 0xc1bb4c0\nr15 0xffffffffffffffff\nrip 0x2dc0ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"a40229a98fa687100721d1899b8379c39b5a13fbd3feb97652495b1d962d2a95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039756884201705472","occurred_at":"2026-04-02T17:28:32Z","failed_at":"e2e","signature_id":"3daab75d8f207e629f011dd756c17a3891ac511c30923489fb33f6c4ff0955fc","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00146d6b0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-ml7xtq, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-ml7xtq\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-ml7xtq, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-ml7xtq\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a074e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-ml7xtq\",\n },\n \u003c*fmt.wrapError | 0xc000972dc0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-ml7xtq\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00146d6b0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-ml7xtq, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-ml7xtq\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-ml7xtq, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-ml7xtq\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a074e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-ml7xtq\",\n },\n \u003c*fmt.wrapError | 0xc000972dc0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-ml7xtq\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-ml7xtq, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-ml7xtq\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7dbb3245ba2adfe36d40679785a616da1431c27d6cab66a4f5200c0ae786eac3","9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039755570902208512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4652,"pr_state":"closed","pr_sha":"ef0f1b19b5b4a0415c36e782700ec263a51ca311","final_merged_sha":"ef0f1b19b5b4a0415c36e782700ec263a51ca311","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T17:23:19Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039754863004356608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4652,"pr_state":"closed","pr_sha":"ef0f1b19b5b4a0415c36e782700ec263a51ca311","final_merged_sha":"ef0f1b19b5b4a0415c36e782700ec263a51ca311","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T17:20:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039753468280836096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4633,"pr_state":"closed","pr_sha":"ba3f04026ea078c237d6bbfdb9a8efe05fa74961","final_merged_sha":"ba3f04026ea078c237d6bbfdb9a8efe05fa74961","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T17:14:58Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"87bd4d9f0ad662b20b56479eefa3ae75d367a42459fe54d453153bde006ff0e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039753468280836096","occurred_at":"2026-04-02T17:14:58Z","failed_at":"e2e","signature_id":"a93a34a24a571d29897ce6a71224cb99556e12ae10cd4dad21ee31adc77dd33e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:179]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000be9020\u003e: \n failed to create route: the server is currently unable to handle the request\n {\n msg: \"failed to create route: the server is currently unable to handle the request\",\n err: \u003c*errors.StatusError | 0xc000f014a0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"the server is currently unable to handle the request\",\n Reason: \"ServiceUnavailable\",\n Details: {\n Name: \"\",\n Group: \"\",\n Kind: \"\",\n UID: \"\",\n Causes: [\n {\n Type: \"UnexpectedServerResponse\",\n Message: \"error trying to reach service: tls: first record does not look like a TLS handshake\",\n Field: \"\",\n },\n ],\n RetryAfterSeconds: 0,\n },\n Code: 503,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:179]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000be9020\u003e: \n failed to create route: the server is currently unable to handle the request\n {\n msg: \"failed to create route: the server is currently unable to handle the request\",\n err: \u003c*errors.StatusError | 0xc000f014a0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"the server is currently unable to handle the request\",\n Reason: \"ServiceUnavailable\",\n Details: {\n Name: \"\",\n Group: \"\",\n Kind: \"\",\n UID: \"\",\n Causes: [\n {\n Type: \"UnexpectedServerResponse\",\n Message: \"error trying to reach service: tls: first record does not look like a TLS handshake\",\n Field: \"\",\n },\n ],\n RetryAfterSeconds: 0,\n },\n Code: 503,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eaacc54ebd46cefe15abd433d5c3a394c2c9eab33016bf077e0076e05e8a3f60","failure_pattern":"failed to create route: the server is currently unable to handle the request","search_query":"failed to create route: the server is currently unable to handle the request"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["eaacc54ebd46cefe15abd433d5c3a394c2c9eab33016bf077e0076e05e8a3f60"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039752772512911360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4633,"pr_state":"closed","pr_sha":"ba3f04026ea078c237d6bbfdb9a8efe05fa74961","final_merged_sha":"ba3f04026ea078c237d6bbfdb9a8efe05fa74961","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T17:12:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4706/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039752172580638720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4706,"pr_state":"closed","pr_sha":"b1ebea216f17cd14d1c5fa7d0361d70642b63c94","final_merged_sha":"b1ebea216f17cd14d1c5fa7d0361d70642b63c94","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T17:09:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039752085490110464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4607,"pr_state":"closed","pr_sha":"db17c8e13fefddc5d7923aa639d1a6106db776c8","final_merged_sha":"db17c8e13fefddc5d7923aa639d1a6106db776c8","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T17:09:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4723/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039744162546323456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4723,"pr_state":"closed","pr_sha":"b1ab92bdd1a079a7b6b823287f57b6cb3e0f6733","final_merged_sha":"b1ab92bdd1a079a7b6b823287f57b6cb3e0f6733","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T16:37:59Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"12d335184c17b8561f95131ec223ac5d15af4cb89e6106d6fd12a4a7671fcbbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4723/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039744162546323456","occurred_at":"2026-04-02T16:37:59Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4720/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039743836728594432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4720,"pr_state":"closed","pr_sha":"064b5a3c0fa857aa9bda97546dcca80e855fbfa5","final_merged_sha":"dedc03a96f4a129238aa9049c53b2ec64b775fd1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T16:36:40Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"b499c4a3f2cbd774ddc3811a577b3062eb82b2a062c416cf60c14f650158c426","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4720/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039743836728594432","occurred_at":"2026-04-02T16:36:40Z","failed_at":"e2e","signature_id":"ee8e15911f3b6073ee7809c5a33796f1f08e1e8a3dd29f64ef927f2ce1d09cf8","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"monitoring (Available=Unknown)\",\n ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]\nExpected\n \u003c[]string | len:1, cap:1\u003e: [\n \"monitoring (Available=Unknown)\",\n ]\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5af2875ed5aad41eea4ec5a530695c8926d7e79287a46eb27aaac5733592ac45","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]"}},{"row_id":"dcf12a6d943ad0b81fd7a216e85bafb76a4ecffb6d4f3b06adc8d30fd45f29a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4720/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039743836728594432","occurred_at":"2026-04-02T16:36:40Z","failed_at":"e2e","signature_id":"83061ad87f3cceef127f94300743da6698fb298b965b5a00b4ae3047fcadd256","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009657a0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-vdlznq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vdlznq, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-vdlznq\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vdlznq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0007d3b30\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vdlznq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009657a0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-vdlznq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vdlznq, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-vdlznq\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vdlznq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0007d3b30\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vdlznq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","5af2875ed5aad41eea4ec5a530695c8926d7e79287a46eb27aaac5733592ac45"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039742819844755456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4688,"pr_state":"closed","pr_sha":"eb1aee5ab64ac54ba0fa9ae82d8aec9a746ffffb","final_merged_sha":"3c3a0340cb6e442be122561ee15042522c88f102","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T16:32:39Z"},"failed_at":["e2e"],"failed_test_count":8,"occurrences":[{"row_id":"03453fcd3a4dc3414377476e83af3d7e4efaac6e91fb568be413180613e7632c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039742819844755456","occurred_at":"2026-04-02T16:32:39Z","failed_at":"e2e","signature_id":"a096ab46efcecdbde6b615ecad2febc50081c214de4e40765e5f69bf1395bd09","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008203a0\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-njnsx.apps.aro.a6o6f3b9m2q2s0q.dcs5.j4755456.hcp.osadev.cloud\": dial tcp 20.171.210.37:443: connect: connection timed out\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-njnsx.apps.aro.a6o6f3b9m2q2s0q.dcs5.j4755456.hcp.osadev.cloud\\\": dial tcp 20.171.210.37:443: connect: connection timed out\",\n err: \u003c*url.Error | 0xc001416000\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-njnsx.apps.aro.a6o6f3b9m2q2s0q.dcs5.j4755456.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000d00140\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: \u003c*net.TCPAddr | 0xc001416150\u003e{\n IP: [20, 171, 210, 37],\n Port: 443,\n Zone: \"\",\n },\n Err: \u003c*os.SyscallError | 0xc000820040\u003e{\n Syscall: \"connect\",\n Err: \u003csyscall.Errno\u003e0x6e,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008203a0\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-njnsx.apps.aro.a6o6f3b9m2q2s0q.dcs5.j4755456.hcp.osadev.cloud\": dial tcp 20.171.210.37:443: connect: connection timed out\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-njnsx.apps.aro.a6o6f3b9m2q2s0q.dcs5.j4755456.hcp.osadev.cloud\\\": dial tcp 20.171.210.37:443: connect: connection timed out\",\n err: \u003c*url.Error | 0xc001416000\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-njnsx.apps.aro.a6o6f3b9m2q2s0q.dcs5.j4755456.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000d00140\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: \u003c*net.TCPAddr | 0xc001416150\u003e{\n IP: [20, 171, 210, 37],\n Port: 443,\n Zone: \"\",\n },\n Err: \u003c*os.SyscallError | 0xc000820040\u003e{\n Syscall: \"connect\",\n Err: \u003csyscall.Errno\u003e0x6e,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2756b831e4b1e0a56a7949bf61fab69ac22501180f2bc329512550fb97dec7fa","failure_pattern":"route host was never found: Get \"\u003curl\u003e\": dial tcp \u003cip\u003e:\u003cport\u003e: connect: connection timed out","search_query":"route host was never found: Get \"https://agnhost-e2e-serving-app-lfmp6.apps.aro.p1p6p1b0v9r5h3k.f9ik.j4755456.hcp.osadev.cloud\": dial tcp 4.236.7.70:443: connect: connection timed out"}},{"row_id":"1794b57b1376e9fd559f3648b691a9933af913cef334572444989d12f9bff73a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039742819844755456","occurred_at":"2026-04-02T16:32:39Z","failed_at":"e2e","signature_id":"5ce944c2356de703e568b16794a622c30f0f63e43d6d9dcebc7ba560118691b2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014c4380\u003e: \n failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-gdtk5x\" to finish creating: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"cilium-np\\\" for cluster \\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-gdtk5x\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014c4380\u003e: \n failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-gdtk5x\" to finish creating: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"cilium-np\\\" for cluster \\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-gdtk5x\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-gdtk5x\" to finish creating: context deadline exceeded"}},{"row_id":"49d41ce3b4594fe7201bd605e677383f0498f0a9d301e640a9ac055a95ae9344","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039742819844755456","occurred_at":"2026-04-02T16:32:39Z","failed_at":"e2e","signature_id":"ae72c390b17e3841482d0f42ae09fc71b5dbef948ecb8213f2f12bcbe4a868b8","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a67860\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-46csdr\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-46csdr, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-46csdr\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-46csdr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000995eb0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-46csdr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a67860\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-46csdr\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-46csdr, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-46csdr\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-46csdr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000995eb0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-46csdr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded"}},{"row_id":"658a42cedc0dc2aae205c25f1680a218594adc15e47f3be84b875c536efb2ee9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039742819844755456","occurred_at":"2026-04-02T16:32:39Z","failed_at":"e2e","signature_id":"19436ba271f8ccddea2b7b95384038677714b853540353043c43627ac5bf717b","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc001422a98\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*fmt.wrapError | 0xc000b08900\u003e{\n msg: \"failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xc000b088e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:242]: Unexpected error:\n \u003c*errors.joinError | 0xc001422a98\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*fmt.wrapError | 0xc000b08900\u003e{\n msg: \"failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xc000b088e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"67f5790f010610c3b1673f2520712810b9adec9b1d7925c7dd437a5c02cdf1a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039742819844755456","occurred_at":"2026-04-02T16:32:39Z","failed_at":"e2e","signature_id":"8d6e9bda6bfa788d82973b1855491ce29cd04bb9e57d1c17446bd55941bf8d68","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00076c140\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-lfmp6.apps.aro.p1p6p1b0v9r5h3k.f9ik.j4755456.hcp.osadev.cloud\": dial tcp 4.236.7.70:443: connect: connection timed out\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-lfmp6.apps.aro.p1p6p1b0v9r5h3k.f9ik.j4755456.hcp.osadev.cloud\\\": dial tcp 4.236.7.70:443: connect: connection timed out\",\n err: \u003c*url.Error | 0xc00173a120\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-lfmp6.apps.aro.p1p6p1b0v9r5h3k.f9ik.j4755456.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc00087a0a0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: \u003c*net.TCPAddr | 0xc00173a540\u003e{IP: [4, 236, 7, 70], Port: 443, Zone: \"\"},\n Err: \u003c*os.SyscallError | 0xc00076c080\u003e{\n Syscall: \"connect\",\n Err: \u003csyscall.Errno\u003e0x6e,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00076c140\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-lfmp6.apps.aro.p1p6p1b0v9r5h3k.f9ik.j4755456.hcp.osadev.cloud\": dial tcp 4.236.7.70:443: connect: connection timed out\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-lfmp6.apps.aro.p1p6p1b0v9r5h3k.f9ik.j4755456.hcp.osadev.cloud\\\": dial tcp 4.236.7.70:443: connect: connection timed out\",\n err: \u003c*url.Error | 0xc00173a120\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-lfmp6.apps.aro.p1p6p1b0v9r5h3k.f9ik.j4755456.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc00087a0a0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: \u003c*net.TCPAddr | 0xc00173a540\u003e{IP: [4, 236, 7, 70], Port: 443, Zone: \"\"},\n Err: \u003c*os.SyscallError | 0xc00076c080\u003e{\n Syscall: \"connect\",\n Err: \u003csyscall.Errno\u003e0x6e,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2756b831e4b1e0a56a7949bf61fab69ac22501180f2bc329512550fb97dec7fa","failure_pattern":"route host was never found: Get \"\u003curl\u003e\": dial tcp \u003cip\u003e:\u003cport\u003e: connect: connection timed out","search_query":"route host was never found: Get \"https://agnhost-e2e-serving-app-lfmp6.apps.aro.p1p6p1b0v9r5h3k.f9ik.j4755456.hcp.osadev.cloud\": dial tcp 4.236.7.70:443: connect: connection timed out"}},{"row_id":"6cd77fd50b19b97f0b502ce6d689bd50cb43c753a858c4bebbef87bce3ad47f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039742819844755456","occurred_at":"2026-04-02T16:32:39Z","failed_at":"e2e","signature_id":"993e68d0fcbd786ebc0f1d6cfff6d41ed001203f3430aa10b3cdee7cd214bf66","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=) dns (Available=False) image-registry (Available=False) ingress (Available=False) insights (Available=) kube-storage-version-migrator (Available=) monitoring (Available=) openshift-samples (Available=) service-ca (Available=) storage (Available=False)]\nExpected\n \u003c[]string | len:10, cap:16\u003e: [\n \"console (Available=)\",\n \"dns (Available=False)\",\n \"image-registry (Available=False)\",\n \"ingress (Available=False)\",\n \"insights (Available=)\",\n \"kube-storage-version-migrator (Available=)\",\n \"monitoring (Available=)\",\n \"openshift-samples (Available=)\",\n \"service-ca (Available=)\",\n \"storage (Available=False)\",\n ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:390]: Timed out after 600.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:389 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=) dns (Available=False) image-registry (Available=False) ingress (Available=False) insights (Available=) kube-storage-version-migrator (Available=) monitoring (Available=) openshift-samples (Available=) service-ca (Available=) storage (Available=False)]\nExpected\n \u003c[]string | len:10, cap:16\u003e: [\n \"console (Available=)\",\n \"dns (Available=False)\",\n \"image-registry (Available=False)\",\n \"ingress (Available=False)\",\n \"insights (Available=)\",\n \"kube-storage-version-migrator (Available=)\",\n \"monitoring (Available=)\",\n \"openshift-samples (Available=)\",\n \"service-ca (Available=)\",\n \"storage (Available=False)\",\n ]\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"58a86e086c65a78dcb36b50f3eca98218bf47bbeb0f39c18697842342bd27ed3","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [console (Available=) dns (Available=False) image-registry (Available=False) ingress (Available=False) insights (Available=) kube-storage-version-migrator (Available=) monitoring","search_query":"All ClusterOperators should report Available=True, but these are not available: [console (Available=) dns (Available=False) image-registry (Available=False) ingress (Available=False) insights (Available=) kube-storage-version-migrator (Available=) monitoring (Available=) openshift-samples (Available=) service-ca (Available=) storage (Available=False)]"}},{"row_id":"887f80f8b4e49d1a3c132d34d937044eca4cf8180b16a95b92212cfe3fd85eca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039742819844755456","occurred_at":"2026-04-02T16:32:39Z","failed_at":"e2e","signature_id":"aa149d7074f6cf6beaa7c5d3c32740d8820972b15b39e1de4c6fb012ec681e94","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:60]: Timed out after 600.000s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00096ab80\u003e: \n subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)\n {\n s: \"subscription openshift-nfd/nfd not ready, state=\\\"\\\" (OLM still processing, no errors detected)\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:60]: Timed out after 600.000s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00096ab80\u003e: \n subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)\n {\n s: \"subscription openshift-nfd/nfd not ready, state=\\\"\\\" (OLM still processing, no errors detected)\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0396910b5edad3e7866fd28fb9eba06691cf430164fe400fccf6d1a093f510c","failure_pattern":"subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)","search_query":"subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)"}},{"row_id":"a54f4d9546195e95bc7a53287ae5905ab5e245b603bb13f8f7fcffc3b4ce686e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039742819844755456","occurred_at":"2026-04-02T16:32:39Z","failed_at":"e2e","signature_id":"672feff3fd40385dbcc3229d6c8d721047c4597e440d11800f3fdf3358197ac4","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:180]: Timed out after 600.007s.\nrouter-default deployment logs should be fetchable\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0004bb040\u003e: \n deployment router-default -n openshift-ingress has no running pods\n {\n s: \"deployment router-default -n openshift-ingress has no running pods\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:180]: Timed out after 600.007s.\nrouter-default deployment logs should be fetchable\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0004bb040\u003e: \n deployment router-default -n openshift-ingress has no running pods\n {\n s: \"deployment router-default -n openshift-ingress has no running pods\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fb9a7b01d917e5d72de3cdddcbd162293d3f2dd6f8179c30acc49df54eaecd59","failure_pattern":"deployment router-default -n openshift-ingress has no running pods","search_query":"deployment router-default -n openshift-ingress has no running pods"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["2756b831e4b1e0a56a7949bf61fab69ac22501180f2bc329512550fb97dec7fa","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","58a86e086c65a78dcb36b50f3eca98218bf47bbeb0f39c18697842342bd27ed3","d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004","f0396910b5edad3e7866fd28fb9eba06691cf430164fe400fccf6d1a093f510c","fb9a7b01d917e5d72de3cdddcbd162293d3f2dd6f8179c30acc49df54eaecd59"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039742386447323136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4687,"pr_state":"closed","pr_sha":"32f08b48387c1a611b8b59cb96977b2f6374112e","final_merged_sha":"32f08b48387c1a611b8b59cb96977b2f6374112e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T16:30:56Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3c90737a3bf59c678c6e6afbe905814fb81f2bf4f3544b279a114ff9fe55d452","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039742386447323136","occurred_at":"2026-04-02T16:30:56Z","failed_at":"e2e","signature_id":"35b3288975621ccf376af36cee6d25f11bfd5c12fedcc3c3e4fd985dd4fd20b9","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:206]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00090ba70\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-hs6rh7\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hs6rh7, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-hs6rh7\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hs6rh7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010b9110\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hs6rh7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:206]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00090ba70\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-hs6rh7\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hs6rh7, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-hs6rh7\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hs6rh7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010b9110\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hs6rh7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039737575685492736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4630,"pr_state":"closed","pr_sha":"708735960ddead74079dcd0f64ffa9f51e648a99","final_merged_sha":"7e07b7c9223871a196c30b61870b1a0f20ef1025","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-02T16:11:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039733043169595392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3973,"pr_state":"closed","pr_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","final_merged_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T15:53:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039724880844296192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4651,"pr_state":"closed","pr_sha":"6d025920a926606b0cf9684cc84a473cb8e69ee0","final_merged_sha":"6d025920a926606b0cf9684cc84a473cb8e69ee0","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T15:21:22Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a12d7c2f43544c9b268b680d37c7da9f51eb6fd2fc33a8f1ae2238133d4d2127","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039724880844296192","occurred_at":"2026-04-02T15:21:22Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-04-02T15:45:11.707Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-04-02T15:45:13.158Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-04-02T15:45:15.083Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=0954c7b8221d8b802b1789f3b78b55cab36ed52110ea91016298121b353b80b4\ntime=2026-04-02T16:00:18.519Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4296192-svc/providers/Microsoft.Resources/deployments/0954c7b8221d8b802b1789f3b78b55cab36ed52110ea91016298121b353b80b4/operationStatuses/08584264621714434367\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4296192-svc/providers/Microsoft.Resources/deployments/metrics-infra\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3960/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039724865174376448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3960,"pr_state":"closed","pr_sha":"b65c3a633c23914dd7243b1a0245bb0be24e2387","final_merged_sha":"b65c3a633c23914dd7243b1a0245bb0be24e2387","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T15:21:18Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a4ca6d7f6a5a171c77ef5c8d1aae3a360af8a6efbc4635d43debe07d6ff7e9cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3960/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039724865174376448","occurred_at":"2026-04-02T15:21:18Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-04-02T15:46:48.325Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-04-02T15:46:50.935Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-04-02T15:46:52.408Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=11f5679ed3673e4a9929371cce32c945149645404637979cf67d14076078d9bb\ntime=2026-04-02T16:02:26.220Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4376448-svc/providers/Microsoft.Resources/deployments/11f5679ed3673e4a9929371cce32c945149645404637979cf67d14076078d9bb/operationStatuses/08584264620738937229\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/aks-svc-kv-access-frontend-cert-prow-j4376448\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/aks-svc-kv-access-admin-api-cert-prow-j4376448\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/aksSPCRead-sessiongate-cert-prow-j4376448\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4376448-svc/providers/Microsoft.Resources/deployments/metrics-infra\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039724856378920960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4601,"pr_state":"closed","pr_sha":"eec30d3130530d4de45de55d7a0842d3ecaf04cc","final_merged_sha":"d4656bcc056e17f94a38d7d26dc82809bf007acc","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T15:21:16Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ceda4205d1ff8f454d8f2a3cdb92976b30663232a8b61316796a716ee03d306a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039724856378920960","occurred_at":"2026-04-02T15:21:16Z","failed_at":"provision","signature_id":"0cfdbc69e36cd9409140a1d94f3eca54c19e2976278c5f5c45b1651fb5c59fcf","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster","test_suite":"step graph","failure_text":"time=2026-04-02T15:47:34.702Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-04-02T15:47:35.558Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster\ntime=2026-04-02T15:47:36.998Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=a42c05c78c4bc79396dff56a02b6117a0992c06d730dc45d2f0f97c73701fdea\ntime=2026-04-02T15:59:09.654Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8920960-mgmt-1/providers/Microsoft.Resources/deployments/a42c05c78c4bc79396dff56a02b6117a0992c06d730dc45d2f0f97c73701fdea/operationStatuses/08584264620293070923\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8920960-mgmt-1/providers/Microsoft.Resources/deployments/cluster-4f5djyc2g2fjc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/1ddf10d9-e1b0-5dfe-8627-4a983a2a3ab1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/1ddf10d9-e1b0-5dfe-8627-4a983a2a3ab1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/d4072e7f-2678-5f20-830e-54226531e4b5\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/d4072e7f-2678-5f20-830e-54226531e4b5\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/388c60cd-66c0-5a3c-bf90-1d1bb63fb700\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/388c60cd-66c0-5a3c-bf90-1d1bb63fb700\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/2c104537-ef93-5756-94dd-a4f0b7a92d1f\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/2c104537-ef93-5756-94dd-a4f0b7a92d1f\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4737/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039721413132161024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4737,"pr_state":"closed","pr_sha":"746ba3b55d1f9fd1376d1584ed8fdb8331877f1c","final_merged_sha":"746ba3b55d1f9fd1376d1584ed8fdb8331877f1c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T15:07:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039715683889057792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4485,"pr_state":"closed","pr_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","final_merged_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T14:44:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039715299619508224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4513,"pr_state":"open","pr_sha":"2070c401b94c3f9d55b6f7bcbda94bb637be5832","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-02T14:43:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039712383005364224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"e3a51ad92ae7729a53e8ce5885c7bce2bdfe554a","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T14:31:42Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4b18194676332c6442e9cfa777b4c0240a139771048b77811440f15b1589a404","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039712383005364224","occurred_at":"2026-04-02T14:31:42Z","failed_at":"e2e","signature_id":"d9b20c2ebd50ec8a3101b0c6aa92002433a2c2a0928bd78a95d5c8e6c3808cf2","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000db6520\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud on 172.30.0.10:53: no such host\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud\\\": dial tcp: lookup agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud on 172.30.0.10:53: no such host\",\n err: \u003c*url.Error | 0xc0008de960\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000b10500\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xc000b10370\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000db6520\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud on 172.30.0.10:53: no such host\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud\\\": dial tcp: lookup agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud on 172.30.0.10:53: no such host\",\n err: \u003c*url.Error | 0xc0008de960\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000b10500\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xc000b10370\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"agnhost-e2e-serving-app-g6d2g.apps.aro.d7t4w3c7c0g3t5g.p107.j5364224.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd93f6786bdbe9aeb08610dcc4ec01a349d1a8e658826e9448f8193966d5a67f","failure_pattern":"route host was never found: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"route host was never found: Get \"https://agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud on 172.30.0.10:53: no such host"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fd93f6786bdbe9aeb08610dcc4ec01a349d1a8e658826e9448f8193966d5a67f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4706/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039710625877200896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4706,"pr_state":"closed","pr_sha":"b1ebea216f17cd14d1c5fa7d0361d70642b63c94","final_merged_sha":"b1ebea216f17cd14d1c5fa7d0361d70642b63c94","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T14:24:43Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1dba69a6e8a6b8eb566f7fa846bea7c2eaee1bbf5431a959b5b42b1740d3d68e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4706/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039710625877200896","occurred_at":"2026-04-02T14:24:43Z","failed_at":"e2e","signature_id":"4875a94f01d4f6adb9fd5e88ecab7e303303b8df54acdcf02d39c52f6fb9940f","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e2e4a0\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud on 172.30.0.10:53: no such host\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud\\\": dial tcp: lookup agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud on 172.30.0.10:53: no such host\",\n err: \u003c*url.Error | 0xc0008d4300\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000b60eb0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xc000a84550\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e2e4a0\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud on 172.30.0.10:53: no such host\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud\\\": dial tcp: lookup agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud on 172.30.0.10:53: no such host\",\n err: \u003c*url.Error | 0xc0008d4300\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000b60eb0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xc000a84550\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd93f6786bdbe9aeb08610dcc4ec01a349d1a8e658826e9448f8193966d5a67f","failure_pattern":"route host was never found: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"route host was never found: Get \"https://agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-d7kh6.apps.aro.g0h7v0y6q1t2m7r.un2z.j7200896.hcp.osadev.cloud on 172.30.0.10:53: no such host"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fd93f6786bdbe9aeb08610dcc4ec01a349d1a8e658826e9448f8193966d5a67f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039709396371509248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4739,"pr_state":"closed","pr_sha":"14b20b863995be4b1553984df21980f388dfdba1","final_merged_sha":"14b20b863995be4b1553984df21980f388dfdba1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T14:19:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4743/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039704782217678848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4743,"pr_state":"closed","pr_sha":"6d1ff15f39e8be519e6ce9b5998586dbc2c332dd","final_merged_sha":"80cdcc675989f7dcc63a578af3e13b5bd81f6807","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T14:01:18Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"efff68b2847bf87ebd5bb232abf0cefc13d365518671f93b10be1f1f6036381f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4743/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039704782217678848","occurred_at":"2026-04-02T14:01:18Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4235/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039704421859856384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4235,"pr_state":"closed","pr_sha":"fea5b77c9fb0f5ab605e84f05f5890edc8162210","final_merged_sha":"fea5b77c9fb0f5ab605e84f05f5890edc8162210","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T14:00:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4741/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039704065788612608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4741,"pr_state":"closed","pr_sha":"80a5f12065160304bbd1d53f7b889648d9535260","final_merged_sha":"80a5f12065160304bbd1d53f7b889648d9535260","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T13:58:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039703704566763520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4742,"pr_state":"closed","pr_sha":"647f7f32a745d3996fab56bbb1e96f2d20376ace","final_merged_sha":"1bfd250498d2614c6267b766c8617d348665ea23","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T13:57:13Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"13f56374bd0e3a87a29b2081bb96bc238e6f1107d4100e7d5ba6247c3b547659","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039703704566763520","occurred_at":"2026-04-02T13:57:13Z","failed_at":"e2e","signature_id":"2803e47444e5e6a9fcb1032c0a08fe6bb0e6e9ce6501d0b808de709dbefb9b93","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ce5680\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-4766tv\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0000d8ff0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ce5680\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-4766tv\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0000d8ff0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039703492766994432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4210,"pr_state":"closed","pr_sha":"48bde5e36b8463b9f1ca9657b5862d73cfb4cf98","final_merged_sha":"ef912c114f3891ce814ee96ef6545953f99c7540","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T13:56:23Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e81a6de7a4478c1cb5a8e59e3323151a5656a3455a6d2e08f7e6a851a006d614","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039703492766994432","occurred_at":"2026-04-02T13:56:23Z","failed_at":"e2e","signature_id":"33198b935f93ff373eb31c8a64312c536d6a63f46fadcbf109c8d0850b91168d","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00095fbf0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-m8rft4\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-m8rft4, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-m8rft4\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-m8rft4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000880cf0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-m8rft4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00095fbf0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-m8rft4\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-m8rft4, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-m8rft4\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-m8rft4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000880cf0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-m8rft4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039701224894238720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4700,"pr_state":"closed","pr_sha":"366b50870e118288ce291e6becfb6caf6087d062","final_merged_sha":"f2c81341ce3bebdf6041e299c18aac51d3a3a807","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T13:47:22Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"7a8d78a19aaaaa1b810b1790f38baba421b09ab03c4c75422ba7091e0cebdde1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039701224894238720","occurred_at":"2026-04-02T13:47:22Z","failed_at":"e2e","signature_id":"b39bb9194703dbebc79d05d377030649f08c16e23d5e10e75b3b077805411168","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001888e40\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-62f87x\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-62f87x, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-62f87x\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-62f87x, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b9c70\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-62f87x\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001888e40\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-62f87x\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-62f87x, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-62f87x\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-62f87x, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b9c70\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-62f87x\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded"}},{"row_id":"d8e451766c6832eb32eb27d265ce702d38941f6073a8774ba542e80d26fe54d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039701224894238720","occurred_at":"2026-04-02T13:47:22Z","failed_at":"e2e","signature_id":"1d04d2d454770960498b5fd3a97650ab6b56772fbf3ac90b25e3681fbec855c8","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.000s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00134a1a0\u003e: \n permission checks failed: expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\n expected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\n expected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\n expected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\n expected allowed but denied: list machine.openshift.io/machines (reason: )\n expected allowed but denied: list machine.openshift.io/machinesets (reason: )\n expected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\n {\n msg: \"permission checks failed: expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\\nexpected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\\nexpected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\\nexpected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\\nexpected allowed but denied: list machine.openshift.io/machines (reason: )\\nexpected allowed but denied: list machine.openshift.io/machinesets (reason: )\\nexpected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\",\n err: \u003c*errors.joinError | 0xc0011e71e8\u003e{\n errs: [\n \u003c*errors.errorString | 0xc0009041c0\u003e{\n s: \"expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\",\n },\n \u003c*errors.errorString | 0xc000904390\u003e{\n s: \"expected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\",\n },\n \u003c*errors.errorString | 0xc0009058a0\u003e{\n s: \"expected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\",\n },\n \u003c*errors.errorString | 0xc0007b3da0\u003e{\n s: \"expected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\",\n },\n \u003c*errors.errorString | 0xc000af1e50\u003e{\n s: \"expected allowed but denied: list machine.openshift.io/machines (reason: )\",\n },\n \u003c*errors.errorString | 0xc000905d40\u003e{\n s: \"expected allowed but denied: list machine.openshift.io/machinesets (reason: )\",\n },\n \u003c*errors.errorString | 0xc0007b3f50\u003e{\n s: \"expected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\",\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.000s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00134a1a0\u003e: \n permission checks failed: expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\n expected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\n expected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\n expected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\n expected allowed but denied: list machine.openshift.io/machines (reason: )\n expected allowed but denied: list machine.openshift.io/machinesets (reason: )\n expected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\n {\n msg: \"permission checks failed: expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\\nexpected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\\nexpected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\\nexpected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\\nexpected allowed but denied: list machine.openshift.io/machines (reason: )\\nexpected allowed but denied: list machine.openshift.io/machinesets (reason: )\\nexpected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\",\n err: \u003c*errors.joinError | 0xc0011e71e8\u003e{\n errs: [\n \u003c*errors.errorString | 0xc0009041c0\u003e{\n s: \"expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\",\n },\n \u003c*errors.errorString | 0xc000904390\u003e{\n s: \"expected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\",\n },\n \u003c*errors.errorString | 0xc0009058a0\u003e{\n s: \"expected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\",\n },\n \u003c*errors.errorString | 0xc0007b3da0\u003e{\n s: \"expected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\",\n },\n \u003c*errors.errorString | 0xc000af1e50\u003e{\n s: \"expected allowed but denied: list machine.openshift.io/machines (reason: )\",\n },\n \u003c*errors.errorString | 0xc000905d40\u003e{\n s: \"expected allowed but denied: list machine.openshift.io/machinesets (reason: )\",\n },\n \u003c*errors.errorString | 0xc0007b3f50\u003e{\n s: \"expected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\",\n },\n ],\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cada5ab0a8f8fe508d400c0c0ce723760422f5303a8840acbdf56fa572eae5f2","failure_pattern":"permission checks failed: expected allowed but denied: list flowcontrol","search_query":"permission checks failed: expected allowed but denied: list flowcontrol"}},{"row_id":"ff2f517f89edf22fd132868c8594f51a25b31ff64b547cc23a22c9b5a0d4825a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039701224894238720","occurred_at":"2026-04-02T13:47:22Z","failed_at":"e2e","signature_id":"04a5961627f5308d36e1c79e1a26c4ec78a544e15c55c0c00b42378420838bf8","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014951a0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-6f5vct\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6f5vct, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-6f5vct\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6f5vct, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000690430\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6f5vct\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014951a0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-6f5vct\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6f5vct, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-6f5vct\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6f5vct, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000690430\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6f5vct\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","cada5ab0a8f8fe508d400c0c0ce723760422f5303a8840acbdf56fa572eae5f2"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4735/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039688475782942720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4735,"pr_state":"closed","pr_sha":"bd6ef88ac059053b5e216e21454e3a1ada922893","final_merged_sha":"bd6ef88ac059053b5e216e21454e3a1ada922893","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T12:56:42Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039687619855519744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4652,"pr_state":"closed","pr_sha":"ef0f1b19b5b4a0415c36e782700ec263a51ca311","final_merged_sha":"ef0f1b19b5b4a0415c36e782700ec263a51ca311","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T12:53:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039682959249772544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4740,"pr_state":"closed","pr_sha":"96f1cd077210cd1bae299cb34a1953455493246b","final_merged_sha":"389466a234f4716ad49f8b021a30f1f018b06ea6","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-02T12:32:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039675248269332480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4705,"pr_state":"closed","pr_sha":"b93a17214b37055e25d1b0f0526393a9f62fa9ad","final_merged_sha":"b93a17214b37055e25d1b0f0526393a9f62fa9ad","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T12:04:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4738/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039671529075118080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4738,"pr_state":"closed","pr_sha":"c0686000dfb7e6c66b1b27dc139ca5419e6326a7","final_merged_sha":"c0686000dfb7e6c66b1b27dc139ca5419e6326a7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T11:49:22Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6a1ad496cd4f5aeeafa0fd1abd1518974533f7f186768436da2957561f58e744","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4738/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039671529075118080","occurred_at":"2026-04-02T11:49:22Z","failed_at":"e2e","signature_id":"2eefbb36d6c0435b375525929bc8696890c902a58c5eec09c02b07a5759781b9","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:344]: Unexpected error:\n \u003c*errors.StatusError | 0xc000cc6280\u003e: \n pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range\n {\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods \\\"wi-test-pod\\\" is forbidden: error fetching namespace \\\"e2e-oidc-wi\\\": unable to find annotation openshift.io/sa.scc.uid-range\",\n Reason: \"Forbidden\",\n Details: {Name: \"wi-test-pod\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:344]: Unexpected error:\n \u003c*errors.StatusError | 0xc000cc6280\u003e: \n pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range\n {\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods \\\"wi-test-pod\\\" is forbidden: error fetching namespace \\\"e2e-oidc-wi\\\": unable to find annotation openshift.io/sa.scc.uid-range\",\n Reason: \"Forbidden\",\n Details: {Name: \"wi-test-pod\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"31bab740c670d74d59ae23b10d74f51b9de483710d0d1e65e2cdb24e5ecc9063","failure_pattern":"pod \"\u003cpod\u003e\" is forbidden: error fetching namespace \"\u003cnamespace\u003e\": unable to find annotation openshift.io/sa.scc.uid-range","search_query":"pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["31bab740c670d74d59ae23b10d74f51b9de483710d0d1e65e2cdb24e5ecc9063"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4737/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039669571090124800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4737,"pr_state":"closed","pr_sha":"b82a2272166853a55ec02dde5f564ad6871829ae","final_merged_sha":"746ba3b55d1f9fd1376d1584ed8fdb8331877f1c","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-02T11:41:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039647256398794752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4705,"pr_state":"closed","pr_sha":"b93a17214b37055e25d1b0f0526393a9f62fa9ad","final_merged_sha":"b93a17214b37055e25d1b0f0526393a9f62fa9ad","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T10:12:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039644149858177024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4652,"pr_state":"closed","pr_sha":"ef0f1b19b5b4a0415c36e782700ec263a51ca311","final_merged_sha":"ef0f1b19b5b4a0415c36e782700ec263a51ca311","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T10:00:34Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"565c6dc55d68b9b1afbef2cfda732f5f0dd1139e5ce8558aa06f226f9d6610a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039644149858177024","occurred_at":"2026-04-02T10:00:34Z","failed_at":"e2e","signature_id":"bcda64cf460179f374028f7c7a0128b965940f6a3e1958921cff52e3fc56f02c","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fd3068\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-fgdwpj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a93580\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-fgdwpj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a93560\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-fgdwpj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a93540\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-fgdwpj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbee2c60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fd3068\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-fgdwpj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a93580\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-fgdwpj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a93560\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-fgdwpj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a93540\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-fgdwpj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbee2c60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-s4wxvw\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4702/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039639943415861248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4702,"pr_state":"closed","pr_sha":"6bcc2d5b15ee4c7adf4bfb529dfa6050b90ac0d2","final_merged_sha":"a5e4983ae91a2cfa4173521de153fc43fb6fefa3","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-02T09:43:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039634347752951808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4705,"pr_state":"closed","pr_sha":"b93a17214b37055e25d1b0f0526393a9f62fa9ad","final_merged_sha":"b93a17214b37055e25d1b0f0526393a9f62fa9ad","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T09:21:37Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"889295567dea1857ff9be43dfcba3e753055e0abaabea733fb3fb539fab77c49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039634347752951808","occurred_at":"2026-04-02T09:21:37Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-04-02T09:42:00.307Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-04-02T09:42:02.470Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-04-02T09:42:03.710Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=63e389fe98fd449bd7e9b47c13ebfac931288071ccae0047c27b89b43672b53b\ntime=2026-04-02T09:45:04.543Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j2951808/providers/Microsoft.Resources/deployments/63e389fe98fd449bd7e9b47c13ebfac931288071ccae0047c27b89b43672b53b/operationStatuses/08584264839624194560\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2951808/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2951808/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j2951808\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 899bf80f-8b91-429f-98ec-271a3fd806ee:4/2/2026 9:43:24 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039624524353245184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4507,"pr_state":"closed","pr_sha":"0e19405a0a977d3cd718004121a44142d3aebb42","final_merged_sha":"0e19405a0a977d3cd718004121a44142d3aebb42","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T08:42:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039624090750291968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4734,"pr_state":"closed","pr_sha":"a356adf456c0f2c49961d3cde20a69329235b457","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-02T08:40:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039614767202570240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4604,"pr_state":"closed","pr_sha":"a1dbbe7bf80566749053e62f1021d697e247684a","final_merged_sha":"a1dbbe7bf80566749053e62f1021d697e247684a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T08:03:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039614358316650496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4558,"pr_state":"closed","pr_sha":"1205c8fffd91fbcc73a425533f183d045beea0a7","final_merged_sha":"1205c8fffd91fbcc73a425533f183d045beea0a7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T08:01:22Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039610750393126912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4682,"pr_state":"closed","pr_sha":"883a4bfc68df92609842fcf7f5c60d2c5073ecb9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T07:47:51Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"bafe3b1cc8b6148395fb60a6eae9429656e883740e837dd96ab3899f1284c3be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039610750393126912","occurred_at":"2026-04-02T07:47:51Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4689/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039601823236493312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4689,"pr_state":"closed","pr_sha":"e2cd2dfe1f97c7006536163cdcfac59302de59bd","final_merged_sha":"e2cd2dfe1f97c7006536163cdcfac59302de59bd","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T07:12:23Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039601679703216128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4705,"pr_state":"closed","pr_sha":"b93a17214b37055e25d1b0f0526393a9f62fa9ad","final_merged_sha":"b93a17214b37055e25d1b0f0526393a9f62fa9ad","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T07:11:49Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e9f0118365eec7f92c25c010aba528d6052b3cc8641aada3da2396dfa8352221","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039601679703216128","occurred_at":"2026-04-02T07:11:49Z","failed_at":"e2e","signature_id":"588e37911009e056bfcbdefadbbe3d1697b33856d1487c8eeb46ce416866fa1d","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00080e300\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-6v7grd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-6v7grd/operationStatuses/08584264903894756834\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-basic-hcp-cluster-6v7grd' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-6v7grd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-6v7grd/operationStatuses/08584264903894756834\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-basic-hcp-cluster-6v7grd' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00080e2e0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-6v7grd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-6v7grd/operationStatuses/08584264903894756834\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-basic-hcp-cluster-6v7grd' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00080e2c0\u003e{\n msg: \"failed waiting for deployment \\\"mi-basic-hcp-cluster-6v7grd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-6v7grd/operationStatuses/08584264903894756834\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-basic-hcp-cluster-6v7grd' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000e09d70\u003e{\n ErrorCode: \"DeploymentNotFound\",\n StatusCode: 404,\n RawResponse: {\n Status: \"404 Not Found\",\n StatusCode: 404,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"44b167ab-be32-4a33-9212-0cdd2b323e2c\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"44b167ab-be32-4a33-9212-0cdd2b323e2c\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 07:54:57 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"112\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADAEAST:20260402T075458Z:44b167ab-be32-4a33-9212-0cdd2b323e2c\",\n ],\n \"Stri...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:86]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00080e300\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-6v7grd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-6v7grd/operationStatuses/08584264903894756834\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-basic-hcp-cluster-6v7grd' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-6v7grd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-6v7grd/operationStatuses/08584264903894756834\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-basic-hcp-cluster-6v7grd' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00080e2e0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-6v7grd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-6v7grd/operationStatuses/08584264903894756834\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-basic-hcp-cluster-6v7grd' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00080e2c0\u003e{\n msg: \"failed waiting for deployment \\\"mi-basic-hcp-cluster-6v7grd\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-6v7grd/operationStatuses/08584264903894756834\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-basic-hcp-cluster-6v7grd' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000e09d70\u003e{\n ErrorCode: \"DeploymentNotFound\",\n StatusCode: 404,\n RawResponse: {\n Status: \"404 Not Found\",\n StatusCode: 404,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"44b167ab-be32-4a33-9212-0cdd2b323e2c\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"44b167ab-be32-4a33-9212-0cdd2b323e2c\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 07:54:57 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"112\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADAEAST:20260402T075458Z:44b167ab-be32-4a33-9212-0cdd2b323e2c\",\n ],\n \"Stri...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039600222362931200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4724,"pr_state":"closed","pr_sha":"0075ef144a4d73824e2ebb42ea77f734e97f46b8","final_merged_sha":"0075ef144a4d73824e2ebb42ea77f734e97f46b8","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T07:06:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039591163756810240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4686,"pr_state":"closed","pr_sha":"efc2b626c7c86cd1147c3e68830ebc5afffd7f83","final_merged_sha":"efc2b626c7c86cd1147c3e68830ebc5afffd7f83","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T06:30:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039590314338947072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"0c0f53c0d4500f52dda4381bb688a37231d63b46","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-02T06:26:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4723/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039588637628174336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4723,"pr_state":"closed","pr_sha":"b1ab92bdd1a079a7b6b823287f57b6cb3e0f6733","final_merged_sha":"b1ab92bdd1a079a7b6b823287f57b6cb3e0f6733","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T06:19:59Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"78cc346ae238f90d4304f3e1233c031511fbf1b98f53e250895f4f74fdc7f715","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4723/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039588637628174336","occurred_at":"2026-04-02T06:19:59Z","failed_at":"e2e","signature_id":"effe52f58af32305fd8b8d6f143ac7d28c926476965d09ffe088451e90ca0730","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000978510\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jrg7gs\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jrg7gs, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-jrg7gs\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jrg7gs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008d0da0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jrg7gs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000978510\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jrg7gs\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jrg7gs, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-jrg7gs\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jrg7gs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008d0da0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jrg7gs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4702/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039585436556333056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4702,"pr_state":"closed","pr_sha":"6bcc2d5b15ee4c7adf4bfb529dfa6050b90ac0d2","final_merged_sha":"a5e4983ae91a2cfa4173521de153fc43fb6fefa3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T06:07:16Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3dc7dfca499be3e4177e8be4400f597e45709fc1590486fc42c895b14af424ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4702/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039585436556333056","occurred_at":"2026-04-02T06:07:16Z","failed_at":"e2e","signature_id":"3412c98acbea4eaec56dca07c75fdfbc4d939ff89519731ebd65a4ac774057d3","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b4d890\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-zv4bv4\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zv4bv4, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-zv4bv4\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zv4bv4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008fa930\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zv4bv4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b4d890\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-zv4bv4\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zv4bv4, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-zv4bv4\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zv4bv4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008fa930\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zv4bv4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039583088895332352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4630,"pr_state":"closed","pr_sha":"949f908375ac97f232c7b4f8105222109de75d65","final_merged_sha":"7e07b7c9223871a196c30b61870b1a0f20ef1025","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T05:57:56Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"dc76c97c27f703ca35c6714d84a22d99a5b010d6921111c5894f18a64e5742a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039583088895332352","occurred_at":"2026-04-02T05:57:56Z","failed_at":"e2e","signature_id":"0dcb184869336e72d6b538ab92b556680b561ead5b45d24d4a859e7f75a5c01e","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:206]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010e9230\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-6rgv8k\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6rgv8k, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-6rgv8k\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6rgv8k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001477610\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6rgv8k\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:206]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010e9230\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-6rgv8k\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6rgv8k, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-6rgv8k\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6rgv8k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001477610\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6rgv8k\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-4766tv\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4766tv, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4717/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039579956580716544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4717,"pr_state":"closed","pr_sha":"127d3544e838e6335a790b1e87537ed8e3a439c9","final_merged_sha":"26416847807a241acf58f8fd074b514f61424362","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T05:45:29Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ab76eb03214998d49c1288e292ccad31678774e7eeb21a15d776d6d8cc0c6baa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4717/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039579956580716544","occurred_at":"2026-04-02T05:45:29Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039576512570593280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4507,"pr_state":"closed","pr_sha":"ba4e7e050d351083322c7ce0b35ab855f5b9ba17","final_merged_sha":"0e19405a0a977d3cd718004121a44142d3aebb42","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-02T05:31:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039572528229257216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4730,"pr_state":"closed","pr_sha":"3f5edbc77a965d57ce9fc10be9ea9e5d6c520fa1","final_merged_sha":"c9f07cdaf77afc3c4d41fed584593740f5f63fcc","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-02T05:15:58Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"51168f4c15ccef71c72d754153b49b6c4d8f2f946d59fe4fda07970fb18ac345","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039572528229257216","occurred_at":"2026-04-02T05:15:58Z","failed_at":"e2e","signature_id":"2b31e8a3dca9200ac8185a15ea18e1b9f04bdc7372ded9b3e499c40f5a53a5a3","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0012142a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-s4wxvw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ee4ee0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-s4wxvw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ee4ec0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-s4wxvw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ee4ea0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-s4wxvw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbedbc60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0012142a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-s4wxvw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ee4ee0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-s4wxvw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ee4ec0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-s4wxvw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ee4ea0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-s4wxvw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbedbc60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-s4wxvw\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039502556765360128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4647,"pr_state":"closed","pr_sha":"cc6cdef062c74786d4f8c17830a6093b07d969fb","final_merged_sha":"cc6cdef062c74786d4f8c17830a6093b07d969fb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T00:37:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4664/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039501812549029888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4664,"pr_state":"closed","pr_sha":"a319b384ee60f5e4d6bd446b908fd1bb0dfd5648","final_merged_sha":"a319b384ee60f5e4d6bd446b908fd1bb0dfd5648","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T00:34:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4723/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039501740646076416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4723,"pr_state":"closed","pr_sha":"b1ab92bdd1a079a7b6b823287f57b6cb3e0f6733","final_merged_sha":"b1ab92bdd1a079a7b6b823287f57b6cb3e0f6733","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T00:34:38Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7f25c2c5defde2fa17bddfa68526bf1950d0421adb7b337dbd452258e1543ae7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4723/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039501740646076416","occurred_at":"2026-04-02T00:34:38Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039501099081142272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4133,"pr_state":"closed","pr_sha":"4f10ef55a408ea39a4f644c08d4ab2d01cf39c2a","final_merged_sha":"4f10ef55a408ea39a4f644c08d4ab2d01cf39c2a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T00:31:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4729/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2039500289257508864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4729,"pr_state":"closed","pr_sha":"88514fc9ff46e759b5d1f2264a7435aa4f659f8b","final_merged_sha":"88514fc9ff46e759b5d1f2264a7435aa4f659f8b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T00:28:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}